diff --git a/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/manifest.txt b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/manifest.txt new file mode 100644 index 0000000000000000000000000000000000000000..53af5782306e9a8d4cf982a07c00e14f795aabd3 --- /dev/null +++ b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/manifest.txt @@ -0,0 +1,19 @@ +timestamp: 2026-09-21T08:23:17+00:00 +host: sp-aiinfra-interactive-aghosh-8-mb99k-8xb200-moe-cg-0 +gpu: NVIDIA B200 x8 +driver: 580.95.05 +study_scripts_branch: granularity-echo +study_scripts_commit: b8bd100 +campaign: echo +backend: echo +phase: tierbprobe +gbs: 8 +seq: 4096 +configs: G1 G2 G3 G4 +capacity: 1.25 +repeats: 3 +nsys: 1 +megatron_repo: /workspace/Megatron-MoE-Echo +megatron_branch_required: token-drop-analysis +megatron_branch_actual: token-drop-analysis +megatron_commit: b099fd84 diff --git a/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/pip_freeze.txt b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/pip_freeze.txt new file mode 100644 index 0000000000000000000000000000000000000000..4a5dfe1c1d96cdd9f602f2679d5f5a722d5c2afb --- /dev/null +++ b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/pip_freeze.txt @@ -0,0 +1,317 @@ +absl-py==2.3.0 +aiohappyeyeballs @ file:///rapids/aiohappyeyeballs-2.6.1-py3-none-any.whl#sha256=f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8 +aiohttp @ file:///rapids/aiohttp-3.12.7-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=94f98e0e5a49f89b252e115844f756c04fc8050f38252a32a3dd994ce8121f10 +aiosignal @ file:///rapids/aiosignal-1.3.2-py2.py3-none-any.whl#sha256=45cde58e409a301715980c2b01d0c28bdde3770d8290b5eb2173759d9acb31a5 +annotated-types==0.7.0 +anyio==4.9.0 +apex @ file:///opt/pytorch/apex/dist/apex-0.1-cp312-cp312-linux_x86_64.whl#sha256=a1508e4a386362e9c8f46c92ccada9110e421212b964c694235d0fd5a224b170 +argon2-cffi==25.1.0 +argon2-cffi-bindings==21.2.0 +arrow==1.3.0 +asciitree @ file:///rapids/asciitree-0.3.3-py3-none-any.whl#sha256=3e507ccfaeb417823158091fdb9d26b0205f6a59ba680b479a71db65c73d01d4 +asttokens==3.0.0 +astunparse==1.6.3 +async-lru==2.0.5 +attrs==25.3.0 +audioread==3.0.1 +babel==2.17.0 +beautifulsoup4==4.13.4 +black==25.1.0 +bleach==6.2.0 +blis==0.7.11 +cachetools @ file:///rapids/cachetools-6.0.0-py3-none-any.whl#sha256=82e73ba88f7b30228b5507dce1a1f878498fc669d972aef2dde4f3a3c24f103e +catalogue==2.0.10 +certifi==2025.4.26 +cffi==1.17.1 +charset-normalizer==3.4.2 +click==8.5.0 +cloudpathlib==0.21.1 +cloudpickle @ file:///rapids/cloudpickle-3.1.1-py3-none-any.whl#sha256=c8c5a44295039331ee9dad40ba100a9c7297b6f988e50e87ccdf3765a668350e +cmake==3.31.6 +comm==0.2.2 +confection==0.1.5 +contourpy==1.3.2 +cuda-bindings @ file:///rapids/cuda_bindings-12.9.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=ff0e28d1e34758654b9c961e1f55e4786e49aee6a4dbceaf3cc24c46c672df7e +cuda-python @ file:///rapids/cuda_python-12.9.0-py3-none-any.whl#sha256=926acba49b2c0a0374c61b7c98f337c085199cf51cdfe4d6423c4129c20547a7 +cudf @ file:///rapids/cudf-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b779a8e8c0e8577b9b560895958496da211bfac49cdc4f11a49e8d18cb82e348 +cudf-polars @ file:///rapids/cudf_polars-25.4.0-py3-none-any.whl#sha256=227b6c5147e93ea86282c8f7219118f916769fa2ca0cdd931e97a148aa66b869 +cugraph @ file:///rapids/cugraph-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b837c2dc2f7b235c702e4f8df64479b8fcbb342f848516d5faf0cdc4b5f31f29 +cugraph-service-client @ file:///rapids/cugraph_service_client-25.4.0-py3-none-any.whl#sha256=7e568c3e41154b9752f5173555ffcc282ac52d72343feeb79653b2e2b1232470 +cugraph-service-server @ file:///rapids/cugraph_service_server-25.4.0-py3-none-any.whl#sha256=a2830dcffafcccb16e12ebf85b98f8f247869d38c01ab4dd8664a1e7439d31be +cuml @ file:///rapids/cuml-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=1e90a464a18b094e24797f1676688c5d7696c4cf97c934504214a53d5aa09aa5 +cupy-cuda12x @ file:///rapids/cupy_cuda12x-13.3.0-cp312-cp312-linux_x86_64.whl#sha256=6a2dfa0aa1e8189a10d1534276c08f1d2f73b747fd28bfbe7b9f4ef66d9af73f +cuvs @ file:///rapids/cuvs-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b12f6b9f80e053a3b38542c2947d2516e468c02cad54a13d8b1a4a61d5aa7914 +cycler==0.12.1 +cymem==2.0.11 +Cython==3.1.1 +dask @ file:///rapids/dask-2025.2.0-py3-none-any.whl#sha256=f0fdeef6ceb0a06569d456c9e704f220f7f54e80f3a6ea42ab98cea6bc642b6e +dask-cuda @ file:///rapids/dask_cuda-25.4.0-py3-none-any.whl#sha256=442d87da0a95d00ca2a262b64c76e785a32ad905c82000fa2227932f81390e19 +dask-cudf @ file:///rapids/dask_cudf-25.4.0-py3-none-any.whl#sha256=5e7cea0b26daea17da6e0bb457435e24460f76ba740358a3a18e239d6015c95c +datasets==5.0.1 +debugpy==1.8.14 +decorator==5.2.1 +-e git+https://github.com/deepseek-ai/DeepEP.git@34152ae28f80bcc3ee38d7a12cb2ad87cfd4ea72#egg=deep_ep +defusedxml==0.7.1 +dill==0.4.1 +distributed @ file:///rapids/distributed-2025.2.0-py3-none-any.whl#sha256=368462084be47092d0a835f6c64c9b852da64dc88302b76d5f93bd5131228c67 +distributed-ucxx @ file:///rapids/distributed_ucxx-0.43.0-py3-none-any.whl#sha256=94f60d0ccb0c94ba8dbfa1eaf455874d18aaab17e2ae218c38590e9fdc2d721d +dm-tree==0.1.9 +einops==0.8.1 +execnet==2.1.1 +executing==2.2.0 +expecttest==0.3.0 +fasteners @ file:///rapids/fasteners-0.19-py3-none-any.whl#sha256=758819cb5d94cdedf4e836988b74de396ceacb8e2794d21f82d131fd9ee77237 +fastjsonschema==2.21.1 +fastrlock @ file:///rapids/fastrlock-0.8.3-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_28_x86_64.whl#sha256=5f13ec08f1adb1aa916c384b05ecb7dbebb8df9ea81abd045f60941c6283a670 +filelock==3.18.0 +flash_attn @ file:///opt/pytorch/flash_attn-2.7.4.post1-cp312-cp312-linux_x86_64.whl#sha256=a10455d6c56b387785b7413fd0bbcb392b42b3d4f4b88fda5d86cc8a9ce9182a +fonttools==4.58.1 +fqdn==1.5.1 +frozenlist @ file:///rapids/frozenlist-1.6.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=4e1be9111cb6756868ac242b3c2bd1f09d9aea09846e4f5c23715e7afb647103 +fsspec==2025.5.1 +gast==0.6.0 +grpcio==1.62.1 +h11==0.16.0 +hf-xet==1.6.0 +httpcore==1.0.9 +httpx==0.28.1 +huggingface_hub==1.32.0 +hypothesis==6.130.8 +idna==3.10 +importlib_metadata==8.7.0 +iniconfig==2.1.0 +intel-openmp==2021.4.0 +ipykernel==6.29.5 +ipython==9.3.0 +ipython_pygments_lexers==1.1.1 +isoduration==20.11.0 +isort==6.0.1 +jedi==0.19.2 +Jinja2==3.1.6 +joblib==1.5.1 +json5==0.12.0 +jsonpointer==3.0.0 +jsonschema==4.24.0 +jsonschema-specifications==2025.4.1 +jupyter-events==0.12.0 +jupyter-lsp==2.2.5 +jupyter_client==8.6.3 +jupyter_core==5.8.1 +jupyter_server==2.16.0 +jupyter_server_terminals==0.5.3 +jupyterlab==4.4.3 +jupyterlab_code_formatter==3.0.2 +jupyterlab_pygments==0.3.0 +jupyterlab_server==2.27.3 +jupyterlab_tensorboard_pro @ file:///builder/jupyterlab_tensorboard_pro-4.0.0-py2.py3-none-any.whl#sha256=6bfcb5ced3358eb72150f4b2ab91312faef2f0ca14d845e8379d8fa52df6902e +jupytext==1.17.2 +kiwisolver==1.4.8 +kvikio @ file:///rapids/kvikio-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=8b40fd8ea9e4a2974c04b50b7fc26b06094ac807f9a233281d0bafbaa71b1d85 +langcodes==3.5.0 +language_data==1.3.0 +lazy_loader==0.4 +libcudf @ file:///rapids/libcudf-25.4.0-py3-none-linux_x86_64.whl#sha256=238320b4958c1737bba2a609673e2539a78216b811b5498a68e00395d9b021a4 +libcugraph @ file:///rapids/libcugraph-25.4.0-py3-none-linux_x86_64.whl#sha256=29085e901b2f7569ff2653bb1355fe6fad20f2fcaa585f5649da9867d8df766d +libcuml @ file:///rapids/libcuml-25.4.0-py3-none-linux_x86_64.whl#sha256=8e2e444c72b0e995e15a48da37ff12fb410a47a2baa69ba86ccc9b1a3db91e8c +libcuvs @ file:///rapids/libcuvs-25.4.0-py3-none-linux_x86_64.whl#sha256=e2f8db61c92b769c590dd752f43439ddd62b2f6592a3c337362b6c1ef220c3b9 +libkvikio @ file:///rapids/libkvikio-25.4.0-py3-none-linux_x86_64.whl#sha256=da8a5713b05fc478bf40529a3e0fb61a2d605bb5a70ca4d461f96a6005dbe8b7 +libraft @ file:///rapids/libraft-25.4.0-py3-none-linux_x86_64.whl#sha256=b921dec005aa4a321dd156776a18376feb47c96a5b67eec125c8a8e19edd1e24 +librmm @ file:///rapids/librmm-25.4.0-py3-none-linux_x86_64.whl#sha256=ff951811bda729d4fa1109ab906225ae1f489a6ec2eeab5d1b674cc34bd4422f +librmm-cu12 @ file:///rapids/librmm_cu12-25.4.0-py3-none-any.whl#sha256=29296ef2b0e90679a0db280e86015be12d6ab9f4e62a181aa085f62dbadd06b3 +librosa==0.11.0 +libucxx @ file:///rapids/libucxx-0.43.0-py3-none-linux_x86_64.whl#sha256=f15849d7ac2fb68f0fd8998838a13321ddcfb0c594f7e6cbc401a3bcc4750566 +lightning-thunder @ file:///opt/pytorch/lightning-thunder +lightning-utilities==0.14.3 +lintrunner==0.12.7 +llvmlite==0.42.0 +locket @ file:///rapids/locket-1.0.0-py2.py3-none-any.whl#sha256=b6c819a722f7b6bd955b80781788e4a66a55628b858d347536b7e81325a3a5e3 +looseversion==1.3.0 +marisa-trie==1.2.1 +Markdown==3.8 +markdown-it-py==3.0.0 +MarkupSafe==3.0.2 +matplotlib==3.10.3 +matplotlib-inline==0.1.7 +mdit-py-plugins==0.4.2 +mdurl==0.1.2 +-e git+ssh://git@github.com/sriram-17-17/my-Megatron-MoE-Echo.git@b099fd8494b09e8c08ad34e26abf91fb6c83636d#egg=megatron_core +mistune==3.1.3 +mkl==2021.1.1 +mkl-devel==2021.1.1 +mkl-include==2021.1.1 +ml_dtypes==0.6.0 +mock==5.2.0 +-e git+ssh://git@github.com/sriram-17-17/MoonEP.git@d8bc6667fad7177e6e09b637d773fa67f08bae93#egg=moonep +mpmath==1.3.0 +msgpack==1.1.0 +multidict @ file:///rapids/multidict-6.4.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=a920f9cf2abdf6e493c519492d892c362007f113c94da4c239ae88429835bad1 +multiprocess==0.70.19 +murmurhash==1.0.13 +mypy_extensions==1.1.0 +nbclient==0.10.2 +nbconvert==7.16.6 +nbformat==5.10.4 +nest-asyncio==1.6.0 +networkx==3.5 +ninja==1.11.1.4 +notebook==7.4.3 +notebook_shim==0.2.4 +numba==0.59.1 +numba-cuda @ file:///rapids/numba_cuda-0.4.0-py3-none-any.whl#sha256=5b95a5f42068771e37142944d44fe844fa87f1b2f89051461a4cb0af91bcc28a +numcodecs @ file:///rapids/numcodecs-0.13.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=da2230484e6102e5fa3cc1a5dd37ca1f92dfbd183d91662074d6f7574e3e8f53 +numpy==1.26.4 +nvdlfw_inspect @ git+https://github.com/NVIDIA/nvidia-dlfw-inspect.git@6b60eb0e675606fb2fbbfcb12667ecbec75eaf87 +nvfuser==0.2.27a0+9bf5aca +nvidia-cuda-cccl-cu12==12.9.27 +nvidia-cudnn-frontend @ file:///opt/pytorch/lightning-thunder/tmp/cudnn_frontend +nvidia-cutlass-dsl==4.4.2 +nvidia-cutlass-dsl-libs-base==4.4.2 +nvidia-dali-cuda120==1.50.0 +nvidia-ml-py @ file:///rapids/nvidia_ml_py-12.575.51-py3-none-any.whl#sha256=eb8641800d98ce40a22f479873f34b482e214a7e80349c63be51c3919845446e +nvidia-modelopt==0.29.0 +nvidia-modelopt-core==0.29.0 +nvidia-nvcomp-cu12==4.2.0.14 +nvidia-nvimgcodec-cu12==0.5.0.13 +nvidia-nvjpeg-cu12==12.4.0.16 +nvidia-nvjpeg2k-cu12==0.8.1.40 +nvidia-nvshmem-cu12==3.7.2 +nvidia-nvtiff-cu12==0.5.0.67 +nvidia-resiliency-ext==0.4.0 +nvtx @ file:///rapids/nvtx-0.2.11-cp312-cp312-linux_x86_64.whl#sha256=73521e37cba42b4db0b3cd68fc5eb814e55c500a61eeed6edcdf6746315ac53e +nx-cugraph @ file:///rapids/nx_cugraph-25.4.0-py3-none-any.whl#sha256=e625444486cc899a77bd48e7da76a2edd95c0006dd95fbf323745feae7f33111 +onnx @ file:///opt/pytorch/pytorch/third_party/onnx +onnx-ir==1.0.0 +onnxscript==0.7.2 +opt_einsum==3.4.0 +optree==0.16.0 +overrides==7.7.0 +packaging==23.2 +pandas @ file:///rapids/pandas-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=fffb8ae78d8af97f849404f21411c95062db1496aeb3e56f146f0355c9989319 +pandocfilters==1.5.1 +parso==0.8.4 +partd @ file:///rapids/partd-1.4.2-py3-none-any.whl#sha256=978e4ac767ec4ba5b86c6eaa52e5a2a3bc748a2ca839e8cc798f1cc6ce6efb0f +pathspec==0.12.1 +pexpect==4.9.0 +pillow==11.2.1 +platformdirs==4.3.8 +pluggy==1.6.0 +ply @ file:///rapids/ply-3.11-py2.py3-none-any.whl#sha256=096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce +polars @ file:///rapids/polars-1.25.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=f7fcbb4f476784384ccda48757fca4e8c2e2c5a0a3aef3717aaf56aee4e30e09 +polygraphy==0.49.20 +pooch==1.8.2 +preshed==3.0.10 +prometheus_client==0.22.1 +prompt_toolkit==3.0.51 +propcache @ file:///rapids/propcache-0.3.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=563f9d8c03ad645597b8d010ef4e9eab359faeb11a0a2ac9f7b4bc8c28ebef25 +protobuf==4.24.4 +psutil==7.0.0 +ptyprocess==0.7.0 +PuLP==3.2.1 +pure_eval==0.2.3 +pyarrow==25.0.1 +pybind11==2.13.6 +pybind11_global==2.13.6 +pycocotools @ git+https://github.com/nvidia/cocoapi.git@08350a157b41f0755a536441b5abec89ab934c5b#subdirectory=PythonAPI +pycparser==2.22 +pydantic==2.11.5 +pydantic_core==2.33.2 +Pygments==2.19.1 +pylibcudf @ file:///rapids/pylibcudf-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=401b8bfd022d06f84e120779ca7c3d01a28b68762aaa8f7873b10d6b1975cf6a +pylibcugraph @ file:///rapids/pylibcugraph-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=061c7389a88c815b54fd5d3a3584fb87bf693e5602f5be335260d0ba575ba557 +pylibcugraphops @ file:///rapids/pylibcugraphops-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=a25ef4b22166c80718d5c7e5bddb755e81a2fe340fa5becd0ac36ec3edfb208a +pylibraft @ file:///rapids/pylibraft-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=f2ecd23fe51f9c2caa87bbbe85cb32fcaff4ecfb52ceb9a9991bc50f82782c2e +pylibwholegraph @ file:///rapids/pylibwholegraph-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b82913d1e38bda76d8c272fbd8cc9072d987209d3d47cc93a464a37c2925aca2 +pynvjitlink @ file:///rapids/pynvjitlink-0.3.0-cp312-cp312-linux_x86_64.whl#sha256=066caf1314730643d3b48c9847bd65789defdd6a7c78993176eb3d8bbe3f4919 +pynvml @ file:///rapids/pynvml-12.0.0-py3-none-any.whl#sha256=fdff84b62a27dbe98e08e1a647eb77342bef1aebe0878bcd15e99a83fcbecb9e +pyparsing==3.2.3 +pytest==8.1.1 +pytest-flakefinder==1.1.0 +pytest-rerunfailures==15.1 +pytest-shard==0.1.2 +pytest-xdist==3.7.0 +python-dateutil==2.9.0.post0 +python-hostlist==2.2.1 +python-json-logger==3.3.0 +pytorch-triton @ file:///tmp/dist/pytorch_triton-3.3.0%2Bgit96316ce52.nvinternal-cp312-cp312-linux_x86_64.whl#sha256=6d6921772b9dfd0243c055e0d054de42a113e92cabcfdd1d3e80a5ad9c08cf7d +pytz @ file:///rapids/pytz-2023.4-py2.py3-none-any.whl#sha256=f90ef520d95e7c46951105338d918664ebfd6f1d995bd7d153127ce90efafa6a +PyYAML==6.0.2 +pyzmq==26.4.0 +raft-dask @ file:///rapids/raft_dask-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=36cf37ba6ce35454c4729c2cb597a360f22780eefb11d514a16a07137da5eb6c +rapids-dask-dependency @ file:///rapids/rapids_dask_dependency-25.4.0a0-py3-none-any.whl#sha256=484faca626e7760112a7d8ad3426f5bdd720b609006d8dca87c2607076a348bd +rapids-logger @ file:///rapids/rapids_logger-0.1.18-py3-none-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=3617b2d86291ac6cb957542895f20fc9737ace29773ca90337e6fb04c73bd120 +referencing==0.36.2 +regex==2024.11.6 +requests==2.32.3 +rfc3339-validator==0.1.4 +rfc3986-validator==0.1.1 +rich==14.0.0 +rmm @ file:///rapids/rmm-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=1824bcab3836296666980a997ddcb9c4bca24dfb873012cfa0a7ecea583ca712 +rpds-py==0.25.1 +safetensors==0.5.3 +scikit-learn==1.6.1 +scipy==1.15.3 +Send2Trash==1.8.3 +setuptools==78.1.1 +shellingham==1.5.4 +six==1.16.0 +smart-open==7.1.0 +sniffio==1.3.1 +sortedcontainers==2.4.0 +soundfile==0.13.1 +soupsieve==2.7 +soxr==0.5.0.post1 +spacy==3.7.5 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +srsly==2.5.1 +stack-data==0.6.3 +sympy==1.14.0 +tabulate==0.9.0 +tbb==2021.13.1 +tblib @ file:///rapids/tblib-3.1.0-py3-none-any.whl#sha256=670bb4582578134b3d81a84afa1b016128b429f3d48e6cbbaecc9d15675e984e +tensorboard==2.16.2 +tensorboard-data-server==0.7.2 +tensorrt @ file:///workspace/TensorRT-10.11.0.33/python/tensorrt-10.11.0.33-cp312-none-linux_x86_64.whl#sha256=c2f18f72758a3d54a71901495c8fda9e7094a8bcdd2a5b6b58967fbf05733b3a +terminado==0.18.1 +thinc==8.2.5 +threadpoolctl==3.6.0 +thriftpy2 @ file:///rapids/thriftpy2-0.5.2-cp312-cp312-linux_x86_64.whl#sha256=c0c7d939bdc5ccf7739154959938445d5fdf72f1d6e12bd563947a36f67e2147 +tinycss2==1.4.0 +toolz @ file:///rapids/toolz-1.0.0-py3-none-any.whl#sha256=292c8f1c4e7516bf9086f8850935c799a874039c8bcf959d47b600e4c44a6236 +torch @ file:///opt/transfer/torch-2.8.0a0%2B5228986c39.nv25.6-29946474-cp312-cp312-linux_x86_64.whl#sha256=7a3c9ff5288d8102a31296b3d62491f3f47c8820a5c9aaefc7dc0fe20e1e8ad5 +torch_tensorrt @ file:///opt/pytorch/torch_tensorrt/dist/torch_tensorrt-2.8.0a0-cp312-cp312-linux_x86_64.whl#sha256=26f4d7a04e7e6e017a5b917b8b98b83720bb2bb0195f48d25f52d567d286bfe1 +torchao @ file:///opt/pytorch/ao +torchprofile==0.0.4 +torchvision @ file:///opt/transfer/torchvision-0.22.0a0%2B95f10a4e-29946474-cp312-cp312-manylinux_2_28_x86_64.whl#sha256=746a3c674cc61680a86f4a14b374b21af1b5c21f629c42765b40c9494f6b234d +tornado==6.5.1 +tqdm==4.67.1 +traitlets==5.14.3 +transformer_engine @ file:///pvcdata/aiinfra/b-aghosh/wheels/transformer_engine-2.10.0.dev0%2B10f6128d-cp312-cp312-linux_x86_64.whl#sha256=35cf5f4b83b2baad864e12e512bb422e0929409ac05610812fa521d4942d0189 +treelite @ file:///rapids/treelite-4.4.1-py3-none-linux_x86_64.whl#sha256=174741724205540a1a5b37da81483aeff55d89548f42b3461cf07acac7ae96eb +typer==0.16.0 +types-dataclasses==0.6.6 +types-python-dateutil==2.9.0.20250516 +typing-inspection==0.4.1 +typing_extensions==4.14.0 +tzdata @ file:///rapids/tzdata-2025.2-py2.py3-none-any.whl#sha256=1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8 +ucx-py @ file:///rapids/ucx_py-0.43.0-cp312-cp312-linux_x86_64.whl#sha256=4a452fd8d8c983283ada3d49086e3657dc2ee94fbad08efdd708d15ab6e0b73a +ucxx @ file:///rapids/ucxx-0.43.0-cp312-cp312-linux_x86_64.whl#sha256=f1b4e55cb45c66fe88a2d8f9c2b87751165f3cfec21fbdc9c47c4ddff75e8a5f +-e git+https://github.com/Dots-Infra/UltraEP.git@94cab099b44fffa99a82fea99e7c12d89cf65e4f#egg=ultra_ep +uri-template==1.3.0 +urllib3==2.0.7 +wasabi==1.1.3 +wcwidth==0.2.13 +weasel==0.4.1 +webcolors==24.11.1 +webencodings==0.5.1 +websocket-client==1.8.0 +Werkzeug==3.1.3 +wheel==0.45.1 +wrapt==1.17.2 +xdoctest==1.0.2 +xgboost @ file:///rapids/xgboost-2.1.4-py3-none-manylinux_2_39_x86_64.whl#sha256=25eeca27f4048fc491940b3a0b4b5782ee48c33cb3aec059cef71f8da542930b +xxhash==4.0.1 +yarl @ file:///rapids/yarl-1.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=bf099e2432131093cc611623e0b0bcc399b8cddd9a91eded8bfb50402ec35018 +zarr @ file:///rapids/zarr-2.18.7-py3-none-any.whl#sha256=ac3dc4033e9ae4e9d7b5e27c97ea3eaf1003cc0a07f010bd83d5134bf8c4b223 +zict @ file:///rapids/zict-3.0.0-py2.py3-none-any.whl#sha256=5796e36bd0e0cc8cf0fbc1ace6a68912611c1dbd74750a3f3026b9b9d6a327ae +zipp==3.22.0 diff --git a/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/provenance.txt b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/provenance.txt index 0728c9ed1e513441d9fd3bc5926a62bc29bce88f..11df87c1f5ccab1c1116362e62ae4826d193828e 100644 --- a/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/provenance.txt +++ b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/provenance.txt @@ -27,3 +27,64 @@ command : run_ladder.sh --phase tierbprobe --gbs 8 --out /workspace/granul 7, 120 MHz, 1965 MHz, 1000.00 W, 29 NVLink topology: + GPU0 GPU1 GPU2 GPU3 GPU4 GPU5 GPU6 GPU7 NIC0 NIC1 NIC2 NIC3 NIC4 NIC5 NIC6 NIC7 NIC8 CPU Affinity NUMA Affinity GPU NUMA ID + GPU0 X NV18 NV18 NV18 NV18 NV18 NV18 NV18 PIX NODE SYS SYS SYS SYS SYS SYS NODE 0-27 0 N/A + GPU1 NV18 X NV18 NV18 NV18 NV18 NV18 NV18 NODE PIX SYS SYS SYS SYS SYS SYS NODE 0-27 0 N/A + GPU2 NV18 NV18 X NV18 NV18 NV18 NV18 NV18 SYS SYS PIX NODE SYS SYS SYS SYS SYS 28-55 1 N/A + GPU3 NV18 NV18 NV18 X NV18 NV18 NV18 NV18 SYS SYS NODE PIX SYS SYS SYS SYS SYS 28-55 1 N/A + GPU4 NV18 NV18 NV18 NV18 X NV18 NV18 NV18 SYS SYS SYS SYS PIX NODE SYS SYS SYS 56-83 2 N/A + GPU5 NV18 NV18 NV18 NV18 NV18 X NV18 NV18 SYS SYS SYS SYS NODE PIX SYS SYS SYS 56-83 2 N/A + GPU6 NV18 NV18 NV18 NV18 NV18 NV18 X NV18 SYS SYS SYS SYS SYS SYS PIX NODE SYS 84-111 3 N/A + GPU7 NV18 NV18 NV18 NV18 NV18 NV18 NV18 X SYS SYS SYS SYS SYS SYS NODE PIX SYS 84-111 3 N/A + NIC0 PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS SYS SYS SYS NODE + NIC1 NODE PIX SYS SYS SYS SYS SYS SYS NODE X SYS SYS SYS SYS SYS SYS NODE + NIC2 SYS SYS PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS SYS SYS + NIC3 SYS SYS NODE PIX SYS SYS SYS SYS SYS SYS NODE X SYS SYS SYS SYS SYS + NIC4 SYS SYS SYS SYS PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS + unavailable + +---- toolchain ----------------------------------------------- + nvcc : release 12.9, V12.9.86 + gcc : gcc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0 + python : 3.12.3 + torch : 2.8.0a0+5228986c39.nv25.06 cuda=12.9 nccl=2.27.3 + torch arch : ['sm_75', 'sm_80', 'sm_86', 'sm_90', 'sm_100', 'sm_120', 'compute_120'] + transformer_engine: 2.10.0.dev0+10f6128d [/usr/local/lib/python3.12/dist-packages/transformer_engine/__init__.py] + deep_ep : 1.2.1+34152ae [/workspace/DeepEP/deep_ep/__init__.py] + ultra_ep : 1.0.0 [/workspace/UltraEP/ultra_ep/__init__.py] + moonep : 0.0.1 [/workspace/MoonEP/moonep/__init__.py] + megatron : 0.14.0rc5 [None] + flash_attn : 2.7.4.post1 [/usr/local/lib/python3.12/dist-packages/flash_attn/__init__.py] + numpy : 1.26.4 [/usr/local/lib/python3.12/dist-packages/numpy/__init__.py] + huggingface_hub: 1.32.0 [/usr/local/lib/python3.12/dist-packages/huggingface_hub/__init__.py] + +---- repositories -------------------------------------------- + Megatron-MoE-Echo token-drop-analysis b099fd8494b0 + Megatron-UltraEP ultraep-bracing-regions-through-kernels 03d57ef7d476 + UltraEP main 94cab099b44f + DeepEP HEAD 34152ae28f80 (+3 untracked: build output) + MoonEP megatron-ddp-symmetric-params d8bc6667fad7 + blackwell_run_scripts granularity-echo b8bd10097bb5 + +---- environment --------------------------------------------- + CUDA_ARCH_LIST=7.5 8.0 8.6 9.0 10.0 12.0 + CUDA_DRIVER_VERSION=575.57.08 + CUDA_HOME=/usr/local/cuda + CUDA_MODULE_LOADING=LAZY + CUDA_VERSION=12.9.1.010 + HF_REMOTE_ROOT=granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16 + NCCL_VERSION=2.27.3 + NVSHMEM_VERSION=3.2.5 + PYTORCH_BUILD_NUMBER=0 + PYTORCH_BUILD_VERSION=2.8.0a0+5228986 + PYTORCH_HOME=/opt/pytorch/pytorch + PYTORCH_VERSION=2.8.0a0+5228986 + TORCH_ALLOW_TF32_CUBLAS_OVERRIDE=1 + TORCH_CUDA_ARCH_LIST=7.5 8.0 8.6 9.0 10.0 12.0+PTX + TORCH_NCCL_USE_COMM_NONBLOCKING=0 + +---- shared library resolution ------------------------------- + libnvshmem_host.so: not in ldconfig + libnccl.so: /lib/x86_64-linux-gnu/libnccl.so.2 + lrwxrwxrwx 1 root root 20 Sep 21 04:57 /usr/local/lib/python3.12/dist-packages/nvidia/nvshmem/lib/libnvshmem_host.so -> libnvshmem_host.so.3 + -rw-r--r-- 1 root root 179045744 Sep 21 04:54 /usr/local/lib/python3.12/dist-packages/nvidia/nvshmem/lib/libnvshmem_host.so.3 diff --git a/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__cap1p08984375/rep1/mem_samples.csv b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__cap1p08984375/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..b7b0e0383d6a31c8db781f04ce977ac165ef4323 --- /dev/null +++ b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__cap1p08984375/rep1/mem_samples.csv @@ -0,0 +1,224 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 4 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 821 +1, 821 +2, 821 +3, 821 +4, 821 +5, 821 +6, 821 +7, 819 +0, 1537 +1, 1537 +2, 1537 +3, 1537 +4, 1537 +5, 1537 +6, 1537 +7, 1537 +0, 1823 +1, 1823 +2, 1823 +3, 1823 +4, 1823 +5, 1823 +6, 1823 +7, 1823 +0, 13231 +1, 13257 +2, 13275 +3, 13263 +4, 13303 +5, 13319 +6, 13263 +7, 13265 +0, 14199 +1, 14199 +2, 14199 +3, 14201 +4, 14195 +5, 14199 +6, 14201 +7, 14195 +0, 14829 +1, 14831 +2, 14831 +3, 14833 +4, 14827 +5, 14831 +6, 14833 +7, 14827 +0, 14913 +1, 14915 +2, 14915 +3, 14917 +4, 14911 +5, 14915 +6, 14917 +7, 14911 +0, 15171 +1, 15173 +2, 15171 +3, 15175 +4, 15169 +5, 15173 +6, 15175 +7, 15169 +0, 15709 +1, 15711 +2, 15711 +3, 15713 +4, 15707 +5, 15711 +6, 15713 +7, 15707 +0, 15849 +1, 15851 +2, 15851 +3, 15853 +4, 15847 +5, 15851 +6, 15853 +7, 15847 +0, 16133 +1, 16133 +2, 16135 +3, 16135 +4, 16129 +5, 16135 +6, 16135 +7, 16129 +0, 16403 +1, 16405 +2, 16405 +3, 16407 +4, 16401 +5, 16405 +6, 16405 +7, 16401 +0, 28445 +1, 28423 +2, 28415 +3, 28425 +4, 28449 +5, 28439 +6, 28447 +7, 28451 +0, 28971 +1, 28971 +2, 28971 +3, 28973 +4, 28967 +5, 28971 +6, 28973 +7, 28967 +0, 28973 +1, 28973 +2, 28973 +3, 28975 +4, 28969 +5, 28973 +6, 28975 +7, 28969 +0, 28973 +1, 28973 +2, 28973 +3, 28975 +4, 28969 +5, 28973 +6, 28975 +7, 28969 +0, 28973 +1, 28973 +2, 28973 +3, 28975 +4, 28969 +5, 28973 +6, 28975 +7, 28969 +0, 29413 +1, 29403 +2, 29411 +3, 29427 +4, 29395 +5, 29453 +6, 29439 +7, 29449 +0, 30229 +1, 30229 +2, 30229 +3, 30231 +4, 30289 +5, 30549 +6, 30295 +7, 30289 +0, 30903 +1, 30873 +2, 30851 +3, 30881 +4, 30953 +5, 31207 +6, 30945 +7, 31003 +0, 37865 +1, 37881 +2, 37913 +3, 37869 +4, 37943 +5, 38205 +6, 37929 +7, 37943 +0, 37865 +1, 37881 +2, 37913 +3, 37869 +4, 37943 +5, 38205 +6, 37929 +7, 37943 +0, 1845 +1, 566 +2, 566 +3, 566 +4, 566 +5, 566 +6, 822 +7, 37147 diff --git a/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__cap1p08984375/rep1/run.env b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__cap1p08984375/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..99306331c10b214daeac2603cde8578f1cec9a54 --- /dev/null +++ b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__cap1p08984375/rep1/run.env @@ -0,0 +1,102 @@ +backend=echo +config=OLMOE_1B_7B_L16 +family=tierB +source_model=OLMoE-1B-7B-0924 +standard_num_layers=16 +base_rung= +repo=/workspace/Megatron-MoE-Echo +branch=token-drop-analysis +commit=b099fd84 +num_layers=16 +hidden=2048 +ffn_hidden=1024 +moe_ffn_hidden=1024 +num_experts=64 +topk=8 +experts_per_rank=8 +expert_params=6291456 +heads=16 +kv_groups=16 +kv_channels=128 +qk_layernorm=0 +shared_ffn=0 +shared_expert_overlap=0 +moe_token_dispatcher=flex +mla=0 +q_lora_rank= +kv_lora_rank=512 +qk_head_dim=128 +qk_pos_emb_head_dim=64 +v_head_dim=128 +dense_first_layers=0 +moe_layers=16 +rotary_base=10000 +rotary_percent=1.0 +rotary_scaling= +mscale= +mscale_all_dim= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-2 +router_pre_softmax=1 +router_fusion=0 +router_dtype=fp32 +router_groups= +router_group_topk= +router_score_function=softmax +router_expert_bias=0 +router_bias_update_rate= +router_topk_scaling_factor= +shared_expert_gate=0 +apply_layernorm_1p=0 +attention_output_gate=0 +linear_attention_type= +linear_attention_freq= +linear_conv_kernel_dim= +linear_key_head_dim= +linear_value_head_dim= +linear_num_key_heads= +linear_num_value_heads= +mtp_num_layers=0 +mtp_loss_scaling_factor=0.1 +no_weight_decay_cond_type= +max_position_embeddings=4096 +init_method_std=0.02 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-5 +seq_length=4096 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=32768 +mean_tokens_per_expert=4096 +slot_policy=matched +slots_per_rank=8 +capacity_mode=static +capacity=1.08984375 +cuda_graph=0 +probe_mode=1 +async_grad_reduce=0 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_mode=mxfp8 +fp8_recipe=mxfp8 +training_seed=1234 +learning_rate=1e-4 +min_learning_rate=1.0e-5 +lr_warmup_iters=5 +state_dump_enable=0 +state_dump_dir= +moonep_assert_no_pending_reduce=0 diff --git a/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__cap1p08984375/rep1/train.log b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__cap1p08984375/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..93f1af2b14a8a4f2d860d5fa5718b054ba3f64f2 --- /dev/null +++ b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__cap1p08984375/rep1/train.log @@ -0,0 +1,8901 @@ +[08:24:33] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +============================================================== + OLMOE_1B_7B_L16 / echo (slots/rank=8 policy=matched) + E=64 K=8 H'=1024 m=8 + L=16 seq=4096 gbs=8 (1 microbatches) + cap_mode=static cap=1.08984375 cg=0 recompute=none + memory_dump=none snapshots=/workspace/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__cap1p08984375/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ b099fd84 +============================================================== +W0921 08:24:34.412000 641899 torch/distributed/run.py:766] +W0921 08:24:34.412000 641899 torch/distributed/run.py:766] ***************************************** +W0921 08:24:34.412000 641899 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0921 08:24:34.412000 641899 torch/distributed/run.py:766] ***************************************** +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 16 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 1024 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 2048 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.02 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 4096 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.01 + moe_debug_drop_logging .......................... True + moe_debug_drop_logging_interval ................. 1 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ True + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. True + moe_enable_hybridep ............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 1024 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ 64 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.08984375 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... True + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-05 + normalization ................................... RMSNorm + num_attention_heads ............................. 16 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 64 + num_layers ...................................... 16 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 16 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 7 is connected to 67 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank 3[Gloo] Rank 62745 is connected to is connected to is connected to is connected to is connected to is connected to 777777 peer ranks. peer ranks. peer ranks. peer ranks. peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : 777777 + + + + + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank5]:[W921 08:24:50.876146704 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W921 08:24:50.876190474 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W921 08:24:50.876221642 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W921 08:24:50.876292928 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W921 08:24:50.876299932 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W921 08:24:50.876361416 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W921 08:24:50.876389398 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.036 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W921 08:24:51.040385778 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.858 seconds +time to initialize megatron (seconds): 6.949 +[after megatron is initialized] datetime: 2026-09-21 08:24:56 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 1281951744 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 7 +Params for bucket 1 (103022592 elements, 103022592 padded size): + module.output_layer.weight +Params for bucket 2 (46542848 elements, 46542848 padded size): + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.final_layernorm.weight + module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.15.self_attention.linear_qkv.weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.14.self_attention.linear_qkv.weight + module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.linear_qkv.weight +Params for bucket 3 (50737152 elements, 50737152 padded size): + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.12.self_attention.linear_qkv.weight + module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.linear_qkv.weight +Params for bucket 4 (50737152 elements, 50737152 padded size): + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.pre_mlp_layernorm.weight +Params for bucket 5 (50737152 elements, 50737152 padded size): + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight +Params for bucket 6 (50737152 elements, 50737152 padded size): + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_proj.weight +Params for bucket 7 (124131328 elements, 124131328 padded size): + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 20 +Params for bucket 1 (41943040 elements, 41943040 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 +Params for bucket 2 (41943040 elements, 41943040 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 +Params for bucket 3 (41943040 elements, 41943040 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 +Params for bucket 4 (41943040 elements, 41943040 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 +Params for bucket 5 (41943040 elements, 41943040 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 +Params for bucket 6 (41943040 elements, 41943040 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 +Params for bucket 7 (41943040 elements, 41943040 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 +Params for bucket 8 (41943040 elements, 41943040 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 +Params for bucket 9 (41943040 elements, 41943040 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 +Params for bucket 10 (41943040 elements, 41943040 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 +Params for bucket 11 (41943040 elements, 41943040 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 +Params for bucket 12 (41943040 elements, 41943040 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 +Params for bucket 13 (41943040 elements, 41943040 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 +Params for bucket 14 (41943040 elements, 41943040 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 +Params for bucket 15 (41943040 elements, 41943040 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 +Params for bucket 16 (41943040 elements, 41943040 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 +Params for bucket 17 (41943040 elements, 41943040 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 +Params for bucket 18 (41943040 elements, 41943040 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 +Params for bucket 19 (41943040 elements, 41943040 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 +Params for bucket 20 (8388608 elements, 8388608 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-09-21 08:24:57 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 96 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-09-21 08:24:57 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (307.96, 311.59) + train/valid/test-data-iterators-setup ..........: (66.30, 74.55) +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-09-21 08:24:57 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1914, 572, 134, 548, 126, 525, 145, 17, 423, 333, 82, 9, 53, 687, 320, 1764, 172, 446, 1405, 1359, 85, 839, 913, 475, 913, 159, 70, 1429, 459, 342, 117, 219, 233, 965, 831, 172, 1028, 293, 334, 450, 278, 330, 271, 11, 1667, 817, 94, 460, 690, 526, 329, 171, 1878, 103, 227, 708, 1076, 83, 625, 236, 47, 305, 379, 97] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[2126, 707, 143, 317, 315, 311, 165, 115, 498, 255, 141, 10, 45, 381, 187, 1690, 278, 332, 1325, 1410, 330, 791, 1422, 177, 911, 249, 249, 1136, 581, 763, 127, 277, 214, 628, 497, 61, 880, 226, 601, 546, 399, 562, 539, 70, 1536, 606, 71, 577, 663, 705, 582, 92, 1793, 332, 41, 250, 554, 22, 867, 327, 41, 296, 321, 105] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1798, 806, 133, 502, 195, 538, 55, 48, 165, 170, 110, 3, 111, 318, 244, 1391, 129, 389, 1577, 1960, 128, 567, 877, 113, 768, 257, 228, 1691, 663, 486, 143, 331, 242, 755, 766, 61, 1153, 258, 632, 446, 643, 533, 427, 20, 1744, 752, 93, 621, 279, 465, 723, 82, 1779, 299, 33, 388, 806, 13, 651, 430, 80, 401, 156, 143] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[2244, 680, 90, 345, 167, 407, 65, 25, 481, 219, 168, 3, 86, 415, 155, 1009, 68, 624, 1413, 1291, 117, 740, 1270, 406, 1268, 169, 54, 1331, 1019, 299, 25, 249, 482, 446, 635, 68, 723, 142, 729, 531, 261, 300, 318, 29, 1541, 652, 91, 499, 715, 1000, 788, 173, 1836, 308, 104, 427, 569, 98, 848, 210, 69, 667, 506, 101] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1849, 669, 102, 339, 84, 432, 107, 50, 539, 163, 86, 4, 97, 413, 271, 1902, 340, 537, 855, 1781, 59, 618, 1510, 195, 1008, 186, 143, 1369, 722, 417, 121, 242, 212, 730, 594, 41, 1268, 373, 405, 553, 348, 559, 431, 18, 1801, 445, 108, 629, 457, 362, 337, 195, 1773, 204, 117, 625, 1044, 65, 490, 238, 130, 523, 325, 158] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[2142, 651, 177, 407, 263, 614, 99, 21, 513, 254, 40, 0, 75, 636, 331, 1414, 99, 737, 1615, 1624, 131, 632, 1102, 424, 995, 240, 147, 1083, 987, 319, 149, 314, 169, 652, 811, 72, 1012, 127, 530, 444, 313, 135, 588, 11, 1301, 529, 34, 786, 458, 756, 791, 106, 1523, 173, 250, 575, 429, 19, 630, 228, 43, 600, 351, 87] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[2135, 755, 166, 224, 233, 591, 194, 25, 385, 280, 83, 2, 91, 395, 112, 1369, 137, 482, 914, 1662, 121, 579, 1120, 515, 1252, 338, 108, 1786, 577, 515, 42, 218, 145, 770, 513, 178, 1151, 218, 491, 428, 471, 206, 262, 10, 2142, 559, 104, 424, 331, 506, 709, 109, 2068, 172, 217, 447, 997, 32, 738, 299, 46, 222, 334, 63] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[2216, 727, 142, 227, 419, 315, 71, 29, 512, 305, 70, 1, 47, 462, 280, 1302, 51, 424, 1622, 1990, 148, 857, 1062, 573, 1060, 227, 154, 1361, 504, 229, 130, 225, 122, 957, 673, 159, 730, 170, 595, 522, 240, 172, 357, 8, 1668, 759, 69, 585, 208, 1023, 633, 59, 1983, 357, 143, 385, 492, 33, 748, 281, 60, 267, 370, 198] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1798, 806, 133, 502, 195, 538, 55, 48, 1, 0, 0, 0, 0, 0, 0, 0, 165, 170, 110, 3, 111, 318, 244, 1391, 458, 308, 296, 84, 0, 0, 0, 0, 129, 389, 1269, 0, 128, 567, 877, 113, 0, 0, 0, 0, 0, 0, 0, 0, 768, 257, 228, 1483, 663, 486, 143, 331, 0, 0, 0, 0, 0, 0, 0, 0, 242, 755, 766, 61, 1153, 258, 632, 446, 124, 13, 0, 0, 0, 0, 0, 0, 643, 533, 427, 20, 1730, 752, 93, 621, 0, 0, 0, 0, 0, 0, 0, 0, 279, 465, 723, 82, 1321, 299, 33, 388, 0, 0, 0, 0, 0, 0, 0, 0, 806, 13, 651, 430, 80, 401, 156, 143, 1664, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4076, 3658, 3472, 4359, 4450, 4819, 3590, 4344] avg=4096.0000 max=4819 max_rank=5 min=3472 min_rank=2 imbalance_ratio=1.176514 max_load_reduction=27949 max_load_reduction_pct=0.852936 moved_to_echo=17203 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1914, 572, 134, 548, 126, 525, 145, 17, 1, 0, 0, 0, 0, 0, 0, 0, 423, 333, 82, 9, 53, 687, 320, 1764, 484, 274, 205, 71, 0, 0, 0, 0, 172, 446, 1131, 0, 85, 839, 913, 475, 0, 0, 0, 0, 0, 0, 0, 0, 913, 159, 70, 1253, 459, 342, 117, 219, 0, 0, 0, 0, 0, 0, 0, 0, 233, 965, 831, 172, 1028, 293, 334, 450, 105, 12, 0, 0, 0, 0, 0, 0, 278, 330, 271, 11, 1654, 817, 94, 460, 0, 0, 0, 0, 0, 0, 0, 0, 690, 526, 329, 171, 1394, 103, 227, 708, 0, 0, 0, 0, 0, 0, 0, 0, 1076, 83, 625, 236, 47, 305, 379, 97, 1154, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3982, 4705, 4061, 3532, 4423, 3915, 4148, 4002] avg=4096.0000 max=4705 max_rank=1 min=3532 min_rank=3 imbalance_ratio=1.148682 max_load_reduction=28063 max_load_reduction_pct=0.856415 moved_to_echo=16488 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2244, 680, 90, 345, 167, 407, 65, 25, 6, 0, 0, 0, 0, 0, 0, 0, 481, 219, 168, 3, 86, 415, 155, 1009, 476, 280, 196, 69, 0, 0, 0, 0, 68, 624, 1133, 0, 117, 740, 1270, 406, 0, 0, 0, 0, 0, 0, 0, 0, 1268, 169, 54, 1160, 1019, 299, 25, 249, 0, 0, 0, 0, 0, 0, 0, 0, 482, 446, 635, 68, 723, 142, 729, 531, 102, 15, 0, 0, 0, 0, 0, 0, 261, 300, 318, 29, 1520, 652, 91, 499, 0, 0, 0, 0, 0, 0, 0, 0, 715, 1000, 788, 173, 1360, 308, 104, 427, 0, 0, 0, 0, 0, 0, 0, 0, 569, 98, 848, 210, 69, 667, 506, 101, 1095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4029, 3557, 4358, 4243, 3873, 3670, 4875, 4163] avg=4096.0000 max=4875 max_rank=6 min=3557 min_rank=1 imbalance_ratio=1.190186 max_load_reduction=27893 max_load_reduction_pct=0.851227 moved_to_echo=16581 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1849, 669, 102, 339, 84, 432, 107, 50, 2, 0, 0, 0, 0, 0, 0, 0, 539, 163, 86, 4, 97, 413, 271, 1902, 457, 167, 270, 68, 0, 0, 0, 0, 340, 537, 688, 0, 59, 618, 1510, 195, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 186, 143, 1201, 722, 417, 121, 242, 0, 0, 0, 0, 0, 0, 0, 0, 212, 730, 594, 41, 1268, 373, 405, 553, 100, 13, 0, 0, 0, 0, 0, 0, 348, 559, 431, 18, 1786, 445, 108, 629, 0, 0, 0, 0, 0, 0, 0, 0, 457, 362, 337, 195, 1316, 204, 117, 625, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 65, 490, 238, 130, 523, 325, 158, 1511, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3634, 4437, 3947, 4040, 4289, 4324, 3613, 4484] avg=4096.0000 max=4484 max_rank=7 min=3613 min_rank=6 imbalance_ratio=1.094727 max_load_reduction=28284 max_load_reduction_pct=0.863159 moved_to_echo=16710 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2126, 707, 143, 317, 315, 311, 165, 115, 1, 0, 0, 0, 0, 0, 0, 0, 498, 255, 141, 10, 45, 381, 187, 1690, 462, 259, 213, 56, 0, 0, 0, 0, 278, 332, 1066, 0, 330, 791, 1422, 177, 0, 0, 0, 0, 0, 0, 0, 0, 911, 249, 249, 997, 581, 763, 127, 277, 0, 0, 0, 0, 0, 0, 0, 0, 214, 628, 497, 61, 880, 226, 601, 546, 83, 11, 0, 0, 0, 0, 0, 0, 399, 562, 539, 70, 1524, 606, 71, 577, 0, 0, 0, 0, 0, 0, 0, 0, 663, 705, 582, 92, 1331, 332, 41, 250, 0, 0, 0, 0, 0, 0, 0, 0, 554, 22, 867, 327, 41, 296, 321, 105, 1197, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4200, 4197, 4396, 4154, 3747, 4348, 3996, 3730] avg=4096.0000 max=4396 max_rank=2 min=3730 min_rank=7 imbalance_ratio=1.073242 max_load_reduction=28372 max_load_reduction_pct=0.865845 moved_to_echo=15821 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2142, 651, 177, 407, 263, 614, 99, 21, 1, 0, 0, 0, 0, 0, 0, 0, 513, 254, 40, 0, 75, 636, 331, 1414, 392, 316, 245, 54, 0, 0, 0, 0, 99, 737, 1299, 0, 131, 632, 1102, 424, 0, 0, 0, 0, 0, 0, 0, 0, 995, 240, 147, 950, 987, 319, 149, 314, 0, 0, 0, 0, 0, 0, 0, 0, 169, 652, 811, 72, 1012, 127, 530, 444, 79, 9, 0, 0, 0, 0, 0, 0, 313, 135, 588, 11, 1291, 529, 34, 786, 0, 0, 0, 0, 0, 0, 0, 0, 458, 756, 791, 106, 1131, 173, 250, 575, 0, 0, 0, 0, 0, 0, 0, 0, 429, 19, 630, 228, 43, 600, 351, 87, 1379, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4375, 4270, 4424, 4101, 3905, 3687, 4240, 3766] avg=4096.0000 max=4424 max_rank=2 min=3687 min_rank=5 imbalance_ratio=1.080078 max_load_reduction=28344 max_load_reduction_pct=0.864990 moved_to_echo=15598 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2135, 755, 166, 224, 233, 591, 194, 25, 2, 0, 0, 0, 0, 0, 0, 0, 385, 280, 83, 2, 91, 395, 112, 1369, 533, 178, 251, 89, 0, 0, 0, 0, 137, 482, 736, 0, 121, 579, 1120, 515, 0, 0, 0, 0, 0, 0, 0, 0, 1252, 338, 108, 1566, 577, 515, 42, 218, 0, 0, 0, 0, 0, 0, 0, 0, 145, 770, 513, 178, 1151, 218, 491, 428, 131, 16, 0, 0, 0, 0, 0, 0, 471, 206, 262, 10, 2124, 559, 104, 424, 0, 0, 0, 0, 0, 0, 0, 0, 331, 506, 709, 109, 1535, 172, 217, 447, 0, 0, 0, 0, 0, 0, 0, 0, 997, 32, 738, 299, 46, 222, 334, 63, 1411, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4325, 3768, 3690, 4616, 4041, 4160, 4026, 4142] avg=4096.0000 max=4616 max_rank=3 min=3690 min_rank=2 imbalance_ratio=1.126953 max_load_reduction=28152 max_load_reduction_pct=0.859131 moved_to_echo=16369 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2216, 727, 142, 227, 419, 315, 71, 29, 1, 0, 0, 0, 0, 0, 0, 0, 512, 305, 70, 1, 47, 462, 280, 1302, 511, 317, 302, 67, 0, 0, 0, 0, 51, 424, 1305, 0, 148, 857, 1062, 573, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 227, 154, 1194, 504, 229, 130, 225, 0, 0, 0, 0, 0, 0, 0, 0, 122, 957, 673, 159, 730, 170, 595, 522, 100, 12, 0, 0, 0, 0, 0, 0, 240, 172, 357, 8, 1655, 759, 69, 585, 0, 0, 0, 0, 0, 0, 0, 0, 208, 1023, 633, 59, 1472, 357, 143, 385, 0, 0, 0, 0, 0, 0, 0, 0, 492, 33, 748, 281, 60, 267, 370, 198, 1688, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4147, 4176, 4420, 3723, 4040, 3845, 4280, 4137] avg=4096.0000 max=4420 max_rank=2 min=3723 min_rank=3 imbalance_ratio=1.079102 max_load_reduction=28348 max_load_reduction_pct=0.865112 moved_to_echo=16302 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1280, 4096, 8704, 0, 1152, 5632, 9344, 2944, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[3584, 2048, 896, 128, 640, 3712, 1920, 11904, 3840, 2176, 2048, 640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8192, 1920, 1280, 9856, 5632, 3456, 896, 2176, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[16512, 5632, 1152, 2944, 1920, 3840, 1024, 384, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3840, 5376, 4992, 1024, 10880, 2048, 1152, 3840, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[6016, 384, 5632, 2304, 640, 3328, 2816, 1024, 11136, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[1920, 6016, 5376, 896, 8064, 1920, 4352, 3968, 896, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3072, 2816, 3200, 256, 13312, 5120, 768, 4608, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1, 373, 119, 5, 0, 2, 1087, 28, 3032, 0, 24, 19, 531, 1, 29, 12, 0, 0, 0, 22, 4, 3548, 0, 1836, 160, 0, 0, 2898, 43, 455, 693, 3952, 0, 3, 2, 623, 0, 0, 0, 3923, 33, 717, 9, 0, 0, 0, 19, 3954, 65, 39, 0, 106, 20, 45, 348, 65, 36, 485, 0, 5, 1490, 1907, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 239, 1861, 3, 49, 0, 1331, 10, 3787, 9, 718, 0, 38, 42, 13, 0, 0, 5, 3, 12, 3, 2542, 0, 107, 81, 48, 0, 2564, 2737, 190, 76, 4072, 0, 48, 67, 7, 7, 0, 16, 3321, 24, 134, 110, 1, 0, 0, 22, 3467, 11, 1, 0, 366, 0, 1, 130, 10, 377, 162, 11, 315, 2462, 1155, 2, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 1, 1565, 0, 115, 207, 380, 259, 2124, 0, 1986, 12, 134, 0, 3556, 148, 45, 0, 0, 4, 0, 1770, 0, 0, 253, 545, 0, 3663, 2196, 1713, 319, 3256, 0, 2, 33, 765, 44, 1, 8, 2248, 10, 313, 23, 14, 19, 0, 5, 2414, 18, 0, 0, 492, 263, 19, 75, 5, 683, 319, 156, 69, 79, 440, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 1110, 2835, 0, 0, 1, 1124, 0, 1137, 9, 3, 88, 101, 20, 320, 0, 0, 0, 0, 0, 0, 3540, 2, 215, 4, 0, 10, 1280, 2412, 1983, 197, 3965, 0, 0, 0, 1558, 0, 0, 2, 4000, 3, 337, 15, 6, 0, 0, 12, 1362, 2, 18, 0, 33, 167, 1509, 1, 0, 583, 88, 878, 121, 1261, 456, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1, 870, 33, 0, 0, 34, 391, 7, 1864, 3, 55, 0, 29, 162, 2, 0, 0, 5, 20, 23, 203, 3843, 0, 1790, 272, 0, 0, 3332, 1720, 1122, 133, 3301, 29, 0, 49, 532, 0, 0, 1, 2016, 380, 1191, 146, 249, 8, 0, 757, 2900, 26, 1, 98, 511, 0, 1, 3, 2109, 50, 0, 17, 180, 1654, 624, 19, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[161, 102, 525, 120, 5, 0, 3345, 0, 816, 3, 6, 0, 15, 0, 396, 9, 0, 0, 0, 0, 0, 3632, 8, 1259, 507, 0, 0, 1272, 1140, 1998, 766, 4046, 0, 6, 0, 1747, 45, 0, 0, 3735, 854, 7, 86, 107, 1, 1, 145, 3334, 0, 307, 0, 246, 6, 33, 51, 0, 7, 251, 15, 119, 236, 1298, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[8, 1, 95, 0, 2, 483, 953, 0, 1145, 5, 0, 0, 1608, 0, 334, 1, 7, 1, 0, 732, 17, 3959, 12, 132, 1500, 6, 0, 1702, 1436, 814, 1341, 3929, 73, 0, 17, 51, 23, 0, 1, 3693, 0, 490, 4, 0, 0, 9, 25, 626, 11, 24, 82, 1096, 0, 1523, 273, 0, 608, 20, 1614, 2, 1387, 752, 141, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1, 1967, 1868, 85, 0, 0, 0, 0, 2971, 2, 1628, 9, 0, 3, 21, 1, 7, 0, 0, 6, 0, 1344, 0, 590, 53, 0, 0, 3479, 21, 2817, 151, 3279, 6, 47, 22, 320, 0, 0, 0, 3754, 57, 1, 296, 2, 0, 49, 8, 1034, 2, 59, 0, 952, 8, 433, 0, 99, 1046, 5, 15, 250, 1956, 2011, 32, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1565, 0, 115, 207, 380, 259, 960, 38, 0, 0, 0, 0, 0, 0, 2124, 0, 1986, 12, 134, 0, 3556, 148, 687, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 4, 0, 1770, 0, 0, 261, 6, 0, 0, 0, 0, 0, 0, 253, 545, 0, 521, 2196, 1713, 319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 33, 765, 44, 1, 8, 2242, 0, 0, 0, 0, 0, 0, 0, 0, 10, 313, 23, 14, 19, 0, 5, 2414, 1265, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 492, 263, 19, 75, 5, 2296, 0, 0, 0, 0, 0, 0, 0, 683, 319, 156, 69, 79, 440, 0, 0, 891, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3525, 8647, 2086, 5547, 3095, 4063, 3168, 2637] avg=4096.0000 max=8647 max_rank=1 min=2086 min_rank=2 imbalance_ratio=2.111084 max_load_reduction=24121 max_load_reduction_pct=0.736115 moved_to_echo=12963 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 239, 1861, 3, 49, 0, 1331, 10, 1202, 26, 0, 0, 0, 0, 0, 0, 3787, 9, 718, 0, 38, 42, 13, 0, 480, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3, 12, 3, 2542, 0, 107, 183, 9, 0, 0, 0, 0, 0, 0, 81, 48, 0, 365, 2737, 190, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 67, 7, 7, 0, 16, 3312, 0, 0, 0, 0, 0, 0, 0, 0, 24, 134, 110, 1, 0, 0, 22, 3467, 886, 0, 0, 0, 0, 0, 0, 0, 11, 1, 0, 366, 0, 1, 130, 10, 2870, 0, 0, 0, 0, 0, 0, 0, 377, 162, 11, 315, 2462, 1155, 2, 1, 624, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4721, 5087, 2864, 3497, 3457, 4644, 3389, 5109] avg=4096.0000 max=5109 max_rank=7 min=2864 min_rank=2 imbalance_ratio=1.247314 max_load_reduction=27659 max_load_reduction_pct=0.844086 moved_to_echo=16599 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 870, 33, 0, 0, 34, 391, 7, 974, 34, 0, 0, 0, 0, 0, 0, 1864, 3, 55, 0, 29, 162, 2, 0, 624, 0, 0, 0, 0, 0, 0, 0, 0, 5, 20, 23, 203, 3843, 0, 1790, 238, 5, 0, 0, 0, 0, 0, 0, 272, 0, 0, 474, 1720, 1122, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 49, 532, 0, 0, 1, 2011, 0, 0, 0, 0, 0, 0, 0, 0, 380, 1191, 146, 249, 8, 0, 757, 2900, 1151, 0, 0, 0, 0, 0, 0, 0, 26, 1, 98, 511, 0, 1, 3, 2109, 2327, 0, 0, 0, 0, 0, 0, 0, 50, 0, 17, 180, 1654, 624, 19, 2, 811, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2344, 2739, 6127, 3721, 2622, 6782, 5076, 3357] avg=4096.0000 max=6782 max_rank=5 min=2344 min_rank=0 imbalance_ratio=1.655762 max_load_reduction=25986 max_load_reduction_pct=0.793030 moved_to_echo=17837 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 373, 119, 5, 0, 2, 1087, 28, 1166, 30, 0, 0, 0, 0, 0, 0, 3032, 0, 24, 19, 531, 1, 29, 12, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4, 3548, 0, 1836, 207, 11, 0, 0, 0, 0, 0, 0, 160, 0, 0, 412, 43, 455, 693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 623, 0, 0, 0, 3912, 0, 0, 0, 0, 0, 0, 0, 0, 33, 717, 9, 0, 0, 0, 19, 3954, 1001, 0, 0, 0, 0, 0, 0, 0, 65, 39, 0, 106, 20, 45, 348, 65, 2786, 0, 0, 0, 0, 0, 0, 0, 36, 485, 0, 5, 1490, 1907, 0, 0, 705, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2811, 4191, 5628, 1763, 4540, 5733, 3474, 4628] avg=4096.0000 max=5733 max_rank=5 min=1763 min_rank=3 imbalance_ratio=1.399658 max_load_reduction=27035 max_load_reduction_pct=0.825043 moved_to_echo=18375 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[161, 102, 525, 120, 5, 0, 3345, 0, 1195, 13, 0, 0, 0, 0, 0, 0, 816, 3, 6, 0, 15, 0, 396, 9, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3632, 8, 1259, 90, 10, 0, 0, 0, 0, 0, 0, 507, 0, 0, 183, 1140, 1998, 766, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1747, 45, 0, 0, 3725, 0, 0, 0, 0, 0, 0, 0, 0, 854, 7, 86, 107, 1, 1, 145, 3334, 439, 0, 0, 0, 0, 0, 0, 0, 0, 307, 0, 246, 6, 33, 51, 0, 2851, 0, 0, 0, 0, 0, 0, 0, 7, 251, 15, 119, 236, 1298, 0, 0, 309, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5466, 1483, 4999, 4594, 5523, 4974, 3494, 2235] avg=4096.0000 max=5523 max_rank=4 min=1483 min_rank=1 imbalance_ratio=1.348389 max_load_reduction=27245 max_load_reduction_pct=0.831451 moved_to_echo=16226 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1110, 2835, 0, 0, 1, 1124, 0, 1171, 17, 0, 0, 0, 0, 0, 0, 1137, 9, 3, 88, 101, 20, 320, 0, 244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3540, 2, 215, 95, 16, 0, 0, 0, 0, 0, 0, 4, 0, 10, 163, 2412, 1983, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1558, 0, 0, 2, 3984, 0, 0, 0, 0, 0, 0, 0, 0, 3, 337, 15, 6, 0, 0, 12, 1362, 446, 0, 0, 0, 0, 0, 0, 0, 2, 18, 0, 33, 167, 1509, 1, 0, 2794, 0, 0, 0, 0, 0, 0, 0, 583, 88, 878, 121, 1261, 456, 0, 0, 315, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6258, 1922, 3868, 4769, 5544, 2181, 4524, 3702] avg=4096.0000 max=6258 max_rank=0 min=1922 min_rank=1 imbalance_ratio=1.527832 max_load_reduction=26510 max_load_reduction_pct=0.809021 moved_to_echo=15951 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8, 1, 95, 0, 2, 483, 953, 0, 1159, 17, 0, 0, 0, 0, 0, 0, 1145, 5, 0, 0, 1608, 0, 334, 1, 319, 0, 0, 0, 0, 0, 0, 0, 7, 1, 0, 732, 17, 3959, 12, 132, 121, 10, 0, 0, 0, 0, 0, 0, 1500, 6, 0, 243, 1436, 814, 1341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 17, 51, 23, 0, 1, 3683, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 4, 0, 0, 9, 25, 626, 588, 0, 0, 0, 0, 0, 0, 0, 11, 24, 82, 1096, 0, 1523, 273, 0, 2770, 0, 0, 0, 0, 0, 0, 0, 608, 20, 1614, 2, 1387, 752, 141, 0, 414, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2718, 3412, 4991, 5340, 3848, 1742, 5779, 4938] avg=4096.0000 max=5779 max_rank=6 min=1742 min_rank=5 imbalance_ratio=1.410889 max_load_reduction=26989 max_load_reduction_pct=0.823639 moved_to_echo=16307 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 1967, 1868, 85, 0, 0, 0, 0, 968, 36, 0, 0, 0, 0, 0, 0, 2971, 2, 1628, 9, 0, 3, 21, 1, 652, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 6, 0, 1344, 0, 590, 248, 10, 0, 0, 0, 0, 0, 0, 53, 0, 0, 495, 21, 2817, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 47, 22, 320, 0, 0, 0, 3744, 0, 0, 0, 0, 0, 0, 0, 0, 57, 1, 296, 2, 0, 49, 8, 1034, 1202, 0, 0, 0, 0, 0, 0, 0, 2, 59, 0, 952, 8, 433, 0, 99, 2311, 0, 0, 0, 0, 0, 0, 0, 1046, 5, 15, 250, 1956, 2011, 32, 1, 846, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4925, 5287, 2205, 3537, 4139, 2649, 3864, 6162] avg=4096.0000 max=6162 max_rank=7 min=2205 min_rank=2 imbalance_ratio=1.504395 max_load_reduction=26606 max_load_reduction_pct=0.811951 moved_to_echo=16814 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[16896, 128, 4480, 128, 2560, 256, 4736, 256, 3840, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1408, 3200, 768, 384, 128, 128, 1024, 19200, 7040, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[256, 4736, 8960, 256, 256, 768, 8704, 384, 8832, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[2944, 640, 128, 2944, 11776, 11136, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[3456, 1408, 2816, 1152, 10624, 8704, 256, 128, 4992, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 512, 256, 3840, 512, 3584, 896, 2304, 21120, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 256, 5632, 128, 128, 128, 26624, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 896, 256, 24192, 128, 6016, 1536, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[18, 40, 0, 0, 0, 0, 29, 0, 2917, 3386, 0, 141, 0, 0, 0, 0, 0, 2, 1, 3950, 0, 15, 7, 4076, 0, 0, 0, 2185, 2, 6, 4050, 8, 2, 0, 10, 0, 27, 88, 1701, 0, 0, 7, 696, 0, 2312, 0, 0, 45, 618, 0, 1, 0, 1774, 26, 1, 0, 50, 0, 0, 541, 41, 3995, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[23, 276, 0, 2, 0, 0, 0, 198, 4, 1678, 0, 0, 0, 1, 0, 20, 188, 1, 0, 0, 7, 1, 64, 3051, 0, 39, 0, 3715, 160, 5, 0, 20, 4000, 0, 2, 62, 0, 601, 45, 2, 0, 0, 72, 0, 3471, 20, 2010, 3558, 2918, 25, 0, 0, 1038, 1, 0, 0, 3238, 9, 15, 0, 2173, 0, 1, 54] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 3, 21, 0, 0, 7, 75, 0, 0, 4083, 5, 1, 0, 0, 0, 0, 0, 7, 70, 40, 0, 31, 0, 3811, 324, 0, 0, 3867, 0, 43, 5, 4, 4082, 0, 279, 5, 61, 108, 2806, 3, 0, 0, 27, 0, 327, 234, 912, 3, 1525, 113, 1724, 0, 4009, 0, 1, 0, 0, 0, 17, 0, 79, 4056, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[6, 0, 0, 6, 0, 60, 0, 0, 0, 4092, 10, 3274, 28, 12, 8, 0, 13, 1, 0, 0, 0, 0, 4, 2011, 0, 2, 2, 3811, 155, 2, 116, 0, 1, 2845, 0, 0, 0, 0, 113, 0, 0, 37, 187, 0, 1531, 31, 338, 1, 3193, 11, 37, 0, 3121, 5, 0, 0, 0, 0, 0, 18, 4042, 3644, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 1134, 7, 1, 8, 0, 2233, 4090, 53, 0, 3015, 0, 3, 1, 0, 89, 0, 51, 0, 0, 0, 1683, 0, 9, 0, 3470, 18, 46, 29, 460, 1, 0, 2, 0, 0, 144, 2242, 20, 3, 2, 3340, 1, 358, 288, 1454, 0, 1022, 0, 249, 0, 108, 0, 0, 0, 3312, 0, 2, 0, 82, 3717, 0, 21] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 91, 0, 2, 1249, 61, 0, 4007, 9, 3975, 2103, 0, 0, 0, 19, 0, 0, 0, 0, 9, 0, 2905, 0, 1277, 4, 3849, 0, 2, 401, 3, 1, 106, 0, 0, 0, 0, 1608, 0, 2, 9, 0, 7, 1642, 0, 109, 447, 799, 15, 2064, 0, 1347, 0, 0, 1, 4009, 0, 0, 0, 80, 553, 0, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 5, 0, 20, 3, 2984, 4027, 0, 479, 0, 0, 1, 0, 0, 98, 3, 0, 0, 1, 0, 3933, 0, 23, 0, 3470, 133, 0, 711, 0, 3986, 133, 10, 0, 19, 423, 467, 13, 1, 0, 83, 0, 2761, 785, 1, 13, 2565, 403, 966, 1, 1, 0, 0, 0, 199, 2, 57, 477, 3409, 98, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 6, 17, 15, 0, 3762, 2485, 261, 1171, 52, 0, 0, 0, 0, 1084, 0, 9, 0, 0, 0, 4048, 0, 0, 0, 3481, 12, 0, 133, 0, 3997, 2381, 6, 1, 2, 0, 125, 0, 0, 2, 45, 0, 1794, 0, 1043, 0, 1852, 14, 36, 14, 218, 0, 1, 0, 33, 0, 0, 0, 3142, 1388, 0, 136] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[18, 40, 0, 0, 0, 0, 29, 0, 2626, 1780, 0, 0, 0, 0, 0, 0, 2917, 760, 0, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 3950, 0, 15, 7, 4076, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1930, 2, 6, 4050, 8, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 10, 0, 27, 88, 1701, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 7, 696, 0, 2312, 0, 0, 45, 491, 47, 0, 0, 0, 0, 0, 0, 618, 0, 1, 0, 1774, 26, 1, 0, 74, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 541, 41, 1724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4493, 3818, 8167, 5996, 1846, 3598, 2494, 2356] avg=4096.0000 max=8167 max_rank=2 min=1846 min_rank=4 imbalance_ratio=1.993896 max_load_reduction=24601 max_load_reduction_pct=0.750763 moved_to_echo=10294 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 21, 0, 0, 7, 75, 0, 3167, 1807, 0, 0, 0, 0, 0, 0, 0, 916, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 70, 40, 0, 31, 0, 3811, 206, 0, 0, 0, 0, 0, 0, 0, 324, 0, 0, 3412, 0, 43, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 279, 5, 61, 108, 2806, 3, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 327, 234, 912, 3, 499, 84, 0, 0, 0, 0, 0, 0, 1525, 113, 1724, 0, 4009, 0, 1, 0, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 79, 1750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5080, 922, 4165, 3788, 7377, 2086, 7504, 1846] avg=4096.0000 max=7504 max_rank=6 min=922 min_rank=1 imbalance_ratio=1.832031 max_load_reduction=25264 max_load_reduction_pct=0.770996 moved_to_echo=18813 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[23, 276, 0, 2, 0, 0, 0, 198, 1301, 0, 0, 0, 0, 0, 0, 0, 4, 377, 0, 0, 0, 1, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 188, 1, 0, 0, 7, 1, 64, 3051, 198, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 3279, 160, 5, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 4000, 0, 2, 62, 0, 601, 45, 2, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 3471, 20, 2010, 3558, 0, 81, 0, 0, 0, 0, 0, 0, 2918, 25, 0, 0, 1038, 1, 0, 0, 126, 0, 0, 0, 0, 0, 0, 0, 3238, 9, 15, 0, 2173, 0, 1, 54, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1800, 402, 3510, 3503, 4743, 9212, 4108, 5490] avg=4096.0000 max=9212 max_rank=5 min=402 min_rank=1 imbalance_ratio=2.249023 max_load_reduction=23556 max_load_reduction_pct=0.718872 moved_to_echo=23553 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 0, 6, 0, 60, 0, 0, 3174, 1623, 0, 0, 0, 0, 0, 0, 0, 918, 10, 3274, 28, 12, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1, 0, 0, 0, 0, 4, 2011, 203, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 3363, 155, 2, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2845, 0, 0, 0, 0, 113, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 37, 187, 0, 1531, 31, 338, 1, 448, 83, 0, 0, 0, 0, 0, 0, 3193, 11, 37, 0, 3121, 5, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 4042, 1573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4869, 4250, 2232, 3640, 2991, 2656, 6497, 5633] avg=4096.0000 max=6497 max_rank=6 min=2232 min_rank=2 imbalance_ratio=1.586182 max_load_reduction=26271 max_load_reduction_pct=0.801727 moved_to_echo=17777 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1134, 7, 1, 8, 0, 3172, 1656, 0, 0, 0, 0, 0, 0, 2233, 918, 53, 0, 3015, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 51, 0, 0, 0, 1683, 185, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 3062, 18, 46, 29, 460, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 144, 2242, 20, 29, 0, 0, 0, 0, 0, 0, 0, 3, 2, 3340, 1, 358, 288, 1454, 0, 457, 76, 0, 0, 0, 0, 0, 0, 1022, 0, 249, 0, 108, 0, 0, 0, 118, 0, 0, 0, 0, 0, 0, 0, 3312, 0, 2, 0, 82, 1604, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5978, 6223, 2008, 3624, 2438, 5979, 1497, 5021] avg=4096.0000 max=6223 max_rank=1 min=1497 min_rank=6 imbalance_ratio=1.519287 max_load_reduction=26545 max_load_reduction_pct=0.810089 moved_to_echo=14935 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 5, 0, 20, 3, 3123, 43, 0, 0, 0, 0, 0, 0, 2984, 904, 0, 479, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 3, 0, 0, 1, 0, 3933, 185, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 3062, 133, 0, 711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3986, 133, 10, 0, 19, 423, 467, 13, 29, 0, 0, 0, 0, 0, 0, 0, 1, 0, 83, 0, 2761, 785, 1, 13, 12, 76, 0, 0, 0, 0, 0, 0, 2565, 403, 966, 1, 1, 0, 0, 0, 118, 0, 0, 0, 0, 0, 0, 0, 199, 2, 57, 477, 3409, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3198, 4368, 4220, 3929, 5080, 3732, 4054, 4187] avg=4096.0000 max=5080 max_rank=4 min=3198 min_rank=0 imbalance_ratio=1.240234 max_load_reduction=27688 max_load_reduction_pct=0.844971 moved_to_echo=17053 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 91, 0, 2, 1249, 61, 3113, 249, 0, 0, 0, 0, 0, 0, 0, 894, 9, 3975, 2103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 9, 0, 2905, 207, 0, 0, 0, 0, 0, 0, 0, 0, 1277, 4, 3383, 0, 2, 401, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 106, 0, 0, 0, 0, 1608, 0, 37, 0, 0, 0, 0, 0, 0, 0, 2, 9, 0, 7, 1642, 0, 109, 447, 72, 87, 0, 0, 0, 0, 0, 0, 799, 15, 2064, 0, 1347, 0, 0, 1, 135, 0, 0, 0, 0, 0, 0, 0, 4009, 0, 0, 0, 80, 232, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4765, 6981, 3140, 5070, 1752, 2375, 4361, 4324] avg=4096.0000 max=6981 max_rank=1 min=1752 min_rank=4 imbalance_ratio=1.704346 max_load_reduction=25787 max_load_reduction_pct=0.786957 moved_to_echo=12812 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 6, 17, 15, 0, 1927, 618, 0, 0, 0, 0, 0, 0, 3762, 558, 261, 1171, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1084, 0, 9, 0, 0, 0, 4048, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3073, 12, 0, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3997, 2381, 6, 1, 2, 0, 125, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 2, 45, 0, 1794, 0, 1043, 0, 170, 76, 0, 0, 0, 0, 0, 0, 1852, 14, 36, 14, 218, 0, 1, 0, 118, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 3142, 600, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2585, 5804, 5326, 3218, 6541, 3130, 2253, 3911] avg=4096.0000 max=6541 max_rank=4 min=2253 min_rank=6 imbalance_ratio=1.596924 max_load_reduction=26227 max_load_reduction_pct=0.800385 moved_to_echo=15835 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[14592, 640, 5120, 128, 11648, 128, 128, 128, 1024, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 384, 128, 1280, 128, 128, 1408, 384, 21632, 7808, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[11904, 6272, 384, 9088, 5248, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 4480, 128, 14208, 1408, 5888, 4096, 2176, 640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[256, 1408, 128, 4096, 128, 128, 128, 25600, 1536, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[16128, 5504, 384, 128, 128, 1408, 9216, 128, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[10880, 128, 128, 1152, 13056, 7552, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[384, 1408, 128, 24576, 512, 128, 5504, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 3, 4083, 55, 3084, 0, 1, 1578, 0, 2883, 0, 2, 0, 1034, 0, 0, 0, 2009, 290, 37, 0, 2357, 0, 0, 0, 0, 963, 0, 0, 7, 0, 0, 0, 1257, 3725, 265, 0, 6, 0, 0, 10, 22, 0, 0, 5, 0, 0, 0, 1, 1288, 62, 4086, 2606, 28, 228, 0, 781, 6, 0, 3, 3, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 95, 0, 0, 1220, 1475, 0, 1658, 595, 2073, 0, 6, 173, 6, 0, 2, 0, 3562, 618, 0, 0, 504, 0, 0, 0, 0, 558, 12, 0, 0, 0, 3, 928, 1445, 4086, 0, 0, 16, 0, 0, 45, 0, 4076, 0, 0, 0, 1053, 153, 21, 72, 0, 6, 3571, 0, 1, 115, 1812, 0, 2, 1954, 848, 3, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1, 12, 3, 1069, 7, 2, 84, 1397, 1, 60, 0, 0, 0, 1125, 0, 973, 0, 1988, 7, 0, 180, 1984, 0, 0, 0, 0, 2594, 34, 3, 12, 74, 14, 823, 259, 3691, 0, 0, 0, 0, 0, 0, 0, 251, 4, 2591, 0, 3879, 0, 0, 3778, 87, 0, 3948, 0, 0, 0, 1669, 0, 4, 146, 7, 0, 0, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 208, 2, 0, 9, 3233, 0, 0, 0, 4078, 0, 0, 8, 0, 8, 0, 8, 3372, 0, 0, 0, 154, 0, 0, 5, 0, 0, 639, 0, 0, 0, 0, 151, 446, 4084, 0, 0, 7, 0, 34, 0, 0, 628, 0, 3420, 0, 30, 0, 0, 0, 0, 0, 4028, 0, 0, 1171, 3840, 0, 0, 3204, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 1217, 155, 195, 0, 62, 1, 0, 0, 3488, 49, 0, 2560, 8, 0, 809, 0, 607, 145, 2, 0, 384, 0, 0, 7, 0, 0, 159, 0, 0, 22, 0, 0, 990, 1578, 0, 0, 0, 3, 1320, 1601, 0, 59, 0, 3783, 0, 2911, 16, 3700, 0, 0, 174, 3691, 0, 0, 0, 871, 0, 0, 2007, 69, 1, 1, 123] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[2, 4071, 0, 12, 0, 1, 0, 17, 246, 3524, 0, 0, 603, 0, 1106, 0, 0, 3526, 0, 0, 0, 0, 0, 0, 3, 17, 2, 0, 0, 13, 0, 480, 1286, 16, 3916, 123, 0, 0, 0, 0, 34, 0, 2779, 0, 2958, 0, 3883, 1, 0, 118, 0, 0, 3953, 0, 0, 0, 0, 0, 0, 0, 13, 0, 65, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 1367, 0, 0, 3996, 0, 21, 1701, 29, 4074, 145, 0, 1130, 1557, 0, 0, 0, 4077, 91, 0, 0, 0, 0, 0, 0, 0, 249, 371, 231, 0, 1, 0, 0, 0, 1034, 3, 1, 2, 0, 0, 12, 0, 1836, 2561, 7, 26, 0, 0, 0, 3658, 0, 360, 3975, 6, 0, 0, 189, 0, 0, 10, 48, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[6, 1359, 2, 54, 0, 0, 1, 167, 0, 0, 0, 0, 0, 571, 55, 0, 0, 3203, 0, 0, 11, 8, 0, 0, 1870, 22, 125, 0, 0, 0, 0, 0, 279, 175, 1941, 4, 7, 0, 0, 1, 2701, 2, 3991, 3, 2137, 0, 3828, 0, 4093, 245, 0, 132, 4077, 4, 1, 0, 23, 0, 0, 8, 1657, 0, 0, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 11, 3, 1069, 7, 2, 84, 1397, 0, 0, 0, 0, 0, 0, 0, 0, 1, 60, 0, 0, 0, 1125, 0, 973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1988, 7, 0, 180, 1984, 0, 0, 239, 174, 1, 0, 0, 0, 0, 0, 0, 0, 2594, 34, 3, 12, 74, 14, 2713, 936, 0, 0, 0, 0, 0, 0, 823, 259, 3517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 4, 2080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3778, 87, 0, 1235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1669, 0, 4, 146, 7, 0, 0, 7, 2943, 272, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2574, 2159, 4573, 6380, 4599, 2335, 5100, 5048] avg=4096.0000 max=6380 max_rank=3 min=2159 min_rank=1 imbalance_ratio=1.557617 max_load_reduction=26388 max_load_reduction_pct=0.805298 moved_to_echo=17082 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 4083, 55, 3084, 0, 1, 1578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2853, 0, 2, 0, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2009, 290, 37, 0, 2357, 0, 0, 0, 175, 0, 30, 0, 0, 0, 0, 0, 0, 963, 0, 0, 7, 0, 0, 1791, 0, 0, 0, 0, 0, 0, 0, 0, 1257, 3550, 265, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 22, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1288, 62, 4086, 815, 28, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 781, 6, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8804, 3889, 4898, 2761, 5078, 37, 6508, 793] avg=4096.0000 max=8804 max_rank=0 min=37 min_rank=5 imbalance_ratio=2.149414 max_load_reduction=23964 max_load_reduction_pct=0.731323 moved_to_echo=12416 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1217, 0, 0, 3996, 0, 21, 1701, 0, 0, 0, 0, 0, 0, 0, 0, 29, 4031, 145, 0, 1130, 1557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 91, 0, 0, 0, 0, 0, 0, 48, 150, 43, 0, 0, 0, 0, 0, 0, 249, 371, 231, 0, 1, 0, 2732, 0, 0, 0, 0, 0, 0, 0, 0, 0, 986, 3, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 1836, 2561, 7, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3658, 0, 360, 1243, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 0, 0, 10, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6935, 6892, 4409, 3584, 992, 4442, 5267, 247] avg=4096.0000 max=6935 max_rank=0 min=247 min_rank=7 imbalance_ratio=1.693115 max_load_reduction=25833 max_load_reduction_pct=0.788361 moved_to_echo=10948 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1083, 155, 195, 0, 62, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3452, 49, 0, 2560, 8, 0, 809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 607, 145, 2, 0, 384, 0, 0, 350, 74, 134, 36, 0, 0, 0, 0, 7, 0, 0, 159, 0, 0, 22, 0, 2536, 702, 0, 0, 0, 0, 0, 0, 0, 990, 1504, 0, 0, 0, 3, 1320, 0, 0, 0, 0, 0, 0, 0, 0, 1601, 0, 59, 0, 3036, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 3700, 0, 0, 174, 1155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 871, 0, 0, 2007, 69, 1, 1, 123, 2209, 397, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1496, 6878, 1732, 3426, 3817, 4712, 5029, 5678] avg=4096.0000 max=6878 max_rank=1 min=1496 min_rank=0 imbalance_ratio=1.679199 max_load_reduction=25890 max_load_reduction_pct=0.790100 moved_to_echo=19236 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2, 3623, 0, 12, 0, 1, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 246, 3487, 0, 0, 603, 0, 1106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3526, 0, 0, 0, 0, 0, 0, 273, 184, 448, 37, 0, 0, 0, 0, 3, 17, 2, 0, 0, 13, 0, 480, 2717, 938, 0, 0, 0, 0, 0, 0, 1286, 16, 3732, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 2779, 0, 2375, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 0, 0, 1236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 65, 0, 2945, 310, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3655, 5442, 4468, 4170, 5157, 5189, 1354, 3333] avg=4096.0000 max=5442 max_rank=1 min=1354 min_rank=6 imbalance_ratio=1.328613 max_load_reduction=27326 max_load_reduction_pct=0.833923 moved_to_echo=15033 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 85, 0, 0, 1220, 1475, 0, 1658, 0, 0, 0, 0, 0, 0, 0, 0, 595, 2052, 0, 6, 173, 6, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3562, 618, 0, 0, 504, 0, 0, 0, 192, 10, 21, 0, 0, 0, 0, 0, 0, 558, 12, 0, 0, 0, 3, 2454, 254, 0, 0, 0, 0, 0, 0, 928, 1445, 3894, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 4076, 0, 0, 0, 0, 153, 0, 0, 0, 0, 0, 0, 0, 0, 21, 72, 0, 6, 1117, 0, 1, 115, 0, 0, 0, 0, 0, 0, 0, 0, 1812, 0, 2, 1954, 848, 3, 1, 0, 799, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4438, 2834, 4907, 3281, 6283, 4274, 1332, 5419] avg=4096.0000 max=6283 max_rank=4 min=1332 min_rank=6 imbalance_ratio=1.533936 max_load_reduction=26485 max_load_reduction_pct=0.808258 moved_to_echo=17308 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[6, 1210, 2, 54, 0, 0, 1, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 571, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3203, 0, 0, 11, 8, 0, 0, 197, 91, 149, 0, 0, 0, 0, 0, 1870, 22, 125, 0, 0, 0, 0, 0, 2802, 924, 0, 0, 0, 0, 0, 0, 279, 175, 1850, 4, 7, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2701, 2, 3991, 3, 1716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 245, 0, 132, 1275, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 8, 1657, 0, 0, 5, 2904, 224, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1440, 626, 3659, 5743, 2316, 8413, 5750, 4821] avg=4096.0000 max=8413 max_rank=5 min=626 min_rank=1 imbalance_ratio=2.053955 max_load_reduction=24355 max_load_reduction_pct=0.743256 moved_to_echo=21300 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 182, 2, 0, 9, 3233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 0, 0, 8, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3372, 0, 0, 0, 154, 0, 0, 320, 196, 26, 46, 0, 0, 0, 0, 5, 0, 0, 639, 0, 0, 0, 0, 2771, 8, 0, 0, 0, 0, 0, 0, 151, 446, 3888, 0, 0, 7, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 628, 0, 2738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1257, 0, 0, 1171, 0, 0, 0, 0, 0, 0, 0, 0, 3840, 0, 0, 3204, 0, 0, 0, 1, 22, 362, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3426, 4048, 4122, 3423, 4526, 3366, 2428, 7429] avg=4096.0000 max=7429 max_rank=7 min=2428 min_rank=6 imbalance_ratio=1.813721 max_load_reduction=25339 max_load_reduction_pct=0.773285 moved_to_echo=17749 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4480, 128, 13696, 2688, 12032, 128, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 7424, 4352, 1408, 8320, 4864, 128, 6528, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1920, 128, 4608, 1280, 256, 128, 128, 512, 20608, 3840, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[9216, 128, 128, 7424, 2688, 128, 128, 256, 11904, 1664, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[896, 19968, 256, 128, 4480, 4352, 1280, 1792, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[7936, 9216, 256, 4864, 9344, 128, 256, 1408, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[3584, 4608, 23040, 512, 128, 128, 128, 1408, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 22400, 1152, 128, 256, 5504, 0, 0, 1408, 1152, 1024, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 161, 29, 2, 0, 0, 3257, 335, 0, 0, 1, 0, 346, 0, 3691, 0, 0, 3969, 394, 607, 4040, 0, 0, 8, 661, 0, 0, 1, 2136, 1255, 63, 0, 0, 0, 0, 0, 0, 254, 513, 0, 4, 0, 2, 0, 51, 0, 5, 2133, 0, 983, 15, 0, 240, 61, 0, 13, 3645, 3893, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[3, 11, 201, 522, 4033, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 512, 35, 113, 4091, 0, 76, 0, 3989, 0, 0, 2, 0, 992, 0, 0, 141, 1658, 0, 0, 2, 4, 0, 98, 0, 0, 0, 1, 1140, 1, 37, 501, 2892, 0, 0, 0, 15, 15, 0, 1319, 0, 0, 0, 0, 2958, 4087, 4, 3314, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 16, 4042, 1131, 0, 0, 13, 0, 2, 0, 0, 0, 0, 0, 2, 493, 0, 12, 15, 0, 0, 3953, 0, 0, 0, 0, 11, 71, 252, 4075, 653, 0, 0, 0, 2, 14, 168, 0, 0, 0, 0, 4, 0, 0, 402, 3922, 0, 5, 110, 0, 3186, 0, 0, 2, 0, 11, 0, 2030, 4078, 0, 4092, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[4, 178, 0, 3, 94, 0, 221, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 3886, 0, 3449, 17, 0, 2, 0, 2, 75, 0, 0, 0, 28, 0, 274, 3421, 1, 0, 1483, 0, 0, 0, 0, 0, 0, 0, 2096, 81, 5, 0, 1849, 1, 2416, 0, 3, 634, 790, 182, 7, 0, 0, 0, 3848, 3531, 0, 34, 58, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 1113, 0, 49, 9, 6, 26, 0, 3743, 0, 15, 24, 8, 2216, 78, 0, 0, 0, 42, 68, 0, 0, 32, 3368, 35, 0, 276, 4080, 0, 1, 35, 19, 244, 0, 0, 0, 1, 6, 0, 3907, 0, 0, 95, 982, 0, 0, 0, 0, 4058, 3514, 0, 0, 1, 2, 270, 4029, 130, 0, 0, 0, 283] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 24, 43, 1204, 2, 2539, 0, 0, 0, 0, 0, 1, 0, 2029, 4080, 224, 0, 0, 0, 4005, 1, 0, 0, 17, 0, 0, 762, 372, 3849, 0, 126, 0, 1, 0, 0, 172, 132, 27, 1, 640, 0, 108, 3533, 242, 0, 0, 138, 6, 4, 0, 0, 442, 0, 2, 2, 3988, 3874, 0, 176, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 1451, 36, 880, 0, 0, 494, 0, 2, 2975, 0, 0, 0, 0, 1181, 160, 0, 1000, 0, 726, 4078, 0, 9, 0, 0, 7, 0, 0, 4, 3348, 0, 4, 0, 0, 0, 0, 0, 1, 299, 10, 2638, 5, 0, 466, 3729, 27, 6, 0, 0, 6, 977, 41, 0, 0, 0, 0, 4091, 648, 0, 3465, 4, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 574, 3577, 5, 48, 0, 0, 13, 6, 0, 0, 3, 0, 2, 0, 1726, 0, 60, 0, 333, 3749, 1, 0, 0, 0, 0, 0, 398, 3356, 3070, 15, 95, 0, 5, 0, 186, 2, 0, 2603, 0, 18, 686, 1, 0, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 681, 0, 3790, 4051, 0, 3602, 9, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 161, 29, 2, 0, 0, 1397, 772, 0, 0, 0, 0, 0, 0, 3257, 335, 0, 0, 1, 0, 346, 0, 2429, 0, 0, 0, 0, 0, 0, 0, 3162, 0, 0, 3969, 394, 607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 661, 0, 0, 1, 2136, 824, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 254, 3645, 214, 0, 0, 0, 0, 0, 0, 513, 0, 4, 0, 2, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2133, 0, 983, 15, 0, 240, 61, 431, 436, 529, 3, 0, 0, 0, 0, 0, 13, 0, 2685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2361, 6368, 8132, 3630, 4176, 567, 4836, 2698] avg=4096.0000 max=8132 max_rank=2 min=567 min_rank=5 imbalance_ratio=1.985352 max_load_reduction=24636 max_load_reduction_pct=0.751831 moved_to_echo=12277 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4, 178, 0, 3, 94, 0, 221, 1, 0, 700, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3329, 0, 3449, 17, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 28, 0, 274, 2246, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1483, 0, 0, 0, 0, 0, 3848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2096, 81, 5, 0, 1734, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2416, 0, 3, 634, 790, 182, 7, 0, 1175, 395, 557, 115, 0, 0, 0, 0, 0, 0, 0, 2436, 0, 34, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1201, 4094, 6799, 2623, 5332, 3917, 6274, 2528] avg=4096.0000 max=6799 max_rank=2 min=1201 min_rank=0 imbalance_ratio=1.659912 max_load_reduction=25969 max_load_reduction_pct=0.792511 moved_to_echo=18051 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3, 11, 201, 522, 4033, 0, 0, 0, 1379, 811, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 512, 2398, 0, 0, 0, 0, 0, 0, 0, 30, 113, 4091, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 992, 0, 0, 141, 1089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 0, 98, 0, 0, 2958, 212, 0, 0, 0, 0, 0, 0, 0, 1, 1140, 1, 37, 501, 2712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 15, 0, 1319, 0, 0, 569, 458, 5, 180, 0, 0, 0, 0, 0, 0, 0, 2818, 4, 3314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6960, 2911, 4310, 2224, 3274, 4392, 2561, 6136] avg=4096.0000 max=6960 max_rank=0 min=2224 min_rank=3 imbalance_ratio=1.699219 max_load_reduction=25808 max_load_reduction_pct=0.787598 moved_to_echo=16363 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 24, 43, 1204, 2, 1386, 768, 0, 0, 0, 0, 0, 0, 2539, 0, 0, 0, 0, 0, 1, 0, 2408, 0, 0, 0, 0, 0, 0, 0, 1738, 4080, 224, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 762, 372, 2527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 0, 1, 0, 0, 172, 132, 3988, 211, 0, 0, 0, 0, 0, 0, 27, 1, 640, 0, 108, 3533, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 6, 4, 0, 0, 442, 0, 1322, 434, 291, 15, 0, 0, 0, 0, 2, 2, 0, 2672, 0, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3429, 4948, 6043, 3678, 4630, 4536, 2652, 2852] avg=4096.0000 max=6043 max_rank=2 min=2652 min_rank=6 imbalance_ratio=1.475342 max_load_reduction=26725 max_load_reduction_pct=0.815582 moved_to_echo=14670 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 1113, 0, 49, 9, 24, 25, 0, 0, 0, 0, 0, 0, 6, 26, 0, 3743, 0, 15, 24, 8, 41, 0, 0, 0, 0, 0, 0, 0, 1899, 78, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 3368, 35, 0, 276, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 35, 19, 244, 0, 0, 0, 1, 4029, 3, 0, 0, 0, 0, 0, 0, 6, 0, 3907, 0, 0, 95, 921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 3514, 0, 0, 1, 0, 14, 317, 61, 0, 0, 0, 0, 2, 270, 0, 91, 0, 0, 0, 283, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1223, 3863, 2019, 7791, 4332, 4929, 7965, 646] avg=4096.0000 max=7965 max_rank=6 min=646 min_rank=7 imbalance_ratio=1.944580 max_load_reduction=24803 max_load_reduction_pct=0.756927 moved_to_echo=17872 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 16, 4042, 1131, 0, 0, 13, 1367, 809, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 2377, 0, 0, 0, 0, 0, 0, 0, 423, 0, 12, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 71, 252, 4075, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 14, 168, 0, 0, 2030, 209, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 402, 3678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 110, 0, 3186, 0, 0, 2, 0, 224, 457, 70, 244, 0, 0, 0, 0, 11, 0, 0, 2812, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7379, 2381, 450, 4838, 2423, 4084, 4298, 6915] avg=4096.0000 max=7379 max_rank=0 min=450 min_rank=2 imbalance_ratio=1.801514 max_load_reduction=25389 max_load_reduction_pct=0.774811 moved_to_echo=17720 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 574, 3577, 5, 48, 0, 1296, 804, 0, 0, 0, 0, 0, 0, 0, 13, 6, 0, 0, 3, 0, 2, 2256, 0, 0, 0, 0, 0, 0, 0, 0, 1726, 0, 60, 0, 333, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 398, 3356, 2016, 0, 0, 0, 0, 0, 0, 0, 0, 15, 95, 0, 5, 0, 186, 2, 0, 3790, 197, 0, 0, 0, 0, 0, 0, 2603, 0, 18, 686, 1, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 454, 0, 6, 0, 0, 0, 0, 681, 0, 0, 2793, 0, 3602, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6304, 2280, 2120, 5770, 4290, 3405, 1514, 7085] avg=4096.0000 max=7085 max_rank=7 min=1514 min_rank=6 imbalance_ratio=1.729736 max_load_reduction=25683 max_load_reduction_pct=0.783783 moved_to_echo=16294 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1451, 36, 880, 0, 0, 1411, 133, 0, 0, 0, 0, 0, 0, 494, 0, 2, 2975, 0, 0, 0, 0, 2452, 0, 0, 0, 0, 0, 0, 0, 1009, 160, 0, 1000, 0, 726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 7, 0, 0, 4, 2194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 4091, 215, 0, 0, 0, 0, 0, 0, 299, 10, 2638, 5, 0, 466, 3493, 27, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 6, 977, 41, 0, 0, 1154, 76, 172, 236, 0, 0, 0, 0, 0, 0, 0, 439, 0, 3465, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3911, 5923, 2895, 2214, 4311, 6938, 2668, 3908] avg=4096.0000 max=6938 max_rank=5 min=2214 min_rank=3 imbalance_ratio=1.693848 max_load_reduction=25830 max_load_reduction_pct=0.788269 moved_to_echo=17825 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 256, 256, 6784, 10112, 1024, 1536, 128, 8320, 4864, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[2432, 2432, 128, 8960, 5376, 1664, 768, 128, 6016, 2816, 2048, 896, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[11648, 6272, 7808, 5120, 512, 1792, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3456, 128, 10496, 896, 256, 5120, 12928, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[768, 384, 0, 16768, 128, 14720, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 4096, 1152, 128, 1792, 14464, 11392, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[10496, 384, 128, 6784, 128, 128, 384, 640, 14464, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 384, 1536, 256, 128, 512, 256, 512, 28416, 1280, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 42, 1236, 15, 0, 0, 21, 0, 0, 4061, 0, 1, 0, 0, 218, 1, 2627, 0, 96, 0, 1088, 0, 0, 3997, 0, 3882, 6, 0, 0, 0, 0, 0, 0, 82, 3997, 9, 3510, 1, 1, 0, 7, 518, 0, 0, 1, 0, 62, 0, 0, 3864, 0, 0, 0, 6, 2106, 0, 0, 0, 0, 0, 0, 1313, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 3818, 0, 0, 0, 0, 109, 1, 517, 19, 16, 0, 1, 24, 0, 4006, 8, 19, 0, 3, 127, 0, 2199, 0, 0, 0, 90, 3744, 7, 6, 0, 3405, 0, 0, 4096, 4071, 0, 9, 3, 0, 540, 3, 1004, 0, 0, 33, 0, 0, 0, 0, 293, 5, 0, 652, 0, 869, 1, 929, 74, 0, 1, 2066] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 2, 0, 3863, 2, 0, 2, 0, 3, 2, 5, 31, 0, 0, 0, 0, 0, 1699, 0, 1, 0, 4096, 1779, 0, 0, 1, 4092, 67, 26, 4094, 0, 0, 0, 0, 0, 1, 0, 3944, 0, 3, 0, 0, 0, 0, 2579, 1654, 0, 0, 0, 23, 3113, 0, 0, 0, 0, 1357, 0, 205, 123, 0, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 154, 0, 2, 0, 0, 0, 1, 39, 0, 3158, 0, 0, 947, 1, 0, 0, 1845, 3, 11, 1749, 13, 3, 0, 631, 0, 2809, 1478, 0, 0, 0, 97, 0, 0, 3, 27, 226, 4092, 0, 0, 0, 17, 3947, 0, 3911, 0, 0, 1, 0, 213, 332, 0, 0, 2, 0, 3116, 0, 1, 0, 0, 0, 0, 0, 3939] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 13, 655, 0, 0, 0, 2, 0, 0, 1, 0, 6, 0, 0, 0, 4, 2852, 0, 2327, 0, 0, 125, 5, 30, 0, 12, 0, 0, 4094, 15, 10, 0, 117, 0, 0, 1171, 4094, 0, 0, 0, 1, 48, 0, 4049, 3939, 0, 0, 0, 0, 3598, 2, 0, 0, 0, 4061, 0, 15, 0, 1163, 1, 0, 7, 351] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 4033, 0, 11, 337, 0, 0, 2, 25, 8, 0, 0, 71, 8, 0, 0, 0, 0, 47, 15, 3845, 0, 3759, 0, 3829, 0, 0, 0, 0, 0, 5, 0, 3, 0, 1, 3382, 793, 0, 41, 0, 1, 0, 331, 0, 1, 0, 0, 0, 37, 1, 0, 0, 0, 4095, 0, 3974, 0, 3878, 225, 8, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 1, 647, 791, 26, 0, 0, 0, 0, 0, 3231, 0, 14, 0, 216, 2471, 0, 696, 0, 0, 200, 0, 185, 1, 0, 1630, 9, 0, 36, 239, 0, 3, 0, 0, 0, 1, 3591, 1376, 0, 0, 0, 0, 3, 10, 3936, 58, 389, 0, 0, 0, 4035, 0, 0, 0, 56, 3670, 0, 251, 0, 929, 0, 0, 0, 4067] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 1674, 3159, 0, 0, 0, 0, 0, 0, 3844, 0, 0, 0, 20, 0, 0, 0, 2, 4096, 26, 18, 958, 0, 2404, 1, 840, 0, 0, 0, 11, 0, 1, 850, 120, 21, 421, 1204, 0, 14, 0, 1113, 2, 0, 2555, 268, 0, 0, 12, 0, 1016, 0, 0, 4, 1550, 4084, 0, 1468, 195, 813, 2, 2, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3818, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 109, 1, 517, 19, 16, 0, 1, 24, 1882, 0, 0, 0, 0, 0, 0, 0, 0, 3480, 8, 19, 0, 3, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2199, 0, 0, 0, 90, 3744, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3405, 0, 0, 4096, 2189, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 540, 3, 1004, 0, 0, 33, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 5, 0, 412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 869, 1, 929, 74, 0, 1, 2066, 164, 18, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3894, 2569, 3637, 6046, 9699, 2091, 710, 4122] avg=4096.0000 max=9699 max_rank=4 min=710 min_rank=6 imbalance_ratio=2.367920 max_load_reduction=23069 max_load_reduction_pct=0.704010 moved_to_echo=16622 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 3863, 2, 0, 2, 0, 2527, 158, 0, 0, 0, 0, 0, 0, 3, 2, 5, 31, 0, 0, 0, 0, 1824, 482, 0, 0, 0, 0, 0, 0, 0, 1476, 0, 1, 0, 4096, 1779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1083, 67, 26, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2120, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2579, 1654, 0, 0, 215, 0, 0, 0, 0, 0, 0, 0, 0, 23, 3113, 0, 0, 0, 0, 858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 205, 123, 0, 0, 1, 0, 0, 341, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6554, 2347, 7352, 5271, 2124, 4448, 3994, 678] avg=4096.0000 max=7352 max_rank=2 min=678 min_rank=7 imbalance_ratio=1.794922 max_load_reduction=25416 max_load_reduction_pct=0.775635 moved_to_echo=11244 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 42, 1236, 15, 0, 0, 21, 2398, 246, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 1, 0, 0, 218, 1623, 459, 0, 0, 0, 0, 0, 0, 1, 2282, 0, 96, 0, 1088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3997, 0, 1025, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 3997, 9, 1887, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 518, 0, 0, 1, 0, 62, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3864, 0, 0, 0, 6, 1330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1313, 0, 530, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3958, 6362, 3467, 5028, 5977, 921, 5200, 1855] avg=4096.0000 max=6362 max_rank=1 min=921 min_rank=5 imbalance_ratio=1.553223 max_load_reduction=26406 max_load_reduction_pct=0.805847 moved_to_echo=13953 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4033, 0, 11, 337, 0, 2364, 478, 0, 0, 0, 0, 0, 0, 0, 2, 25, 8, 0, 0, 71, 8, 1564, 451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 15, 3845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3759, 0, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3, 0, 1, 1818, 793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 1, 0, 331, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 1, 0, 0, 0, 2586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3974, 0, 3878, 225, 8, 0, 2, 1031, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7223, 2129, 3907, 4773, 2620, 374, 2624, 9118] avg=4096.0000 max=9118 max_rank=7 min=374 min_rank=5 imbalance_ratio=2.226074 max_load_reduction=23650 max_load_reduction_pct=0.721741 moved_to_echo=14736 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 13, 655, 0, 0, 0, 2, 7, 474, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, 0, 1893, 1, 0, 0, 0, 0, 0, 0, 4, 2477, 0, 2327, 0, 0, 125, 5, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 4, 0, 0, 4094, 15, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 0, 0, 1171, 2201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 48, 0, 4049, 3939, 0, 0, 362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3598, 2, 0, 0, 0, 2564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 1163, 1, 0, 7, 351, 1023, 13, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1151, 1901, 4938, 4153, 3489, 8399, 6164, 2573] avg=4096.0000 max=8399 max_rank=5 min=1151 min_rank=0 imbalance_ratio=2.050537 max_load_reduction=24369 max_load_reduction_pct=0.743683 moved_to_echo=20625 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1674, 3159, 0, 0, 0, 0, 518, 477, 0, 0, 0, 0, 0, 0, 0, 0, 3844, 0, 0, 0, 20, 0, 556, 99, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 26, 18, 958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2404, 1, 223, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 850, 120, 21, 421, 648, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1113, 2, 0, 2555, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 1016, 0, 0, 4, 1550, 2579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1468, 195, 813, 2, 2, 0, 0, 1028, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5828, 4519, 5100, 2639, 2075, 3938, 5161, 3508] avg=4096.0000 max=5828 max_rank=0 min=2075 min_rank=4 imbalance_ratio=1.422852 max_load_reduction=26940 max_load_reduction_pct=0.822144 moved_to_echo=14682 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 647, 791, 26, 0, 0, 0, 8, 432, 0, 0, 0, 0, 0, 0, 0, 0, 3231, 0, 14, 0, 216, 2471, 640, 1, 0, 0, 0, 0, 0, 0, 0, 599, 0, 0, 200, 0, 185, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1630, 0, 0, 36, 239, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3591, 736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 10, 3936, 58, 389, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 0, 0, 0, 56, 2310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 0, 929, 0, 0, 0, 4067, 928, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1905, 6573, 985, 1908, 4328, 4487, 6401, 6181] avg=4096.0000 max=6573 max_rank=1 min=985 min_rank=2 imbalance_ratio=1.604736 max_load_reduction=26195 max_load_reduction_pct=0.799408 moved_to_echo=21397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 154, 0, 2, 0, 0, 0, 1, 1734, 364, 0, 0, 0, 0, 0, 0, 39, 0, 3158, 0, 0, 947, 1, 0, 1892, 331, 0, 0, 0, 0, 0, 0, 0, 1603, 3, 11, 1749, 13, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 631, 0, 744, 1478, 0, 0, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 27, 226, 2200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 3947, 0, 3911, 0, 0, 1, 234, 0, 0, 0, 0, 0, 0, 0, 0, 213, 332, 0, 0, 2, 0, 1967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3939, 785, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2255, 6368, 3382, 2950, 2456, 8110, 2514, 4733] avg=4096.0000 max=8110 max_rank=5 min=2255 min_rank=0 imbalance_ratio=1.979980 max_load_reduction=24658 max_load_reduction_pct=0.752502 moved_to_echo=17813 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 4480, 256, 4096, 9600, 13824, 896, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 6784, 384, 7808, 384, 128, 1408, 10496, 5888, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 1152, 5120, 128, 18432, 6016, 512, 128, 1792, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 256, 16000, 128, 384, 128, 1664, 14720, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 256, 2432, 17664, 128, 128, 384, 128, 9600, 2816, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 12032, 128, 6656, 2048, 5248, 7040, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[256, 128, 14848, 128, 128, 1024, 384, 2816, 11904, 1920, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[13056, 1664, 4096, 1664, 256, 12288, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 44, 0, 0, 6, 0, 0, 0, 1084, 0, 0, 0, 44, 1, 3, 0, 2, 27, 0, 0, 0, 2398, 2164, 81, 3912, 4, 130, 0, 0, 0, 0, 2726, 770, 159, 0, 0, 0, 449, 1333, 1304, 0, 1, 0, 0, 15, 0, 2278, 151, 1, 0, 0, 178, 2070, 3999, 0, 3376, 4057, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1371, 0, 0, 3500, 0, 1493, 616, 210, 0, 65, 0, 0, 0, 0, 0, 0, 161, 4, 0, 3083, 45, 0, 3991, 0, 0, 0, 3362, 3419, 0, 3868, 0, 0, 41, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 3943, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2399, 0, 0, 0, 1188, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3, 0, 0, 0, 374, 0, 0, 1, 2, 0, 1179, 0, 0, 0, 3773, 134, 1126, 0, 1, 4096, 0, 1, 1556, 1, 3918, 0, 0, 0, 0, 0, 0, 16, 10, 37, 0, 3188, 0, 4071, 754, 208, 0, 8, 0, 2, 0, 0, 3, 199, 34, 0, 0, 0, 18, 3428, 0, 161, 4070, 0, 396] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[517, 3786, 0, 0, 2, 229, 2, 101, 0, 0, 0, 31, 56, 0, 1217, 0, 68, 0, 0, 444, 0, 80, 4070, 141, 10, 3, 0, 506, 73, 3290, 41, 0, 549, 0, 0, 610, 0, 3912, 51, 0, 0, 0, 1823, 150, 3818, 0, 0, 1754, 1, 0, 0, 5, 1092, 0, 21, 0, 0, 212, 0, 0, 0, 399, 0, 3704] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 1297, 2425, 0, 0, 0, 160, 0, 0, 0, 0, 1, 0, 1, 607, 2623, 3, 911, 0, 0, 1928, 0, 0, 160, 0, 3672, 376, 3, 2471, 0, 45, 15, 0, 109, 0, 0, 0, 0, 3535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 239, 4012, 12, 0, 0, 4063, 1, 0, 46, 3869, 0, 182] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1, 6, 0, 0, 3, 2, 1, 29, 0, 0, 421, 0, 0, 0, 0, 3980, 15, 67, 0, 2992, 0, 0, 0, 0, 10, 1633, 0, 4096, 0, 3613, 60, 0, 3823, 0, 0, 0, 32, 0, 96, 0, 0, 0, 1648, 0, 476, 12, 0, 1, 0, 288, 5, 36, 0, 392, 54, 0, 39, 2470, 3612, 5, 44, 2805, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 0, 1805, 0, 9, 0, 108, 0, 0, 0, 16, 0, 264, 0, 0, 0, 4002, 0, 0, 0, 0, 0, 0, 87, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 3772, 0, 0, 110, 1054, 4018, 382, 0, 5, 0, 284, 0, 0, 3, 0, 0, 705, 0, 0, 0, 215, 4084, 0, 4092, 3656, 0, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1991, 2, 0, 0, 1437, 0, 19, 149, 20, 3, 0, 0, 0, 0, 138, 0, 0, 0, 11, 37, 0, 7, 4089, 0, 3295, 0, 0, 0, 0, 0, 63, 0, 3836, 0, 0, 65, 3658, 1209, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 3, 0, 462, 4051, 0, 1, 4092, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 44, 0, 196, 1008, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 1084, 0, 0, 3049, 532, 0, 0, 0, 0, 0, 0, 0, 44, 1, 3, 0, 2, 27, 0, 875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2398, 2164, 81, 322, 4, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2726, 770, 159, 0, 1323, 87, 0, 0, 0, 0, 0, 0, 0, 0, 166, 1333, 1304, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 2278, 151, 1, 0, 0, 3058, 0, 0, 0, 0, 0, 0, 0, 178, 2070, 1801, 0, 3376, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1248, 4671, 952, 5099, 5065, 2804, 5503, 7426] avg=4096.0000 max=7426 max_rank=7 min=952 min_rank=2 imbalance_ratio=1.812988 max_load_reduction=25342 max_load_reduction_pct=0.773376 moved_to_echo=20798 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1371, 0, 0, 3500, 0, 1493, 616, 210, 0, 295, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 893, 526, 0, 0, 0, 0, 0, 0, 161, 4, 0, 3083, 45, 0, 3991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3362, 3419, 0, 319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3023, 0, 0, 0, 0, 0, 0, 0, 0, 2399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7485, 1484, 7284, 7100, 49, 3943, 3024, 2399] avg=4096.0000 max=7485 max_rank=0 min=49 min_rank=4 imbalance_ratio=1.827393 max_load_reduction=25283 max_load_reduction_pct=0.771576 moved_to_echo=9415 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[517, 3786, 0, 0, 2, 229, 2, 101, 796, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 56, 0, 1217, 0, 300, 447, 0, 0, 0, 0, 0, 0, 68, 0, 0, 444, 0, 80, 4070, 141, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3, 0, 506, 73, 272, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 0, 0, 610, 0, 3912, 51, 0, 0, 355, 0, 0, 0, 0, 0, 0, 0, 0, 672, 150, 3818, 0, 0, 1754, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5, 1092, 0, 21, 0, 2571, 0, 0, 0, 0, 0, 0, 0, 0, 212, 0, 0, 0, 0, 0, 3704, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5532, 2051, 4803, 905, 5477, 6394, 3690, 3916] avg=4096.0000 max=6394 max_rank=5 min=905 min_rank=3 imbalance_ratio=1.561035 max_load_reduction=26374 max_load_reduction_pct=0.804871 moved_to_echo=19477 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3, 0, 0, 1778, 1011, 0, 0, 0, 0, 0, 0, 0, 374, 0, 0, 1, 2, 0, 1179, 3059, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3773, 134, 1126, 0, 1, 750, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 1556, 1, 323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 10, 37, 0, 1134, 794, 0, 0, 0, 0, 0, 0, 3188, 0, 1499, 754, 208, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3, 199, 34, 0, 0, 3062, 0, 0, 0, 0, 0, 0, 0, 0, 18, 1544, 0, 161, 0, 0, 396, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2792, 5148, 5784, 5977, 1991, 5657, 3300, 2119] avg=4096.0000 max=5977 max_rank=3 min=1991 min_rank=4 imbalance_ratio=1.459229 max_load_reduction=26791 max_load_reduction_pct=0.817596 moved_to_echo=13067 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 1297, 2425, 0, 1544, 962, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 0, 0, 0, 1, 2907, 499, 0, 0, 0, 0, 0, 0, 0, 1, 607, 2623, 3, 911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1928, 0, 0, 160, 0, 303, 376, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2471, 0, 45, 15, 0, 109, 0, 0, 0, 689, 0, 0, 0, 0, 0, 0, 0, 0, 1302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 239, 4012, 12, 0, 2870, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 1, 0, 46, 0, 0, 182, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6230, 3567, 4145, 2770, 3329, 1302, 7133, 4292] avg=4096.0000 max=7133 max_rank=6 min=1302 min_rank=5 imbalance_ratio=1.741455 max_load_reduction=25635 max_load_reduction_pct=0.782318 moved_to_echo=16056 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1991, 2, 0, 528, 1016, 0, 0, 0, 0, 0, 0, 0, 1437, 0, 19, 149, 20, 3, 0, 3076, 448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 0, 0, 0, 11, 887, 0, 0, 0, 0, 0, 0, 0, 37, 0, 7, 4089, 0, 272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 3836, 0, 0, 1341, 235, 0, 0, 0, 0, 0, 0, 65, 3658, 446, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 3, 2575, 0, 0, 0, 0, 0, 0, 0, 0, 462, 1823, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3537, 5152, 1036, 4405, 5475, 8264, 2611, 2288] avg=4096.0000 max=8264 max_rank=5 min=1036 min_rank=2 imbalance_ratio=2.017578 max_load_reduction=24504 max_load_reduction_pct=0.747803 moved_to_echo=18638 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 6, 0, 0, 3, 2, 1, 29, 719, 697, 0, 0, 0, 0, 0, 0, 0, 0, 421, 0, 0, 0, 0, 3980, 2108, 491, 0, 0, 0, 0, 0, 0, 15, 67, 0, 2992, 0, 0, 0, 0, 791, 0, 0, 0, 0, 0, 0, 0, 10, 1633, 0, 4096, 0, 298, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3823, 0, 0, 0, 32, 0, 96, 0, 1195, 321, 0, 0, 0, 0, 0, 0, 0, 0, 608, 0, 476, 12, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 288, 5, 36, 0, 392, 54, 0, 2824, 0, 0, 0, 0, 0, 0, 0, 39, 2470, 1626, 5, 44, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1458, 7000, 3865, 6097, 5467, 1097, 3599, 4185] avg=4096.0000 max=7000 max_rank=1 min=1097 min_rank=5 imbalance_ratio=1.708984 max_load_reduction=25768 max_load_reduction_pct=0.786377 moved_to_echo=14348 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 0, 1805, 0, 9, 1758, 909, 0, 0, 0, 0, 0, 0, 0, 108, 0, 0, 0, 16, 0, 264, 2747, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4002, 0, 0, 0, 0, 897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3772, 0, 0, 1355, 788, 0, 0, 0, 0, 0, 0, 110, 1054, 1472, 382, 0, 5, 0, 284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 705, 0, 0, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 215, 1832, 0, 4092, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4486, 3695, 4899, 415, 5915, 3307, 3908, 6143] avg=4096.0000 max=6143 max_rank=7 min=415 min_rank=3 imbalance_ratio=1.499756 max_load_reduction=26625 max_load_reduction_pct=0.812531 moved_to_echo=19273 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[6912, 0, 128, 768, 2816, 12544, 384, 0, 6400, 3328, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3456, 4736, 6272, 10752, 5888, 128, 128, 2048, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[6144, 1664, 5888, 16000, 256, 2560, 512, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 128, 640, 17152, 256, 2176, 8192, 256, 4224, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1920, 3840, 0, 3584, 128, 6912, 3200, 384, 7424, 6016, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 2048, 640, 128, 256, 1152, 1280, 5504, 18176, 4096, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 12032, 8704, 128, 7808, 0, 0, 4352, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 384, 128, 2432, 1792, 5248, 128, 128, 23296, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[8, 243, 0, 0, 4093, 2911, 0, 545, 1, 851, 26, 0, 0, 92, 0, 1, 1, 4050, 1, 0, 3, 1721, 1071, 3736, 0, 25, 5, 0, 0, 0, 0, 22, 0, 0, 1948, 3, 3154, 50, 0, 0, 272, 886, 0, 0, 3, 0, 0, 89, 3717, 0, 0, 230, 0, 0, 0, 0, 0, 0, 1635, 1, 943, 430, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 3992, 2971, 3989, 0, 75, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 1526, 0, 1, 12, 4, 1901, 0, 391, 0, 512, 0, 2, 11, 3598, 0, 73, 16, 3646, 8, 3871, 12, 1, 3, 0, 191, 1, 0, 0, 14, 0, 0, 2759, 0, 0, 0, 0, 160, 0, 0, 299, 0, 24, 5, 284, 2336, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 2953, 0, 0, 3, 4042, 1195, 0, 0, 0, 0, 0, 7, 8, 0, 0, 0, 2742, 0, 0, 0, 11, 0, 2392, 0, 0, 0, 1, 32, 0, 1593, 0, 12, 8, 4091, 0, 3830, 0, 46, 0, 0, 0, 29, 0, 13, 0, 0, 3649, 4078, 0, 0, 10, 0, 0, 5, 0, 0, 0, 0, 1, 82, 0, 0, 1935] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 148, 4085, 3452, 15, 2709, 0, 0, 69, 0, 1, 0, 0, 0, 625, 100, 1123, 0, 63, 0, 0, 0, 2629, 0, 0, 0, 2, 0, 0, 53, 0, 364, 121, 1356, 3, 4072, 0, 115, 0, 5, 0, 167, 0, 5, 0, 0, 0, 4019, 1, 0, 0, 1933, 0, 1582, 19, 1, 3, 22, 15, 2204, 1687, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 89, 0, 431, 0, 2901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3976, 188, 0, 0, 6, 48, 0, 0, 0, 203, 0, 982, 0, 2610, 0, 0, 0, 3994, 0, 4017, 0, 0, 0, 0, 63, 25, 3038, 0, 0, 0, 24, 3966, 0, 0, 0, 0, 9, 4013, 0, 0, 0, 0, 0, 2185, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 274, 217, 3156, 403, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1033, 0, 3853, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 5, 2829, 0, 0, 0, 0, 0, 216, 1, 3624, 0, 1, 43, 0, 0, 418, 2, 0, 0, 0, 0, 3981, 0, 0, 0, 224, 0, 4086, 273, 75, 0, 0, 0, 280, 3682, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 650, 0, 255, 3159, 12, 0, 24, 0, 0, 0, 0, 0, 267, 1, 1631, 0, 2497, 0, 0, 13, 56, 1489, 4038, 0, 0, 1514, 0, 0, 3, 2410, 0, 31, 1276, 2306, 0, 150, 0, 0, 2820, 0, 72, 182, 3919, 0, 7, 0, 0, 3808, 0, 0, 1, 0, 0, 0, 0, 177, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 153, 639, 1, 4023, 2623, 95, 0, 0, 0, 0, 0, 21, 937, 0, 0, 0, 2992, 4, 0, 0, 2, 0, 25, 0, 0, 0, 4, 220, 0, 39, 0, 655, 1283, 3708, 1709, 4096, 0, 0, 130, 0, 0, 1, 3963, 5, 2700, 0, 0, 1634, 1, 13, 0, 811, 9, 1, 0, 0, 3, 0, 3, 261, 0, 0, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[8, 243, 0, 0, 0, 2487, 0, 545, 0, 0, 0, 0, 0, 0, 0, 0, 1, 851, 26, 0, 0, 92, 0, 1, 2834, 554, 0, 0, 0, 0, 0, 0, 1, 1631, 1, 0, 3, 1721, 1071, 3736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 5, 0, 0, 0, 0, 22, 2882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1948, 3, 320, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 272, 886, 0, 0, 3, 0, 0, 89, 1139, 532, 424, 0, 0, 0, 0, 0, 2578, 0, 0, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1635, 1, 943, 430, 0, 1, 1887, 657, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3283, 4359, 8164, 2934, 2321, 3345, 2808, 5554] avg=4096.0000 max=8164 max_rank=2 min=2321 min_rank=4 imbalance_ratio=1.993164 max_load_reduction=24604 max_load_reduction_pct=0.750854 moved_to_echo=14028 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 148, 4085, 0, 13, 2709, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 1, 0, 0, 0, 625, 3659, 468, 0, 0, 0, 0, 0, 0, 100, 453, 0, 63, 0, 0, 0, 2629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 53, 0, 2430, 0, 0, 0, 0, 0, 0, 0, 364, 121, 1356, 3, 413, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 167, 0, 5, 0, 0, 0, 1232, 147, 2, 0, 0, 0, 0, 0, 2787, 1, 0, 0, 1933, 0, 1582, 19, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 22, 15, 2204, 1687, 0, 0, 523, 554, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6955, 4822, 3245, 2485, 2372, 1558, 6322, 5009] avg=4096.0000 max=6955 max_rank=0 min=1558 min_rank=5 imbalance_ratio=1.697998 max_load_reduction=25813 max_load_reduction_pct=0.787750 moved_to_echo=15261 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 650, 0, 255, 0, 11, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 267, 1, 1631, 134, 427, 0, 0, 0, 0, 0, 0, 0, 1006, 0, 0, 13, 56, 1489, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1514, 0, 0, 3, 2410, 0, 2224, 0, 0, 0, 0, 0, 0, 0, 31, 1276, 2306, 0, 16, 0, 0, 2820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 182, 3919, 0, 7, 0, 0, 1167, 328, 1, 0, 0, 0, 0, 0, 2641, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177, 0, 0, 0, 0, 0, 0, 0, 1163, 508, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[940, 2460, 6602, 6151, 6449, 5676, 2642, 1848] avg=4096.0000 max=6602 max_rank=2 min=940 min_rank=0 imbalance_ratio=1.611816 max_load_reduction=26166 max_load_reduction_pct=0.798523 moved_to_echo=16615 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3992, 2971, 0, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 0, 0, 3478, 540, 0, 0, 0, 0, 0, 0, 0, 615, 0, 1, 12, 4, 1901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 391, 0, 512, 0, 2, 11, 3598, 0, 2809, 0, 0, 0, 0, 0, 0, 0, 73, 16, 3646, 8, 393, 12, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 191, 1, 0, 0, 14, 0, 0, 846, 200, 0, 0, 0, 0, 0, 0, 1913, 0, 0, 0, 0, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 299, 0, 24, 5, 284, 2336, 0, 2, 711, 640, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7038, 4096, 2533, 7323, 4152, 1252, 2073, 4301] avg=4096.0000 max=7323 max_rank=3 min=1252 min_rank=5 imbalance_ratio=1.787842 max_load_reduction=25445 max_load_reduction_pct=0.776520 moved_to_echo=11778 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 2953, 0, 0, 0, 3453, 1195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8, 0, 0, 3441, 0, 0, 0, 0, 0, 0, 0, 0, 1105, 0, 0, 0, 11, 0, 2392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 32, 0, 1593, 0, 2, 0, 0, 0, 0, 0, 0, 0, 12, 8, 4091, 0, 389, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 13, 0, 0, 3649, 1250, 360, 589, 0, 0, 0, 0, 0, 2828, 0, 0, 10, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 82, 0, 0, 1935, 1277, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7601, 3456, 3508, 1628, 4546, 5890, 2843, 3296] avg=4096.0000 max=7601 max_rank=0 min=1628 min_rank=3 imbalance_ratio=1.855713 max_load_reduction=25167 max_load_reduction_pct=0.768036 moved_to_echo=16575 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 89, 0, 0, 0, 2901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3609, 58, 0, 0, 0, 0, 0, 0, 0, 1602, 188, 0, 0, 6, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 203, 0, 982, 0, 2610, 0, 303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3994, 0, 408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 25, 3038, 0, 0, 0, 24, 1216, 522, 0, 0, 0, 0, 0, 0, 2750, 0, 0, 0, 0, 9, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2185, 0, 0, 0, 1852, 70, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2990, 3667, 1844, 4098, 4402, 4888, 6772, 4107] avg=4096.0000 max=6772 max_rank=6 min=1844 min_rank=2 imbalance_ratio=1.653320 max_load_reduction=25996 max_load_reduction_pct=0.793335 moved_to_echo=20169 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 153, 639, 1, 0, 2241, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 937, 0, 0, 3680, 546, 0, 0, 0, 0, 0, 0, 0, 1205, 4, 0, 0, 2, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 220, 0, 39, 0, 2831, 0, 0, 0, 0, 0, 0, 0, 655, 1283, 3708, 1709, 416, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3963, 5, 2700, 0, 0, 501, 393, 382, 0, 0, 0, 0, 0, 1133, 1, 13, 0, 811, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 261, 0, 0, 4, 1394, 646, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3129, 5184, 1236, 3094, 7901, 7945, 1968, 2311] avg=4096.0000 max=7945 max_rank=5 min=1236 min_rank=2 imbalance_ratio=1.939697 max_load_reduction=24823 max_load_reduction_pct=0.757538 moved_to_echo=20125 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 274, 217, 0, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1033, 3260, 429, 0, 0, 0, 0, 0, 0, 0, 1546, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2829, 0, 0, 0, 2221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 1, 364, 0, 1, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 418, 2, 0, 0, 0, 0, 1224, 508, 62, 0, 0, 0, 0, 0, 2757, 0, 0, 0, 224, 0, 4086, 273, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 280, 3682, 0, 0, 1799, 506, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[832, 4724, 5636, 5055, 625, 2214, 7340, 6342] avg=4096.0000 max=7340 max_rank=6 min=625 min_rank=4 imbalance_ratio=1.791992 max_load_reduction=25428 max_load_reduction_pct=0.776001 moved_to_echo=16521 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[384, 1280, 896, 11008, 128, 2816, 0, 3840, 8576, 3072, 1536, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 4096, 5248, 7552, 0, 8576, 7040, 640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[19456, 128, 128, 256, 3072, 256, 9728, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 9216, 256, 128, 128, 1920, 4608, 17024, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[512, 128, 2304, 128, 4096, 128, 10368, 128, 15744, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1152, 2816, 21376, 1792, 2816, 128, 256, 3072, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[640, 128, 1792, 128, 6272, 8192, 0, 2048, 10624, 3584, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 1024, 128, 128, 128, 1408, 128, 3328, 24192, 3072, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 9, 1, 3819, 0, 3019, 0, 0, 6, 1244, 0, 588, 0, 106, 0, 0, 2580, 0, 0, 0, 0, 0, 36, 0, 0, 2, 0, 2, 4085, 0, 7, 0, 0, 0, 0, 4043, 0, 0, 6, 3840, 2194, 0, 10, 66, 1, 159, 0, 1, 4021, 0, 0, 0, 6, 0, 2091, 0, 0, 0, 808, 1, 17, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 1839, 0, 5, 4091, 71, 3821, 0, 0, 63, 0, 0, 0, 34, 0, 0, 4029, 3996, 233, 2163, 3984, 0, 6, 93, 142, 0, 0, 0, 10, 0, 0, 0, 824, 0, 2, 14, 2185, 0, 0, 0, 3015, 0, 0, 0, 0, 4, 0, 168, 0, 0, 36, 1888, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 33, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 2, 2603, 4, 0, 3805, 0, 1, 0, 0, 0, 563, 1, 0, 7, 643, 3, 37, 567, 4054, 0, 0, 338, 0, 926, 0, 230, 0, 178, 0, 3, 37, 0, 0, 98, 3172, 0, 5, 0, 3485, 0, 0, 0, 1, 0, 0, 3345, 11, 0, 233, 362, 0, 0, 0, 3946, 0, 1, 0, 24, 4083, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 1232, 3292, 0, 1447, 0, 0, 0, 3003, 3053, 3710, 0, 13, 0, 0, 31, 0, 2, 8, 0, 0, 0, 19, 0, 3, 2223, 0, 0, 0, 17, 6, 54, 3960, 0, 0, 725, 4, 208, 207, 0, 0, 46, 0, 0, 0, 0, 0, 3920, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1488, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 43, 3599, 191, 0, 0, 0, 0, 0, 0, 0, 0, 3711, 0, 0, 233, 0, 5, 0, 0, 1, 0, 0, 5, 3907, 3181, 1182, 0, 0, 0, 1324, 4092, 0, 0, 2767, 6, 4090, 0, 30, 0, 0, 32, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 3743, 0, 529, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 422, 5, 3722, 3724, 245, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3321, 3929, 0, 166, 13, 0, 0, 21, 1, 8, 0, 0, 0, 3661, 0, 0, 2575, 0, 0, 0, 1688, 0, 0, 0, 3416, 0, 0, 49, 0, 0, 0, 0, 1529, 0, 0, 9, 2, 0, 8, 6, 4, 0, 0, 0, 147, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1065, 46, 4054, 96, 153, 0, 0, 0, 69, 0, 0, 0, 3985, 3003, 0, 202, 4035, 0, 0, 2, 0, 0, 0, 0, 0, 4066, 0, 0, 15, 3, 0, 16, 4072, 0, 0, 3, 4082, 0, 0, 3, 0, 0, 0, 3760, 6, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 1, 4093, 3765, 11, 1282, 1, 1, 0, 0, 0, 1, 75, 0, 0, 0, 0, 0, 0, 0, 242, 0, 0, 1, 4, 0, 0, 3994, 2257, 0, 0, 3149, 2300, 0, 2, 1190, 0, 0, 136, 3, 0, 0, 0, 0, 23, 0, 3432, 0, 87, 0, 0, 0, 0, 0, 2310, 4, 0, 0, 0, 4096, 0, 0, 308] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 9, 1, 3819, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1244, 0, 588, 0, 106, 0, 2087, 0, 0, 0, 0, 0, 0, 0, 0, 2580, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 4085, 0, 7, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2905, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 3332, 2194, 0, 10, 66, 1, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4021, 0, 0, 0, 6, 0, 2091, 0, 1138, 41, 0, 0, 0, 0, 0, 0, 0, 0, 808, 1, 17, 0, 0, 0, 746, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4015, 4031, 2616, 4563, 2911, 5762, 7298, 1572] avg=4096.0000 max=7298 max_rank=6 min=1572 min_rank=7 imbalance_ratio=1.781738 max_load_reduction=25470 max_load_reduction_pct=0.777283 moved_to_echo=17543 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 1232, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1447, 0, 0, 0, 3003, 3053, 3710, 0, 2275, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 31, 0, 2, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 3, 2223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 6, 54, 2846, 0, 0, 725, 0, 0, 0, 0, 0, 0, 0, 0, 4, 208, 207, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3920, 0, 0, 0, 0, 5, 1114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1488, 0, 0, 5, 813, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1437, 13488, 44, 2245, 3648, 465, 5039, 6402] avg=4096.0000 max=13488 max_rank=1 min=44 min_rank=2 imbalance_ratio=3.292969 max_load_reduction=19280 max_load_reduction_pct=0.588379 moved_to_echo=15554 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 2603, 4, 0, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 563, 1, 0, 7, 2630, 0, 0, 0, 0, 0, 0, 0, 115, 3, 37, 567, 4054, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 926, 0, 230, 0, 178, 0, 3, 424, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 98, 2279, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3024, 0, 0, 0, 1, 0, 0, 3345, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 233, 362, 0, 0, 0, 3946, 254, 893, 37, 0, 0, 0, 0, 0, 0, 1, 0, 24, 4083, 0, 0, 0, 274, 940, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2844, 3202, 5114, 1761, 2419, 6370, 5736, 5322] avg=4096.0000 max=6370 max_rank=5 min=1761 min_rank=3 imbalance_ratio=1.555176 max_load_reduction=26398 max_load_reduction_pct=0.805603 moved_to_echo=19847 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 43, 223, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2487, 0, 0, 0, 0, 0, 0, 0, 653, 0, 0, 233, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5, 3907, 3181, 1182, 0, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1324, 4092, 0, 0, 2767, 6, 0, 0, 0, 0, 0, 0, 0, 0, 3545, 0, 30, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 1472, 0, 46, 0, 0, 0, 0, 0, 0, 0, 15, 0, 3743, 0, 529, 0, 1586, 889, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[459, 2487, 891, 8775, 8189, 3607, 1598, 6762] avg=4096.0000 max=8775 max_rank=3 min=459 min_rank=0 imbalance_ratio=2.142334 max_load_reduction=23993 max_load_reduction_pct=0.732208 moved_to_echo=20156 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1839, 0, 5, 4091, 71, 236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 34, 0, 0, 2641, 0, 0, 0, 0, 0, 0, 0, 716, 3996, 233, 2163, 3984, 0, 6, 93, 0, 0, 0, 0, 0, 0, 0, 0, 142, 0, 0, 0, 10, 0, 0, 0, 366, 0, 0, 0, 0, 0, 0, 0, 824, 0, 2, 14, 1570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2617, 0, 0, 0, 0, 4, 0, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 1888, 0, 0, 19, 0, 1595, 615, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 1718, 944, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6242, 2738, 11191, 518, 2410, 2789, 4185, 2695] avg=4096.0000 max=11191 max_rank=2 min=518 min_rank=3 imbalance_ratio=2.732178 max_load_reduction=21577 max_load_reduction_pct=0.658478 moved_to_echo=12079 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1065, 46, 250, 96, 0, 0, 0, 0, 0, 0, 0, 0, 153, 0, 0, 0, 69, 0, 0, 0, 2802, 0, 0, 0, 0, 0, 0, 0, 708, 3003, 0, 202, 4035, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 496, 0, 0, 0, 0, 0, 0, 0, 15, 3, 0, 16, 2926, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3543, 0, 0, 3, 0, 0, 0, 3760, 0, 0, 0, 0, 0, 0, 0, 0, 6, 27, 0, 0, 0, 0, 0, 0, 1577, 1146, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1700, 1002, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1457, 3024, 7950, 4562, 2963, 7306, 2799, 2707] avg=4096.0000 max=7950 max_rank=2 min=1457 min_rank=0 imbalance_ratio=1.940918 max_load_reduction=24818 max_load_reduction_pct=0.757385 moved_to_echo=15775 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 422, 5, 3722, 3724, 245, 253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2831, 0, 0, 0, 0, 0, 0, 0, 591, 3929, 0, 166, 13, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8, 0, 0, 0, 3661, 0, 0, 415, 0, 0, 0, 0, 0, 0, 0, 2575, 0, 0, 0, 1213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2965, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1529, 0, 0, 9, 2, 0, 8, 6, 1314, 475, 36, 0, 0, 0, 0, 0, 4, 0, 0, 0, 147, 0, 0, 1, 1416, 1012, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8371, 2831, 4720, 4085, 3788, 3014, 3379, 2580] avg=4096.0000 max=8371 max_rank=0 min=2580 min_rank=7 imbalance_ratio=2.043701 max_load_reduction=24397 max_load_reduction_pct=0.744537 moved_to_echo=12761 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 4093, 3765, 11, 72, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 75, 0, 0, 890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 3994, 2257, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3149, 2300, 0, 2, 853, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 3432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 0, 0, 0, 2310, 0, 337, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 308, 0, 320, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7943, 967, 242, 6259, 6440, 3455, 2734, 4728] avg=4096.0000 max=7943 max_rank=0 min=242 min_rank=2 imbalance_ratio=1.939209 max_load_reduction=24825 max_load_reduction_pct=0.757599 moved_to_echo=17357 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[2816, 13568, 384, 3456, 12160, 256, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1664, 128, 1280, 0, 4224, 3200, 3840, 128, 18688, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[19072, 2432, 256, 128, 128, 128, 256, 10752, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[1664, 4224, 384, 6272, 128, 128, 128, 8448, 6272, 5760, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[6656, 2432, 1408, 4352, 14592, 0, 2816, 896, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 896, 16256, 1536, 640, 384, 6784, 6784, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[256, 1024, 128, 256, 7936, 19712, 1280, 128, 2688, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 2304, 128, 10496, 16512, 1664, 1664, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 981, 0, 0, 1220, 0, 0, 0, 2184, 14, 361, 1017, 101, 3, 0, 0, 6, 0, 0, 0, 2, 640, 0, 65, 1317, 4092, 0, 2, 0, 0, 530, 0, 0, 1233, 4078, 0, 0, 6, 261, 3391, 0, 0, 0, 3692, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 2454, 3175, 1713, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[54, 0, 26, 1416, 0, 0, 0, 476, 0, 0, 0, 0, 0, 12, 0, 0, 11, 0, 0, 3, 1, 0, 0, 0, 3311, 198, 4093, 3392, 0, 0, 0, 0, 0, 3754, 1748, 4073, 0, 38, 0, 4088, 0, 638, 49, 0, 78, 0, 0, 325, 0, 2131, 348, 0, 2390, 0, 0, 0, 0, 0, 0, 0, 112, 3, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 116, 0, 0, 7, 0, 3, 0, 0, 0, 0, 0, 266, 0, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4046, 105, 0, 0, 0, 0, 4078, 0, 0, 242, 0, 0, 0, 721, 0, 3655, 5, 10, 5, 4091, 0, 0, 1, 3854, 0, 2671, 0, 0, 0, 2, 0, 0, 0, 907, 3886, 4093, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 1605, 54, 0, 15, 0, 0, 3880, 86, 356, 0, 0, 142, 0, 0, 22, 293, 0, 0, 0, 0, 0, 0, 4067, 0, 4061, 0, 44, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 8, 0, 1481, 717, 0, 0, 0, 2, 3990, 1288, 3730, 0, 0, 0, 0, 3908, 1, 0, 0, 2975, 0, 0, 28] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[2, 0, 0, 29, 68, 0, 0, 1968, 7, 1, 0, 3081, 0, 0, 0, 1, 0, 0, 797, 0, 0, 0, 7, 11, 4060, 0, 3203, 0, 1215, 0, 2396, 0, 0, 0, 1, 0, 3147, 0, 4019, 2, 0, 0, 0, 643, 4065, 14, 0, 2, 0, 1977, 0, 1962, 0, 0, 0, 0, 3, 48, 0, 0, 0, 39, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 71, 2, 0, 0, 0, 0, 0, 6, 3, 0, 237, 0, 12, 0, 168, 0, 0, 0, 0, 0, 1272, 0, 1, 4043, 156, 2067, 3227, 1, 0, 416, 0, 202, 0, 0, 2941, 0, 0, 0, 0, 1, 0, 1488, 32, 95, 0, 0, 8, 178, 2579, 1308, 3451, 0, 0, 1, 1347, 3, 3841, 3611, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 41, 0, 0, 183, 5, 4095, 53, 0, 0, 0, 4096, 0, 0, 0, 172, 0, 0, 0, 0, 2083, 1, 0, 4068, 3476, 0, 0, 0, 0, 4068, 0, 23, 0, 0, 0, 339, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 1107, 0, 0, 0, 0, 0, 0, 853, 0, 3987, 4092, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 10, 3404, 0, 0, 0, 0, 0, 1551, 4, 208, 0, 264, 3932, 0, 822, 12, 0, 0, 0, 0, 3969, 93, 0, 0, 0, 0, 2704, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3, 23, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 1752, 17, 4096, 0, 0, 0, 0, 0, 0, 3058, 2751, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1605, 54, 0, 15, 0, 0, 0, 758, 263, 0, 0, 0, 0, 0, 3880, 86, 356, 0, 0, 142, 0, 0, 2499, 0, 0, 0, 0, 0, 0, 0, 22, 293, 0, 0, 0, 0, 0, 0, 3804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 804, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 1481, 717, 0, 0, 0, 0, 876, 0, 0, 0, 0, 0, 0, 2, 3114, 1288, 3730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3908, 1, 0, 0, 2975, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2695, 6963, 4119, 848, 15, 3082, 8134, 6912] avg=4096.0000 max=8134 max_rank=6 min=15 min_rank=4 imbalance_ratio=1.985840 max_load_reduction=24634 max_load_reduction_pct=0.751770 moved_to_echo=18143 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 29, 68, 0, 0, 1968, 0, 596, 261, 0, 0, 0, 0, 0, 7, 1, 0, 3081, 0, 0, 0, 1, 1969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 797, 0, 0, 0, 7, 11, 3799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 638, 0, 1215, 0, 2396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3147, 0, 4019, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 643, 4065, 14, 0, 2, 0, 434, 0, 0, 0, 0, 0, 0, 0, 1543, 0, 1962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 48, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2924, 5059, 4614, 4249, 7169, 5158, 3505, 90] avg=4096.0000 max=7169 max_rank=4 min=90 min_rank=7 imbalance_ratio=1.750244 max_load_reduction=25599 max_load_reduction_pct=0.781219 moved_to_echo=15922 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 71, 2, 0, 0, 0, 0, 0, 2721, 385, 261, 0, 0, 0, 0, 0, 6, 3, 0, 237, 0, 12, 0, 168, 1271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1272, 0, 1, 3782, 0, 0, 0, 0, 0, 0, 0, 0, 156, 411, 3227, 1, 0, 416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 202, 0, 0, 2941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1488, 32, 95, 0, 0, 8, 0, 566, 0, 0, 0, 0, 0, 0, 178, 2013, 1308, 3451, 0, 0, 1, 1347, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3841, 890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3440, 1697, 5055, 4211, 3143, 2190, 8298, 4734] avg=4096.0000 max=8298 max_rank=6 min=1697 min_rank=1 imbalance_ratio=2.025879 max_load_reduction=24470 max_load_reduction_pct=0.746765 moved_to_echo=18365 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 981, 0, 0, 1220, 0, 0, 0, 1849, 0, 85, 0, 0, 0, 0, 0, 2184, 14, 361, 1017, 101, 3, 0, 0, 0, 502, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 2, 640, 0, 65, 1232, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 2, 0, 0, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1233, 4078, 0, 0, 6, 261, 1793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3692, 193, 0, 0, 0, 1096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 605, 3175, 1713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4135, 4182, 1945, 4624, 7371, 4981, 0, 5530] avg=4096.0000 max=7371 max_rank=4 min=0 min_rank=6 imbalance_ratio=1.799561 max_load_reduction=25397 max_load_reduction_pct=0.775055 moved_to_echo=17882 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[54, 0, 26, 1416, 0, 0, 0, 476, 0, 762, 214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 2516, 602, 0, 0, 0, 0, 0, 0, 11, 0, 0, 3, 1, 0, 0, 0, 3097, 0, 0, 0, 0, 0, 0, 0, 0, 198, 815, 3392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3754, 1748, 4073, 0, 38, 0, 2166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 638, 49, 0, 78, 0, 0, 325, 1320, 467, 0, 0, 0, 0, 0, 0, 0, 1664, 348, 0, 2390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2948, 3130, 3112, 4405, 11779, 2877, 4402, 115] avg=4096.0000 max=11779 max_rank=4 min=115 min_rank=7 imbalance_ratio=2.875732 max_load_reduction=20989 max_load_reduction_pct=0.640533 moved_to_echo=19173 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 116, 0, 0, 7, 0, 3, 0, 2928, 0, 260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, 0, 3, 0, 0, 538, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3786, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 242, 0, 0, 0, 721, 0, 1937, 0, 0, 0, 0, 0, 0, 0, 0, 5, 10, 5, 4091, 0, 0, 1, 3854, 1180, 586, 0, 0, 0, 0, 0, 0, 0, 2085, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 907, 958, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3314, 807, 3787, 4183, 2900, 9732, 2087, 5958] avg=4096.0000 max=9732 max_rank=5 min=807 min_rank=1 imbalance_ratio=2.375977 max_load_reduction=23036 max_load_reduction_pct=0.703003 moved_to_echo=20677 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 10, 3404, 0, 0, 0, 0, 0, 2307, 0, 0, 0, 0, 0, 0, 0, 1551, 4, 208, 0, 264, 3932, 0, 822, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 3969, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 387, 0, 0, 0, 0, 0, 0, 0, 1365, 17, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 751, 2751, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5721, 6781, 4074, 2704, 29, 4479, 5478, 3502] avg=4096.0000 max=6781 max_rank=1 min=29 min_rank=4 imbalance_ratio=1.655518 max_load_reduction=25987 max_load_reduction_pct=0.793060 moved_to_echo=13488 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 41, 0, 0, 183, 5, 4095, 3005, 0, 262, 0, 0, 0, 0, 0, 53, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 0, 0, 0, 0, 2083, 1, 0, 3806, 0, 0, 0, 0, 0, 0, 0, 0, 3476, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 243, 0, 0, 0, 0, 0, 0, 0, 864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 853, 0, 982, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7591, 4149, 6062, 7544, 362, 269, 864, 5927] avg=4096.0000 max=7591 max_rank=0 min=269 min_rank=5 imbalance_ratio=1.853271 max_load_reduction=25177 max_load_reduction_pct=0.768341 moved_to_echo=7422 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 5248, 5888, 7040, 3584, 768, 4352, 6016, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[256, 12672, 3072, 13312, 2432, 128, 128, 1408, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 1280, 5120, 1536, 1408, 256, 128, 6656, 12928, 2560, 1664, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[7808, 128, 1024, 4352, 4736, 4224, 128, 1024, 8320, 1664, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4864, 4864, 4224, 14208, 4864, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 768, 1664, 14080, 5248, 128, 128, 4224, 3712, 3584, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 8064, 2688, 9344, 1280, 0, 11520, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[256, 384, 896, 128, 128, 8064, 128, 128, 23424, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=32896 slack_slots=2816 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 3742, 71, 318, 0, 47, 0, 3547, 0, 0, 0, 0, 0, 0, 0, 182, 0, 392, 3679, 516, 0, 0, 3674, 1700, 4012, 33, 0, 3, 3530, 125, 0, 0, 0, 0, 0, 0, 0, 3595, 0, 0, 0, 173, 2595, 504, 290, 1, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 10, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 1242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3706, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 2, 0, 1271, 269, 0, 0, 3829, 0, 0, 8, 322, 0, 0, 0, 3621, 4094, 1, 0, 2, 1670, 4042, 16, 1, 0, 918, 0, 0, 0, 0, 8, 1, 51, 62, 0, 2, 112, 1, 0, 3426, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1, 229, 0, 340, 3450, 3357, 47, 0, 0, 0, 4006, 0, 472, 0, 9, 3, 7, 0, 0, 1, 1648, 0, 2912, 0, 0, 0, 3303, 0, 0, 0, 249, 0, 0, 1482, 0, 0, 0, 0, 0, 2681, 1, 4090, 33, 0, 0, 0, 1311, 0, 0, 3, 190, 0, 0, 0, 0, 270, 0, 2396, 0, 165, 0, 0, 112, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3909, 352, 0, 0, 0, 0, 137, 0, 0, 0, 0, 23, 0, 359, 0, 192, 0, 19, 0, 3773, 0, 2264, 29, 21, 0, 1, 250, 0, 0, 0, 4086, 36, 0, 1134, 0, 0, 1301, 2311, 0, 1, 0, 3974, 0, 543, 1, 0, 0, 0, 0, 4049, 0, 0, 0, 0, 0, 101, 0, 3902, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 2263, 0, 8, 2, 0, 8, 0, 0, 0, 608, 0, 4055, 0, 381, 53, 0, 0, 1347, 0, 6, 0, 0, 0, 4049, 1, 0, 0, 0, 982, 0, 0, 5, 0, 0, 0, 0, 3985, 0, 2828, 165, 0, 4014, 0, 0, 0, 0, 30, 0, 4044, 52, 163, 3693, 0, 0, 0, 26, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 1, 3772, 144, 104, 0, 0, 2500, 0, 1, 0, 54, 1, 0, 0, 42, 283, 0, 0, 4068, 0, 0, 132, 0, 28, 4015, 424, 0, 0, 2999, 1, 0, 0, 390, 1, 0, 0, 0, 0, 0, 0, 0, 10, 3945, 3748, 0, 1, 2095, 0, 0, 3948, 0, 0, 2, 0, 0, 0, 0, 39, 0, 0, 20, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[246, 0, 0, 4096, 10, 0, 0, 0, 0, 60, 272, 0, 2343, 1, 1647, 0, 0, 4068, 0, 0, 3441, 0, 2918, 0, 440, 1631, 4027, 15, 0, 5, 20, 0, 0, 0, 0, 115, 0, 0, 0, 1948, 0, 0, 19, 0, 19, 0, 70, 0, 0, 0, 0, 3345, 0, 0, 0, 0, 0, 0, 2010, 0, 0, 2, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[272, 159, 14, 4089, 3720, 2, 0, 0, 0, 0, 2846, 0, 0, 2, 456, 5, 59, 3987, 0, 47, 3816, 0, 613, 0, 0, 0, 3740, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 4085, 4092, 1, 0, 757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3909, 352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 359, 0, 137, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3773, 0, 742, 29, 21, 0, 1, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 36, 0, 1134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1301, 2311, 0, 1, 0, 3974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 1, 0, 0, 0, 0, 4049, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 3902, 0, 1522, 20, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4261, 137, 538, 4816, 5256, 7587, 4628, 5545] avg=4096.0000 max=7587 max_rank=5 min=137 min_rank=1 imbalance_ratio=1.852295 max_load_reduction=25181 max_load_reduction_pct=0.768463 moved_to_echo=23016 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 229, 0, 284, 3450, 3357, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 0, 472, 0, 9, 3, 54, 383, 0, 0, 0, 0, 0, 0, 7, 0, 0, 1, 1168, 0, 2912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1082, 0, 0, 0, 249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1482, 0, 0, 0, 0, 0, 2298, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4090, 33, 0, 0, 0, 1311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 190, 0, 0, 0, 0, 270, 0, 302, 2, 0, 0, 0, 0, 0, 0, 2396, 0, 165, 0, 0, 112, 0, 2221, 178, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7368, 4927, 4088, 1331, 3780, 5435, 767, 5072] avg=4096.0000 max=7368 max_rank=0 min=767 min_rank=6 imbalance_ratio=1.798828 max_load_reduction=25400 max_load_reduction_pct=0.775146 moved_to_echo=15054 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3706, 0, 197, 584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 417, 269, 0, 0, 3829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 322, 0, 0, 0, 3621, 3510, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 1670, 2625, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 918, 0, 0, 0, 0, 8, 1, 51, 1417, 751, 8, 0, 0, 0, 0, 0, 62, 0, 2, 112, 1, 0, 3426, 0, 854, 443, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1037, 4489, 2895, 4517, 7461, 4315, 3154, 4900] avg=4096.0000 max=7461 max_rank=4 min=1037 min_rank=0 imbalance_ratio=1.821533 max_load_reduction=25307 max_load_reduction_pct=0.772308 moved_to_echo=19830 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3122, 71, 318, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3547, 0, 0, 0, 0, 0, 0, 596, 513, 0, 0, 0, 0, 0, 0, 0, 182, 0, 392, 2605, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3674, 1700, 1314, 33, 0, 3, 3530, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173, 1685, 504, 290, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 910, 676, 24, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 10, 0, 0, 2698, 398, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3558, 4656, 3695, 10379, 3082, 2653, 1621, 3124] avg=4096.0000 max=10379 max_rank=3 min=1621 min_rank=6 imbalance_ratio=2.533936 max_load_reduction=22389 max_load_reduction_pct=0.683258 moved_to_echo=10480 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[246, 0, 0, 3418, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 272, 0, 2343, 1, 1647, 0, 652, 278, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 2436, 0, 2918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 440, 1631, 1319, 15, 0, 5, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 0, 1670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 13, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3345, 0, 0, 0, 0, 6, 632, 26, 0, 0, 0, 0, 0, 0, 0, 2010, 0, 0, 2, 0, 0, 2708, 373, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3674, 5253, 9422, 3430, 1785, 102, 4009, 5093] avg=4096.0000 max=9422 max_rank=2 min=102 min_rank=5 imbalance_ratio=2.300293 max_load_reduction=23346 max_load_reduction_pct=0.712463 moved_to_echo=10989 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1889, 0, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 608, 0, 4055, 0, 360, 569, 0, 0, 0, 0, 0, 0, 381, 53, 0, 0, 954, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1326, 1, 0, 0, 0, 982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 3416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2828, 165, 0, 2607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 4044, 52, 163, 3693, 0, 1407, 247, 14, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 2723, 146, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1899, 5600, 1394, 2309, 3421, 5600, 9650, 2895] avg=4096.0000 max=9650 max_rank=6 min=1394 min_rank=2 imbalance_ratio=2.355957 max_load_reduction=23118 max_load_reduction_pct=0.705505 moved_to_echo=21566 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 3143, 144, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2500, 0, 1, 0, 54, 1, 0, 0, 603, 0, 0, 0, 0, 0, 0, 0, 42, 283, 0, 0, 2871, 0, 0, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 1309, 424, 0, 0, 2999, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 390, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 2559, 3748, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2095, 0, 0, 3948, 0, 0, 2, 0, 1386, 752, 26, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 20, 0, 2706, 445, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3392, 3159, 3328, 4761, 391, 6318, 8209, 3210] avg=4096.0000 max=8209 max_rank=6 min=391 min_rank=4 imbalance_ratio=2.004150 max_load_reduction=24559 max_load_reduction_pct=0.749481 moved_to_echo=18128 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[272, 159, 14, 3412, 3720, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2846, 0, 0, 2, 456, 5, 651, 587, 0, 0, 0, 0, 0, 0, 59, 3987, 0, 47, 2702, 0, 613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 4085, 3505, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 701, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2515, 413, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7579, 4547, 7408, 1225, 7592, 758, 730, 2929] avg=4096.0000 max=7592 max_rank=4 min=730 min_rank=6 imbalance_ratio=1.853516 max_load_reduction=25176 max_load_reduction_pct=0.768311 moved_to_echo=12009 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[7936, 3456, 8832, 896, 128, 128, 10752, 1408, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3584, 128, 256, 11392, 128, 256, 7808, 384, 5248, 4096, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 8320, 3328, 1920, 9600, 4352, 5760, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[512, 8576, 384, 512, 15872, 640, 6528, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[640, 512, 128, 16384, 7424, 7808, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 2432, 2176, 384, 128, 128, 7552, 0, 18048, 2432, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 1536, 768, 4224, 128, 0, 8960, 17536, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[2560, 3712, 7168, 256, 3584, 128, 9984, 128, 3200, 2944, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 1, 2, 4072, 0, 38, 0, 20, 0, 0, 51, 0, 0, 208, 0, 843, 0, 0, 0, 0, 94, 6, 0, 2315, 2, 0, 1527, 0, 0, 0, 3919, 0, 0, 0, 0, 3627, 0, 0, 4038, 0, 1, 0, 2315, 3857, 96, 0, 0, 3, 0, 3076, 13, 0, 0, 1122, 0, 0, 34, 1150, 0, 0, 0, 71, 267] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 279, 4068, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3657, 1235, 0, 0, 0, 15, 2, 1217, 0, 438, 2, 109, 0, 0, 0, 3156, 0, 0, 2139, 4096, 0, 9, 3757, 150, 0, 4095, 0, 0, 48, 0, 0, 0, 80, 0, 0, 116, 3, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[205, 0, 34, 12, 0, 0, 0, 0, 0, 0, 3582, 0, 0, 147, 3908, 0, 0, 4072, 0, 13, 0, 0, 0, 0, 707, 0, 1, 5, 521, 742, 2548, 3462, 73, 0, 0, 0, 0, 0, 0, 3859, 0, 0, 0, 0, 6, 0, 5, 1, 0, 588, 3977, 63, 4093, 0, 0, 112, 0, 32, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[1, 0, 1035, 2, 3, 0, 0, 0, 2, 0, 0, 1308, 0, 0, 3188, 0, 0, 0, 0, 507, 4093, 0, 259, 15, 0, 0, 0, 118, 0, 0, 93, 3972, 0, 368, 0, 6, 0, 9, 0, 289, 0, 0, 0, 0, 3514, 0, 4092, 0, 0, 17, 2392, 3569, 0, 0, 0, 3911, 3, 1, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[789, 415, 4094, 0, 1, 184, 0, 0, 0, 0, 119, 0, 0, 0, 0, 0, 1, 0, 19, 27, 4094, 11, 0, 0, 3176, 0, 0, 0, 0, 303, 0, 871, 0, 0, 0, 0, 12, 0, 0, 3866, 0, 0, 3507, 0, 0, 0, 0, 0, 3781, 0, 4066, 0, 0, 0, 0, 48, 0, 168, 3176, 2, 0, 0, 0, 38] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[241, 0, 27, 0, 44, 20, 998, 0, 0, 0, 0, 0, 3929, 0, 1, 0, 0, 0, 3, 0, 0, 1, 0, 0, 3768, 0, 5, 62, 0, 0, 0, 32, 1, 0, 0, 0, 0, 1, 0, 4055, 0, 0, 4096, 4092, 0, 1638, 746, 0, 3843, 262, 3998, 0, 0, 377, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 3250, 0, 0, 80, 30, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4073, 263, 4092, 125, 0, 0, 0, 1200, 0, 0, 0, 0, 8, 0, 2845, 0, 0, 0, 6, 1, 3856, 0, 1082, 12, 0, 2913, 5, 964, 0, 0, 22, 0, 0, 3846, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 1894, 2, 0, 6, 12, 0, 0, 1707, 0, 0, 6, 14, 31, 0, 0, 408, 0, 4077, 0, 5, 0, 4047, 3253, 0, 8, 0, 0, 0, 2688, 0, 0, 0, 2, 0, 0, 0, 1275, 0, 589, 1, 2818, 2747, 0, 4008, 0, 0, 0, 552, 0, 0, 0, 0, 2192, 0, 0, 0, 426, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 2, 4072, 0, 38, 0, 1898, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 51, 0, 0, 208, 0, 0, 64, 0, 0, 0, 0, 0, 0, 843, 0, 0, 0, 0, 94, 6, 0, 1597, 353, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1527, 0, 0, 0, 3341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3627, 0, 0, 4038, 578, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3857, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 455, 13, 0, 0, 1122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 1150, 0, 0, 0, 71, 267, 2621, 718, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6011, 343, 2893, 4870, 8243, 3954, 1593, 4861] avg=4096.0000 max=8243 max_rank=4 min=343 min_rank=1 imbalance_ratio=2.012451 max_load_reduction=24525 max_load_reduction_pct=0.748444 moved_to_echo=18651 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 279, 4068, 0, 3358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1784, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2523, 623, 0, 0, 0, 0, 0, 0, 0, 1235, 0, 0, 0, 15, 2, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 438, 2, 109, 0, 0, 0, 3156, 179, 340, 0, 0, 0, 0, 0, 0, 0, 0, 2139, 0, 0, 9, 1633, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 116, 3, 0, 1, 0, 0, 1134, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7705, 5995, 3146, 2290, 4224, 3931, 4143, 1334] avg=4096.0000 max=7705 max_rank=0 min=1334 min_rank=7 imbalance_ratio=1.881104 max_load_reduction=25063 max_load_reduction_pct=0.764862 moved_to_echo=13632 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1035, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1308, 0, 0, 3188, 0, 1943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 4093, 0, 259, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 0, 0, 93, 3386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 368, 0, 6, 0, 9, 0, 289, 586, 370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3514, 0, 1779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 354, 3569, 0, 0, 0, 3911, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 1, 0, 0, 0, 0, 0, 2038, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1041, 6441, 4874, 3597, 1628, 5293, 7851, 2043] avg=4096.0000 max=7851 max_rank=6 min=1041 min_rank=0 imbalance_ratio=1.916748 max_load_reduction=24917 max_load_reduction_pct=0.760406 moved_to_echo=16815 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[205, 0, 34, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3582, 0, 0, 147, 3908, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 13, 0, 0, 0, 0, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 521, 742, 2548, 2951, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 3859, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 588, 588, 63, 4093, 0, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 3389, 220, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[251, 7639, 4572, 6768, 4443, 10, 5444, 3641] avg=4096.0000 max=7639 max_rank=1 min=10 min_rank=5 imbalance_ratio=1.864990 max_load_reduction=25129 max_load_reduction_pct=0.766876 moved_to_echo=13538 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[789, 415, 4094, 0, 1, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 19, 27, 4094, 11, 0, 0, 2192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 303, 0, 743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 3866, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3781, 0, 601, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 3176, 2, 0, 0, 0, 38, 3465, 984, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5483, 119, 6344, 1046, 4006, 3507, 4430, 7833] avg=4096.0000 max=7833 max_rank=7 min=119 min_rank=1 imbalance_ratio=1.912354 max_load_reduction=24935 max_load_reduction_pct=0.760956 moved_to_echo=19776 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1894, 2, 0, 6, 12, 2312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1707, 0, 0, 6, 14, 31, 1904, 78, 0, 0, 0, 0, 0, 0, 0, 0, 408, 0, 4077, 0, 5, 0, 2793, 428, 0, 0, 0, 0, 0, 0, 0, 3253, 0, 8, 0, 0, 0, 2288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1275, 400, 366, 0, 0, 0, 0, 0, 0, 0, 589, 1, 0, 2747, 0, 1738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 2192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 426, 0, 0, 0, 0, 470, 1254, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4226, 3740, 7711, 5549, 2043, 5075, 2274, 2150] avg=4096.0000 max=7711 max_rank=2 min=2043 min_rank=4 imbalance_ratio=1.882568 max_load_reduction=25057 max_load_reduction_pct=0.764679 moved_to_echo=11542 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[241, 0, 27, 0, 44, 20, 998, 0, 3355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3929, 0, 1, 0, 354, 113, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 2599, 624, 0, 0, 0, 0, 0, 0, 0, 0, 5, 62, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 4055, 4, 67, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1638, 325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3843, 262, 591, 0, 0, 377, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3407, 1169, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4685, 4397, 3227, 95, 4128, 6059, 5601, 4576] avg=4096.0000 max=6059 max_rank=5 min=95 min_rank=3 imbalance_ratio=1.479248 max_load_reduction=26709 max_load_reduction_pct=0.815094 moved_to_echo=20364 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3250, 0, 0, 80, 30, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 263, 4092, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1200, 0, 0, 0, 0, 8, 0, 2845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3856, 0, 1082, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 429, 5, 964, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3846, 0, 0, 0, 0, 0, 2484, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3366, 4094, 1, 8553, 4053, 4939, 1432, 6330] avg=4096.0000 max=8553 max_rank=3 min=1 min_rank=2 imbalance_ratio=2.088135 max_load_reduction=24215 max_load_reduction_pct=0.738983 moved_to_echo=16754 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 8576, 384, 5888, 768, 1152, 2688, 13824, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 5504, 1408, 12160, 256, 7424, 128, 6016, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[7680, 4992, 3200, 3712, 5120, 384, 1664, 6400, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 256, 8192, 640, 128, 0, 128, 384, 17920, 5504, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1280, 512, 8448, 1920, 4224, 640, 5248, 128, 11008, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 640, 9856, 0, 10240, 5632, 5504, 1280, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1280, 896, 128, 128, 3712, 128, 0, 23424, 2432, 1152, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[896, 4096, 512, 640, 12288, 128, 384, 128, 12288, 2048, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 98, 192, 0, 3875, 0, 0, 17, 0, 0, 44, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4054, 0, 0, 0, 21, 20, 0, 0, 0, 0, 3865, 0, 0, 5, 672, 0, 1, 0, 0, 0, 0, 1, 63, 0, 0, 0, 4, 286, 3958, 40, 0, 0, 17, 1479, 0, 0, 0, 1972, 4092, 0, 3948, 4042, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[3511, 0, 0, 226, 0, 0, 0, 35, 88, 0, 0, 0, 0, 0, 105, 0, 60, 0, 0, 1234, 0, 3984, 27, 0, 0, 0, 0, 0, 787, 0, 2, 501, 0, 0, 0, 2870, 0, 0, 13, 0, 0, 7, 313, 0, 0, 3653, 0, 4096, 1, 0, 0, 0, 9, 3662, 0, 0, 4088, 31, 0, 3, 3462, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[10, 0, 0, 25, 202, 0, 35, 0, 0, 0, 265, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4014, 3405, 15, 32, 11, 1013, 0, 0, 36, 21, 0, 895, 0, 0, 0, 4022, 645, 0, 410, 0, 0, 41, 2540, 919, 0, 3941, 1, 0, 1324, 0, 0, 0, 73, 0, 4076, 0, 0, 0, 869, 0, 0, 3927, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[3481, 0, 0, 181, 0, 0, 3, 0, 386, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3068, 2548, 0, 0, 0, 3794, 1, 117, 0, 0, 24, 0, 0, 0, 23, 717, 0, 3750, 460, 0, 44, 4, 0, 3002, 953, 1742, 0, 0, 0, 3286, 735, 33, 31, 256] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[7, 0, 0, 0, 0, 0, 0, 0, 4032, 4091, 0, 7, 0, 1, 0, 0, 0, 0, 0, 2145, 2179, 4092, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 128, 0, 368, 3575, 1, 935, 0, 0, 0, 0, 1, 747, 0, 1361, 5, 0, 0, 0, 0, 0, 4095, 1476, 0, 0, 0, 2, 0, 0, 3430, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[339, 0, 0, 0, 0, 3862, 1, 0, 1569, 0, 0, 20, 3822, 0, 0, 12, 102, 9, 0, 2674, 0, 3967, 113, 0, 0, 0, 0, 0, 80, 0, 0, 1427, 0, 238, 0, 3803, 0, 0, 75, 0, 0, 0, 0, 0, 4, 784, 0, 0, 0, 1784, 0, 18, 0, 0, 0, 1, 2637, 2, 0, 4030, 13, 0, 1382, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[1253, 0, 1, 2677, 4, 3495, 9, 97, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3836, 2706, 0, 0, 0, 0, 0, 0, 0, 191, 186, 24, 0, 0, 0, 3059, 0, 0, 0, 14, 0, 0, 4, 10, 4071, 2, 4089, 4007, 0, 83, 0, 0, 0, 0, 0, 0, 0, 1237, 1468, 10, 0, 103, 69] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[4088, 2, 0, 2498, 0, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 2, 3741, 3876, 4059, 75, 0, 4091, 0, 0, 1705, 0, 0, 0, 0, 12, 0, 0, 0, 0, 2, 0, 0, 28, 0, 0, 192, 1, 12, 4033, 1, 0, 0, 0, 0, 0, 0, 199, 0, 0, 0, 5, 0, 3990, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 98, 192, 0, 3875, 0, 0, 0, 335, 0, 0, 0, 0, 0, 0, 17, 0, 0, 44, 0, 0, 0, 0, 2337, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 20, 0, 0, 0, 0, 3865, 2199, 475, 0, 0, 0, 0, 0, 0, 0, 0, 5, 672, 0, 1, 0, 0, 1749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 63, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 286, 3958, 40, 0, 0, 17, 1479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1972, 3757, 0, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4500, 2398, 1244, 6580, 2427, 68, 5780, 9771] avg=4096.0000 max=9771 max_rank=7 min=68 min_rank=5 imbalance_ratio=2.385498 max_load_reduction=22997 max_load_reduction_pct=0.701813 moved_to_echo=18046 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 0, 25, 202, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 265, 1, 0, 0, 0, 0, 1963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4014, 1043, 15, 32, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1013, 0, 0, 36, 21, 0, 895, 2186, 399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 645, 0, 410, 0, 1741, 0, 0, 0, 0, 0, 0, 0, 0, 41, 2540, 919, 0, 3941, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1324, 0, 0, 0, 73, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[272, 2229, 5104, 4561, 6818, 7442, 5473, 869] avg=4096.0000 max=7442 max_rank=5 min=272 min_rank=0 imbalance_ratio=1.816895 max_load_reduction=25326 max_load_reduction_pct=0.772888 moved_to_echo=20602 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3481, 0, 0, 181, 0, 0, 3, 0, 346, 269, 7, 0, 0, 0, 0, 0, 386, 0, 0, 0, 0, 0, 33, 0, 2362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3068, 2548, 18, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3794, 1, 117, 0, 0, 15, 282, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 23, 717, 0, 3122, 0, 0, 0, 0, 0, 0, 0, 0, 460, 0, 44, 4, 0, 2995, 953, 1742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3017, 735, 0, 31, 256, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4287, 2781, 1254, 6114, 4209, 3886, 6198, 4039] avg=4096.0000 max=6198 max_rank=6 min=1254 min_rank=2 imbalance_ratio=1.513184 max_load_reduction=26570 max_load_reduction_pct=0.810852 moved_to_echo=18332 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3511, 0, 0, 226, 0, 0, 0, 35, 378, 0, 8, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 105, 0, 2297, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 1234, 0, 1220, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 787, 0, 2, 501, 0, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2870, 0, 0, 13, 0, 0, 308, 0, 0, 0, 0, 0, 0, 0, 7, 313, 0, 0, 3653, 0, 3410, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 9, 3654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 31, 0, 3, 3462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4158, 2490, 2541, 1757, 3191, 7383, 3664, 7584] avg=4096.0000 max=7584 max_rank=7 min=1757 min_rank=3 imbalance_ratio=1.851562 max_load_reduction=25184 max_load_reduction_pct=0.768555 moved_to_echo=21822 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[339, 0, 0, 0, 0, 3862, 1, 0, 0, 330, 0, 0, 0, 0, 0, 0, 1569, 0, 0, 20, 3822, 0, 0, 12, 2287, 0, 0, 0, 0, 0, 0, 0, 102, 9, 0, 2674, 0, 1215, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 1427, 0, 465, 0, 0, 0, 0, 0, 0, 0, 238, 0, 3803, 0, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1784, 0, 18, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2637, 2, 0, 3700, 13, 0, 1382, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4532, 7710, 4113, 1972, 4116, 788, 1803, 7734] avg=4096.0000 max=7734 max_rank=7 min=788 min_rank=5 imbalance_ratio=1.888184 max_load_reduction=25034 max_load_reduction_pct=0.763977 moved_to_echo=14441 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4088, 2, 0, 2498, 0, 0, 0, 0, 373, 3, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 51, 2235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3741, 1187, 4059, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 1705, 0, 0, 0, 2222, 454, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 2, 0, 1768, 305, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 192, 1, 12, 3355, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6964, 2391, 9064, 8472, 2087, 3588, 200, 2] avg=4096.0000 max=9064 max_rank=2 min=2 min_rank=7 imbalance_ratio=2.212891 max_load_reduction=23704 max_load_reduction_pct=0.723389 moved_to_echo=5877 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 0, 0, 0, 0, 0, 0, 3, 0, 11, 0, 0, 0, 0, 0, 4032, 4091, 0, 7, 0, 1, 0, 0, 2363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2145, 2179, 1244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 1910, 485, 0, 0, 0, 0, 0, 0, 0, 128, 0, 368, 3575, 1, 935, 0, 1520, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 747, 0, 1361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 1476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21, 10494, 5568, 2485, 6529, 2109, 5560, 2] avg=4096.0000 max=10494 max_rank=1 min=2 min_rank=7 imbalance_ratio=2.562012 max_load_reduction=22274 max_load_reduction_pct=0.679749 moved_to_echo=14200 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1253, 0, 1, 2677, 4, 3495, 9, 97, 378, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 2212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1174, 2706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 191, 186, 0, 450, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 3059, 0, 0, 0, 0, 308, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4, 10, 4071, 2, 3403, 0, 0, 0, 0, 0, 0, 0, 0, 4007, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1237, 1348, 10, 0, 103, 69, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8034, 2275, 3880, 827, 3391, 7504, 4090, 2767] avg=4096.0000 max=8034 max_rank=0 min=827 min_rank=3 imbalance_ratio=1.961426 max_load_reduction=24734 max_load_reduction_pct=0.754822 moved_to_echo=17752 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[6784, 128, 4096, 11904, 4224, 0, 5632, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[6272, 4096, 384, 256, 3840, 128, 256, 128, 18176, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 128, 0, 6144, 9984, 9600, 7040, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 2944, 1024, 1024, 13184, 1408, 13312, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 384, 128, 15616, 7296, 128, 1536, 0, 6912, 1280, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 5248, 128, 0, 2816, 128, 3328, 9472, 8576, 3712, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[12800, 128, 128, 5888, 256, 11264, 128, 256, 1536, 1152, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[6144, 5760, 256, 128, 128, 10752, 8064, 2048, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[1, 0, 4011, 3791, 36, 131, 3, 106, 0, 0, 0, 0, 3480, 52, 0, 0, 0, 0, 0, 0, 0, 3923, 0, 25, 1, 12, 15, 0, 3706, 803, 0, 0, 0, 5, 0, 596, 0, 2995, 0, 309, 0, 0, 0, 0, 11, 0, 148, 3952, 0, 0, 56, 279, 0, 3, 2, 4075, 224, 0, 0, 0, 0, 0, 0, 17] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4049, 2249, 3897, 4094, 0, 0, 0, 0, 0, 1, 0, 2214, 0, 1654, 0, 0, 0, 2, 4, 1436, 0, 0, 384, 0, 0, 0, 4051, 0, 0, 0, 86, 0, 0, 3, 0, 0, 0, 4095, 0, 308, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 73, 0, 71, 0, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 3892, 558, 4096, 114, 0, 0, 0, 0, 0, 641, 0, 0, 0, 189, 0, 0, 30, 14, 19, 183, 3, 0, 0, 147, 0, 4084, 0, 4023, 3706, 0, 0, 0, 0, 0, 3798, 0, 3030, 0, 649, 0, 0, 3, 184, 0, 6, 56, 0, 0, 0, 0, 2, 0, 49, 22, 0, 0, 0, 0, 0, 0, 3270] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[4095, 123, 3406, 2, 1665, 2, 0, 0, 67, 0, 14, 2, 0, 0, 0, 0, 0, 0, 31, 43, 1141, 0, 0, 0, 1265, 0, 0, 6, 0, 0, 3532, 0, 1782, 0, 0, 820, 0, 3852, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 135, 0, 0, 0, 0, 3, 0, 4074, 3765, 0, 0, 0, 2835, 0, 0, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[3956, 0, 0, 11, 27, 2740, 0, 0, 58, 0, 145, 4071, 44, 0, 0, 0, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3861, 3648, 54, 0, 109, 1, 0, 0, 0, 0, 43, 0, 0, 3568, 2063, 0, 0, 4007, 0, 0, 0, 0, 0, 2, 0, 0, 1632, 187, 0, 665, 0, 105, 0, 11, 686, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 1, 3902, 2373, 0, 0, 0, 0, 0, 0, 0, 1537, 0, 0, 0, 14, 0, 1, 3820, 0, 3554, 351, 0, 2, 0, 301, 5, 4089, 0, 4, 0, 641, 2441, 0, 0, 0, 3317, 1, 4083, 1, 0, 0, 0, 0, 0, 76, 0, 0, 347, 0, 0, 0, 0, 0, 0, 92, 0, 10, 0, 0, 0, 1805, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[2039, 43, 0, 4072, 0, 0, 0, 11, 0, 457, 0, 0, 0, 0, 0, 4022, 0, 0, 0, 2993, 0, 3908, 0, 3731, 4056, 142, 215, 0, 3133, 1, 0, 0, 0, 0, 1184, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 166, 0, 0, 0, 834, 121, 0, 1, 0, 933, 690, 0, 0, 14, 0, 1, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[4, 0, 0, 0, 0, 3792, 0, 0, 0, 16, 0, 0, 1, 0, 0, 3466, 0, 0, 0, 0, 0, 1095, 0, 1, 1034, 0, 2, 0, 512, 3519, 369, 28, 1, 0, 0, 0, 0, 0, 0, 897, 0, 2, 75, 52, 793, 0, 2, 0, 2, 0, 0, 183, 0, 390, 41, 4089, 3396, 0, 76, 2012, 4005, 0, 2913, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 558, 621, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 641, 0, 0, 0, 2425, 440, 0, 0, 0, 0, 0, 0, 189, 0, 0, 30, 14, 19, 183, 3, 1050, 260, 0, 0, 0, 0, 0, 0, 0, 0, 147, 0, 329, 0, 4023, 3706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3798, 0, 2125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 649, 0, 0, 3, 184, 0, 6, 3755, 460, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 2, 0, 49, 2992, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 3270, 645, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1293, 3506, 1748, 8205, 5923, 5057, 3099, 3937] avg=4096.0000 max=8205 max_rank=3 min=1293 min_rank=0 imbalance_ratio=2.003174 max_load_reduction=24563 max_load_reduction_pct=0.749603 moved_to_echo=18016 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4049, 0, 3897, 621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2214, 0, 1654, 2424, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 1436, 0, 0, 1049, 352, 0, 0, 0, 0, 0, 0, 384, 0, 0, 0, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 3, 0, 0, 0, 2871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 308, 0, 0, 0, 0, 0, 1, 3725, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 71, 1728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 872, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8567, 6547, 2843, 710, 2960, 4301, 1872, 4968] avg=4096.0000 max=8567 max_rank=0 min=710 min_rank=3 imbalance_ratio=2.091553 max_load_reduction=24201 max_load_reduction_pct=0.738556 moved_to_echo=14101 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 4011, 0, 36, 21, 3, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3480, 52, 0, 0, 77, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3923, 0, 25, 33, 26, 0, 0, 0, 0, 0, 0, 1, 12, 15, 0, 298, 803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 596, 0, 2995, 0, 218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 148, 3952, 3408, 447, 0, 0, 0, 0, 0, 0, 0, 0, 56, 279, 0, 3, 2, 4075, 2915, 0, 0, 0, 0, 0, 0, 0, 224, 0, 0, 0, 0, 0, 0, 17, 65, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4178, 4038, 4007, 1129, 3814, 7966, 7330, 306] avg=4096.0000 max=7966 max_rank=5 min=306 min_rank=7 imbalance_ratio=1.944824 max_load_reduction=24802 max_load_reduction_pct=0.756897 moved_to_echo=19416 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4095, 123, 3406, 0, 1665, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 14, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 43, 1141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1265, 0, 0, 6, 0, 0, 3532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1782, 0, 0, 820, 0, 3852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 0, 0, 0, 0, 3, 0, 4074, 2, 0, 0, 0, 0, 0, 0, 0, 3765, 0, 0, 0, 2835, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9290, 84, 1215, 4803, 6454, 100, 4214, 6608] avg=4096.0000 max=9290 max_rank=0 min=84 min_rank=1 imbalance_ratio=2.268066 max_load_reduction=23478 max_load_reduction_pct=0.716492 moved_to_echo=17376 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3956, 0, 0, 0, 27, 411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 145, 4071, 44, 0, 0, 0, 1624, 2, 0, 0, 0, 0, 0, 0, 1074, 0, 0, 0, 0, 0, 0, 0, 705, 306, 0, 0, 0, 0, 0, 0, 0, 0, 3861, 3648, 5, 0, 109, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 2502, 0, 0, 0, 0, 0, 0, 0, 0, 2063, 0, 0, 4007, 0, 0, 0, 0, 49, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1632, 187, 0, 665, 8, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 11, 686, 0, 0, 0, 760, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4394, 5944, 2085, 7624, 2545, 6120, 2494, 1562] avg=4096.0000 max=7624 max_rank=3 min=1562 min_rank=7 imbalance_ratio=1.861328 max_load_reduction=25144 max_load_reduction_pct=0.767334 moved_to_echo=12721 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 0, 0, 0, 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 1, 0, 0, 3466, 2245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1095, 0, 1, 972, 77, 0, 0, 0, 0, 0, 0, 1034, 0, 2, 0, 42, 3519, 369, 28, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 75, 52, 793, 0, 2, 0, 470, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 183, 0, 390, 41, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 3396, 0, 76, 2012, 4005, 0, 2913, 0, 191, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[579, 5728, 2145, 4994, 630, 1394, 4705, 12593] avg=4096.0000 max=12593 max_rank=7 min=579 min_rank=0 imbalance_ratio=3.074463 max_load_reduction=20175 max_load_reduction_pct=0.615692 moved_to_echo=19322 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2039, 43, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 457, 0, 0, 0, 0, 0, 4022, 0, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2993, 0, 3908, 0, 3731, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 142, 215, 0, 252, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 166, 0, 0, 2881, 481, 0, 0, 0, 0, 0, 0, 0, 834, 121, 0, 1, 0, 933, 690, 3129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2093, 4941, 10632, 4666, 1184, 3529, 5708, 15] avg=4096.0000 max=10632 max_rank=2 min=15 min_rank=7 imbalance_ratio=2.595703 max_load_reduction=22136 max_load_reduction_pct=0.675537 moved_to_echo=10436 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 2373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1537, 0, 0, 0, 0, 443, 0, 0, 0, 0, 0, 0, 14, 0, 1, 3820, 0, 3554, 351, 0, 0, 353, 0, 0, 0, 0, 0, 0, 2, 0, 301, 5, 325, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 641, 2441, 0, 0, 0, 3317, 1, 2858, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 76, 0, 3764, 460, 0, 0, 0, 0, 0, 0, 0, 347, 0, 0, 0, 0, 0, 0, 2999, 0, 0, 0, 0, 0, 0, 0, 92, 0, 10, 0, 0, 0, 1805, 0, 872, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2374, 1980, 8093, 637, 9258, 4301, 3346, 2779] avg=4096.0000 max=9258 max_rank=4 min=637 min_rank=3 imbalance_ratio=2.260254 max_load_reduction=23510 max_load_reduction_pct=0.717468 moved_to_echo=19684 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[7552, 128, 128, 2048, 7552, 0, 8832, 3328, 3456, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1280, 0, 128, 6912, 1280, 13952, 640, 3840, 3840, 1408, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[10112, 256, 11520, 0, 8576, 2304, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[6784, 256, 4608, 3712, 1664, 4352, 8064, 3840, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[2560, 2560, 1280, 1536, 128, 14080, 128, 11264, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[2176, 1024, 128, 4096, 896, 512, 256, 3968, 18176, 2176, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 512, 256, 4096, 5760, 2304, 0, 9216, 8832, 2048, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[256, 1280, 256, 512, 1664, 768, 1024, 13824, 13824, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[8, 3876, 0, 0, 14, 0, 333, 0, 0, 4095, 0, 0, 0, 0, 6, 0, 0, 2039, 0, 0, 0, 630, 0, 1682, 0, 116, 3823, 0, 0, 0, 0, 0, 0, 0, 34, 2, 4088, 0, 13, 0, 109, 0, 3, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 1001, 3973, 36, 0, 0, 6, 0, 217, 0, 132, 0, 2454] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 3680, 0, 0, 47, 0, 0, 0, 5, 20, 0, 0, 4012, 0, 0, 0, 57, 0, 511, 0, 0, 2, 0, 322, 0, 0, 3759, 0, 10, 157, 2806, 0, 0, 0, 0, 0, 770, 0, 0, 0, 0, 0, 0, 0, 0, 3820, 3170, 0, 561, 0, 0, 0, 0, 4090, 99, 1, 3185, 68, 0, 0, 0, 1616, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 4036, 3197, 0, 0, 14, 0, 7, 226, 0, 0, 3111, 4096, 0, 4, 2687, 0, 253, 392, 1107, 0, 21, 0, 52, 4064, 0, 9, 0, 1236, 0, 0, 14, 0, 0, 0, 3794, 2454, 0, 0, 0, 0, 0, 0, 1987, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 18, 1, 21, 1049, 0, 1, 1609, 4061, 0, 0, 0, 3416, 3565, 0, 0, 0, 0, 0, 0, 0, 76, 3, 4011, 0, 0, 545, 62, 0, 0, 1, 253, 1, 0, 0, 2805, 14, 5, 150, 358, 0, 7, 0, 0, 0, 0, 3572, 0, 0, 681, 0, 0, 2413, 0, 0, 0, 0, 3, 0, 0, 19, 4048, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 828, 0, 0, 0, 0, 796, 52, 0, 0, 0, 0, 324, 2146, 0, 0, 0, 0, 0, 0, 553, 16, 3708, 3903, 376, 1744, 0, 87, 2, 1681, 138, 0, 230, 3, 0, 0, 214, 0, 4, 0, 0, 0, 0, 0, 0, 3901, 3688, 0, 0, 0, 0, 0, 0, 0, 171, 0, 0, 4096, 11, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[220, 4093, 4084, 535, 22, 0, 51, 0, 19, 0, 0, 0, 4049, 2198, 0, 0, 0, 0, 0, 0, 0, 0, 258, 260, 0, 1117, 0, 0, 0, 586, 10, 0, 4, 0, 339, 0, 0, 7, 0, 0, 808, 0, 0, 4081, 4, 3638, 2256, 0, 1, 0, 0, 0, 0, 0, 3166, 0, 0, 0, 659, 0, 0, 0, 0, 303] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 1556, 12, 0, 0, 901, 0, 3032, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3406, 0, 0, 2, 0, 0, 0, 0, 0, 0, 186, 0, 0, 0, 0, 0, 281, 0, 0, 0, 817, 0, 0, 2028, 657, 0, 3361, 4096, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 4, 0, 0, 0, 1815, 2458, 0, 11] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[2, 0, 3598, 0, 37, 0, 0, 0, 0, 4044, 0, 7, 4048, 4033, 0, 0, 160, 0, 1, 320, 0, 0, 0, 3969, 0, 93, 4096, 0, 0, 0, 0, 48, 0, 0, 0, 82, 0, 0, 0, 0, 0, 126, 3, 0, 500, 1550, 2628, 0, 0, 1120, 0, 0, 2256, 0, 0, 0, 0, 0, 8, 37, 0, 2, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[8, 3876, 0, 0, 14, 0, 333, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2039, 0, 0, 0, 630, 0, 1629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 3823, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 2, 4088, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 0, 3, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 3973, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 217, 0, 132, 0, 2454, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4284, 4101, 4298, 3939, 4137, 4190, 5010, 2809] avg=4096.0000 max=5010 max_rank=6 min=2809 min_rank=7 imbalance_ratio=1.223145 max_load_reduction=27758 max_load_reduction_pct=0.847107 moved_to_echo=16146 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 3680, 0, 0, 47, 0, 0, 0, 345, 10, 0, 0, 0, 0, 0, 0, 5, 20, 0, 0, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 511, 0, 0, 2, 0, 312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3759, 0, 10, 157, 2806, 0, 1337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 770, 0, 0, 0, 2344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3820, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 0, 0, 0, 0, 4090, 99, 1, 1445, 0, 0, 0, 0, 0, 0, 0, 3185, 68, 0, 0, 0, 1616, 0, 0, 602, 626, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 284, 882, 8069, 3114, 4044, 6196, 6097] avg=4096.0000 max=8069 max_rank=3 min=284 min_rank=1 imbalance_ratio=1.969971 max_load_reduction=24699 max_load_reduction_pct=0.753754 moved_to_echo=19451 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 2, 0, 1, 0, 347, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 261, 3197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 7, 226, 0, 0, 3111, 3967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2687, 0, 253, 392, 1107, 0, 1345, 0, 0, 0, 0, 0, 0, 0, 21, 0, 52, 4064, 0, 9, 0, 1236, 2806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 267, 2454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1987, 0, 1453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 721, 630, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[483, 3458, 7325, 5788, 8188, 2735, 3440, 1351] avg=4096.0000 max=8188 max_rank=4 min=483 min_rank=0 imbalance_ratio=1.999023 max_load_reduction=24580 max_load_reduction_pct=0.750122 moved_to_echo=15714 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 18, 1, 21, 1049, 0, 1, 1609, 293, 126, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 0, 222, 3565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 3, 3885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 62, 0, 0, 1, 253, 1138, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2805, 14, 5, 150, 358, 2642, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 681, 0, 0, 2413, 0, 0, 0, 1230, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 19, 4048, 0, 0, 679, 533, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3118, 7848, 3964, 1999, 5975, 258, 4324, 5282] avg=4096.0000 max=7848 max_rank=1 min=258 min_rank=5 imbalance_ratio=1.916016 max_load_reduction=24920 max_load_reduction_pct=0.760498 moved_to_echo=15839 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 828, 0, 0, 0, 0, 796, 52, 27, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 2146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 16, 3708, 3780, 0, 0, 0, 0, 0, 0, 0, 0, 376, 1744, 0, 87, 2, 1681, 138, 0, 108, 0, 0, 0, 0, 0, 0, 0, 230, 3, 0, 0, 214, 0, 4, 0, 2728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3901, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 171, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 4096, 701, 50, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1826, 2169, 8057, 4136, 3179, 4160, 287, 8954] avg=4096.0000 max=8954 max_rank=7 min=287 min_rank=6 imbalance_ratio=2.186035 max_load_reduction=23814 max_load_reduction_pct=0.726746 moved_to_echo=16580 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1556, 12, 0, 0, 901, 0, 3032, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3406, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 281, 0, 0, 0, 817, 3029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2028, 657, 0, 3361, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1815, 2458, 0, 11, 778, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5501, 4095, 3408, 186, 4127, 6335, 4050, 5066] avg=4096.0000 max=6335 max_rank=5 min=186 min_rank=3 imbalance_ratio=1.546631 max_load_reduction=26433 max_load_reduction_pct=0.806671 moved_to_echo=19578 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 3598, 0, 37, 0, 0, 0, 348, 125, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 7, 262, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 1, 320, 0, 0, 0, 3844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 4096, 0, 0, 0, 0, 48, 1349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 1944, 0, 0, 0, 0, 0, 0, 0, 0, 126, 3, 0, 500, 1550, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1120, 0, 0, 2256, 0, 0, 0, 1457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 37, 0, 2, 0, 0, 499, 632, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 8346, 4325, 5586, 2026, 2364, 4833, 1178] avg=4096.0000 max=8346 max_rank=1 min=1178 min_rank=7 imbalance_ratio=2.037598 max_load_reduction=24422 max_load_reduction_pct=0.745300 moved_to_echo=10401 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[220, 4093, 4084, 535, 22, 0, 51, 0, 350, 9, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 250, 2198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 258, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1117, 0, 0, 0, 586, 10, 0, 1352, 0, 0, 0, 0, 0, 0, 0, 4, 0, 339, 0, 0, 7, 0, 0, 1672, 0, 0, 0, 0, 0, 0, 0, 808, 0, 0, 4081, 4, 3638, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3166, 0, 1461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 659, 0, 0, 0, 0, 303, 433, 636, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9364, 2467, 509, 3065, 2022, 8682, 4628, 2031] avg=4096.0000 max=9364 max_rank=0 min=509 min_rank=2 imbalance_ratio=2.286133 max_load_reduction=23404 max_load_reduction_pct=0.714233 moved_to_echo=17363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3200, 4224, 768, 256, 1920, 8320, 0, 6912, 4480, 3200, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[640, 1920, 0, 0, 5760, 8064, 9600, 128, 7168, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[256, 14080, 7808, 640, 1280, 1024, 1280, 4736, 1792, 640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 2048, 640, 3968, 640, 768, 7168, 17792, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1024, 256, 2048, 4864, 512, 20352, 1664, 2560, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[384, 3200, 14976, 256, 384, 3072, 4096, 384, 6656, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[256, 128, 512, 7296, 5120, 128, 256, 2432, 17280, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 0, 128, 1280, 15232, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 3827, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2691, 0, 0, 7, 0, 64, 707, 0, 0, 0, 0, 0, 1457, 0, 0, 0, 0, 13, 3311, 116, 19, 85, 0, 45, 0, 0, 410, 0, 0, 0, 3953, 2, 0, 0, 0, 120, 0, 0, 4096, 0, 0, 0, 0, 0, 1692, 1017, 0, 0, 4060, 3400, 1647, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 156, 4, 0, 61, 29, 2755, 0, 342, 0, 0, 184, 0, 0, 4, 0, 0, 0, 49, 1, 0, 502, 0, 3838, 0, 0, 0, 16, 12, 0, 0, 3802, 0, 0, 0, 1913, 4091, 4089, 0, 17, 0, 5, 468, 2, 0, 2227, 5, 2, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 4071, 19, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 17, 0, 0, 3342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3578, 0, 0, 1, 0, 0, 3681, 536, 0, 0, 0, 0, 0, 0, 4031, 0, 0, 191, 0, 13, 4048, 0, 25, 0, 3766, 0, 3065, 0, 0, 184, 924, 0, 0, 537, 896, 0, 0, 0, 3903, 0, 0, 30, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[22, 0, 0, 0, 4021, 0, 0, 0, 0, 0, 4094, 5, 0, 0, 376, 0, 9, 131, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 0, 0, 376, 13, 4090, 0, 118, 3920, 0, 0, 4052, 0, 4096, 17, 0, 394, 0, 0, 0, 3, 82, 0, 0, 0, 0, 0, 0, 0, 0, 54, 2779, 32] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 154, 0, 0, 451, 3639, 0, 7, 0, 0, 0, 0, 0, 0, 1, 3748, 0, 269, 1822, 0, 0, 4011, 38, 844, 0, 0, 0, 4, 0, 239, 3371, 4096, 0, 0, 0, 2311, 2301, 0, 0, 0, 0, 13, 991, 0, 0, 0, 4, 4044, 1, 0, 12, 0, 0, 0, 0, 0, 397, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 2947, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3958, 4080, 0, 0, 0, 1, 0, 1521, 0, 0, 0, 0, 63, 2703, 2773, 0, 0, 46, 21, 0, 2319, 0, 30, 0, 0, 0, 4024, 1, 0, 0, 0, 0, 4096, 0, 0, 29, 0, 0, 7, 0, 0, 0, 0, 0, 0, 24, 4096, 16] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 13, 0, 0, 121, 69, 4096, 0, 0, 0, 0, 4, 0, 0, 0, 42, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4022, 0, 4088, 0, 0, 3971, 0, 3286, 3836, 0, 1, 0, 0, 0, 3823, 0, 0, 1054, 0, 0, 0, 0, 245, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 1219, 0, 0, 0, 2077, 3864, 0, 0, 13, 0, 421, 0, 0, 0, 4020, 0, 0, 0, 754, 0, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 316, 0, 0, 4003, 0, 4096, 0, 0, 0, 0, 57, 0, 0, 0, 5, 252, 0, 0, 39, 0, 0, 0, 3240, 0, 0, 0, 0, 4047, 1174, 0, 3069, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 154, 0, 0, 451, 3604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 1, 7, 1403, 761, 0, 0, 0, 0, 0, 1594, 0, 269, 1822, 0, 0, 4011, 38, 0, 0, 0, 0, 0, 0, 0, 0, 844, 0, 0, 0, 4, 0, 239, 3371, 1393, 0, 26, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 908, 2301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 991, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 1, 0, 12, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4209, 2179, 7734, 5877, 7305, 1001, 4066, 397] avg=4096.0000 max=7734 max_rank=2 min=397 min_rank=7 imbalance_ratio=1.888184 max_load_reduction=25034 max_load_reduction_pct=0.763977 moved_to_echo=12769 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3827, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2691, 0, 0, 7, 2250, 0, 0, 0, 0, 0, 0, 0, 0, 64, 707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1457, 0, 0, 0, 0, 13, 3311, 116, 0, 24, 0, 0, 0, 0, 0, 0, 19, 85, 0, 45, 0, 0, 410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1703, 2, 0, 0, 0, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1692, 1017, 0, 0, 4060, 3400, 1623, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3856, 4948, 771, 4921, 559, 1825, 4096, 11792] avg=4096.0000 max=11792 max_rank=7 min=559 min_rank=4 imbalance_ratio=2.878906 max_load_reduction=20976 max_load_reduction_pct=0.640137 moved_to_echo=18272 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 156, 4, 0, 61, 29, 2728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 342, 0, 0, 184, 0, 0, 4, 0, 266, 2486, 0, 0, 0, 0, 0, 0, 0, 0, 49, 1, 0, 502, 0, 3838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 12, 0, 0, 3802, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 1913, 1605, 4089, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 202, 2, 0, 2227, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 4071, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2978, 3282, 4390, 3850, 7624, 2443, 4089, 4112] avg=4096.0000 max=7624 max_rank=4 min=2443 min_rank=5 imbalance_ratio=1.861328 max_load_reduction=25144 max_load_reduction_pct=0.767334 moved_to_echo=18268 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[22, 0, 0, 0, 4021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 5, 0, 0, 376, 0, 2307, 2483, 1, 0, 0, 0, 0, 0, 5, 131, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 3, 41, 0, 0, 0, 0, 0, 0, 0, 0, 376, 13, 1607, 0, 118, 3920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1745, 0, 4096, 17, 0, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 2738, 32, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4043, 9266, 195, 4069, 6034, 6252, 85, 2824] avg=4096.0000 max=9266 max_rank=1 min=85 min_rank=6 imbalance_ratio=2.262207 max_load_reduction=23502 max_load_reduction_pct=0.717224 moved_to_echo=15195 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2947, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2291, 1407, 804, 0, 0, 0, 0, 0, 1683, 4080, 0, 0, 0, 1, 0, 1521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 2703, 2773, 0, 1471, 61, 0, 0, 0, 0, 0, 0, 0, 46, 21, 0, 912, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1733, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 29, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 4035, 16, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2960, 4502, 7285, 7071, 1009, 1734, 4132, 4075] avg=4096.0000 max=7285 max_rank=2 min=1009 min_rank=4 imbalance_ratio=1.778564 max_load_reduction=25483 max_load_reduction_pct=0.777679 moved_to_echo=10950 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 17, 0, 0, 3310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1745, 2457, 727, 0, 0, 0, 0, 0, 1521, 0, 0, 1, 0, 0, 3681, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4031, 0, 1330, 0, 24, 0, 0, 0, 0, 0, 0, 191, 0, 13, 1591, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3766, 0, 1320, 0, 0, 184, 924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 896, 0, 0, 0, 3903, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3327, 4929, 5739, 5385, 1820, 6194, 5344, 30] avg=4096.0000 max=6194 max_rank=5 min=30 min_rank=7 imbalance_ratio=1.512207 max_load_reduction=26574 max_load_reduction_pct=0.810974 moved_to_echo=13388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 13, 0, 0, 121, 69, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 42, 2327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 30, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 0, 1761, 0, 0, 3971, 0, 3286, 0, 0, 0, 0, 0, 0, 0, 0, 3836, 0, 1, 0, 0, 0, 3823, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 0, 0, 0, 0, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4259, 2373, 4094, 34, 2, 13040, 7670, 1296] avg=4096.0000 max=13040 max_rank=5 min=2 min_rank=4 imbalance_ratio=3.183594 max_load_reduction=19728 max_load_reduction_pct=0.602051 moved_to_echo=22008 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1219, 0, 0, 0, 2053, 3864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 421, 0, 0, 0, 35, 0, 820, 0, 0, 0, 0, 0, 1704, 0, 0, 0, 754, 0, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1496, 48, 17, 0, 0, 0, 0, 0, 316, 0, 0, 4003, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 5, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 3240, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 1174, 0, 3021, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7136, 1289, 2560, 1561, 8415, 279, 3286, 8242] avg=4096.0000 max=8415 max_rank=4 min=279 min_rank=5 imbalance_ratio=2.054443 max_load_reduction=24353 max_load_reduction_pct=0.743195 moved_to_echo=20222 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[7808, 128, 8576, 1024, 4096, 6400, 1024, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1792, 2560, 0, 4096, 5248, 7552, 11776, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4480, 384, 512, 6016, 6656, 10496, 640, 3968, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[384, 0, 4224, 256, 3200, 0, 384, 128, 11264, 10240, 3200, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[6528, 4352, 1152, 1920, 4992, 512, 7808, 6016, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[2304, 0, 0, 128, 128, 2816, 10368, 11392, 5760, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[16128, 640, 5120, 128, 128, 0, 11008, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 7040, 1536, 128, 4224, 640, 15872, 3968, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.rerun_state_machine:Result validation enabled +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +Number of parameters in transformer block in billions: 6.71 [2026-09-21 08:25:35] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 38483.0 | throughput per GPU (TFLOP/s/GPU): 0.8 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.118499E+01 | loss scale: 1.0 | grad norm: 35.478 | number of skipped iterations: 0 | number of nan iterations: 0 | + +Number of parameters in embedding layers in billions: 0.21 +Total number of parameters in billions: 6.92 +Number of parameters in most loaded shard in billions: 6.9171 +Theoretical memory footprints: weight and optimizer=49474.72 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 19764.76171875 | max allocated: 23493.02880859375 | reserved: 24210.0 | max reserved: 24210.0 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[2329, 913, 156, 359, 307, 408, 81, 40, 415, 332, 197, 6, 45, 401, 229, 1476, 120, 514, 1171, 1677, 343, 693, 1417, 206, 1188, 140, 149, 1034, 447, 386, 67, 255, 217, 718, 556, 93, 1312, 222, 318, 410, 406, 244, 214, 3, 1657, 519, 23, 719, 476, 408, 407, 76, 1970, 353, 152, 667, 743, 28, 815, 584, 61, 445, 276, 175] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1958, 780, 94, 285, 284, 525, 151, 27, 441, 117, 62, 5, 85, 408, 370, 1483, 183, 497, 1341, 1794, 182, 662, 1317, 173, 811, 269, 153, 1620, 602, 342, 30, 291, 358, 595, 812, 130, 1289, 128, 264, 389, 819, 333, 651, 11, 1447, 515, 82, 391, 353, 671, 643, 117, 1914, 158, 211, 317, 657, 37, 697, 387, 145, 524, 260, 121] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[2340, 723, 75, 281, 298, 379, 64, 57, 650, 368, 84, 4, 172, 288, 285, 1633, 153, 572, 1275, 1179, 204, 560, 1126, 146, 989, 235, 49, 1897, 826, 368, 48, 210, 331, 895, 812, 47, 807, 270, 672, 462, 228, 535, 285, 33, 1544, 498, 152, 340, 306, 513, 523, 68, 2310, 321, 124, 216, 703, 25, 543, 215, 16, 541, 817, 78] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[2452, 688, 150, 280, 249, 300, 133, 31, 280, 275, 119, 12, 80, 556, 161, 1473, 138, 269, 1491, 1503, 277, 438, 1435, 240, 943, 275, 181, 1531, 671, 486, 39, 305, 271, 653, 679, 187, 1032, 287, 549, 645, 364, 472, 380, 18, 1603, 499, 69, 457, 505, 733, 657, 126, 1798, 141, 115, 389, 1007, 32, 414, 283, 16, 347, 480, 99] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1575, 847, 207, 428, 221, 560, 216, 198, 368, 352, 90, 0, 72, 188, 362, 1658, 186, 731, 1186, 1944, 58, 557, 870, 42, 487, 235, 208, 1051, 575, 644, 79, 193, 273, 643, 628, 27, 1038, 135, 156, 305, 1031, 573, 690, 52, 1419, 261, 162, 390, 430, 562, 693, 135, 1853, 203, 23, 355, 1092, 25, 1612, 510, 93, 344, 576, 91] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[2035, 671, 119, 272, 309, 363, 56, 17, 354, 289, 40, 1, 62, 327, 195, 1370, 97, 446, 1471, 1809, 122, 759, 1142, 270, 859, 95, 128, 1916, 533, 273, 37, 216, 480, 931, 644, 115, 1185, 91, 440, 451, 479, 301, 462, 8, 1622, 686, 60, 540, 972, 716, 637, 49, 2053, 377, 155, 214, 563, 29, 674, 219, 55, 522, 260, 125] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1700, 690, 122, 298, 213, 635, 173, 60, 298, 365, 68, 2, 136, 302, 216, 1440, 67, 693, 910, 2074, 104, 831, 994, 289, 1063, 195, 151, 1623, 493, 633, 79, 188, 306, 825, 724, 100, 1548, 167, 435, 511, 396, 338, 225, 26, 1426, 825, 73, 653, 801, 486, 441, 127, 1880, 279, 111, 448, 592, 22, 914, 215, 65, 230, 313, 161] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1562, 1025, 61, 466, 142, 327, 171, 9, 435, 231, 94, 0, 32, 435, 137, 1860, 72, 418, 1165, 1795, 95, 995, 1023, 653, 866, 212, 140, 1319, 703, 407, 92, 183, 240, 886, 824, 86, 610, 107, 616, 343, 428, 475, 203, 6, 1944, 697, 144, 728, 621, 393, 882, 19, 1973, 207, 136, 975, 629, 47, 611, 251, 18, 281, 147, 116] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2304, 913, 156, 359, 307, 408, 81, 40, 0, 0, 0, 0, 0, 0, 0, 0, 415, 332, 197, 6, 45, 401, 229, 1476, 522, 484, 37, 0, 0, 0, 0, 0, 120, 514, 1099, 0, 343, 693, 1417, 206, 0, 0, 0, 0, 0, 0, 0, 0, 1188, 140, 149, 1034, 447, 386, 67, 255, 7, 0, 0, 0, 0, 0, 0, 0, 217, 718, 556, 93, 1312, 222, 318, 410, 35, 25, 1, 0, 0, 0, 0, 0, 406, 244, 214, 3, 1649, 519, 23, 719, 0, 0, 0, 0, 0, 0, 0, 0, 476, 408, 407, 76, 1448, 353, 152, 667, 0, 0, 0, 0, 0, 0, 0, 0, 743, 28, 815, 584, 61, 445, 276, 175, 1193, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4568, 4144, 4392, 3673, 3907, 3777, 3987, 4320] avg=4096.0000 max=4568 max_rank=0 min=3673 min_rank=3 imbalance_ratio=1.115234 max_load_reduction=28200 max_load_reduction_pct=0.860596 moved_to_echo=15991 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2315, 723, 75, 281, 298, 379, 64, 57, 0, 0, 0, 0, 0, 0, 0, 0, 650, 368, 84, 4, 172, 288, 285, 1633, 612, 341, 40, 0, 0, 0, 0, 0, 153, 572, 1197, 0, 204, 560, 1126, 146, 0, 0, 0, 0, 0, 0, 0, 0, 989, 235, 49, 1897, 826, 368, 48, 210, 6, 0, 0, 0, 0, 0, 0, 0, 331, 895, 812, 47, 807, 270, 672, 462, 38, 25, 1, 0, 0, 0, 0, 0, 228, 535, 285, 33, 1537, 498, 152, 340, 0, 0, 0, 0, 0, 0, 0, 0, 306, 513, 523, 68, 1698, 321, 124, 216, 0, 0, 0, 0, 0, 0, 0, 0, 703, 25, 543, 215, 16, 541, 817, 78, 838, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4192, 4477, 3958, 4628, 4360, 3608, 3769, 3776] avg=4096.0000 max=4628 max_rank=3 min=3608 min_rank=5 imbalance_ratio=1.129883 max_load_reduction=28140 max_load_reduction_pct=0.858765 moved_to_echo=15513 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1937, 780, 94, 285, 284, 525, 151, 27, 0, 0, 0, 0, 0, 0, 0, 0, 441, 117, 62, 5, 85, 408, 370, 1483, 507, 518, 43, 0, 0, 0, 0, 0, 183, 497, 1258, 0, 182, 662, 1317, 173, 0, 0, 0, 0, 0, 0, 0, 0, 811, 269, 153, 1620, 602, 342, 30, 291, 6, 0, 0, 0, 0, 0, 0, 0, 358, 595, 812, 130, 1289, 128, 264, 389, 40, 21, 1, 0, 0, 0, 0, 0, 819, 333, 651, 11, 1440, 515, 82, 391, 0, 0, 0, 0, 0, 0, 0, 0, 353, 671, 643, 117, 1407, 158, 211, 317, 0, 0, 0, 0, 0, 0, 0, 0, 657, 37, 697, 387, 145, 524, 260, 121, 1276, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4039, 4272, 4124, 4027, 4242, 3877, 4104] avg=4096.0000 max=4272 max_rank=2 min=3877 min_rank=6 imbalance_ratio=1.042969 max_load_reduction=28496 max_load_reduction_pct=0.869629 moved_to_echo=16250 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2013, 671, 119, 272, 309, 363, 56, 17, 0, 0, 0, 0, 0, 0, 0, 0, 354, 289, 40, 1, 62, 327, 195, 1370, 544, 522, 47, 0, 0, 0, 0, 0, 97, 446, 1380, 0, 122, 759, 1142, 270, 0, 0, 0, 0, 0, 0, 0, 0, 859, 95, 128, 1916, 533, 273, 37, 216, 7, 0, 0, 0, 0, 0, 0, 0, 480, 931, 644, 115, 1185, 91, 440, 451, 44, 22, 1, 0, 0, 0, 0, 0, 479, 301, 462, 8, 1614, 686, 60, 540, 0, 0, 0, 0, 0, 0, 0, 0, 972, 716, 637, 49, 1509, 377, 155, 214, 0, 0, 0, 0, 0, 0, 0, 0, 563, 29, 674, 219, 55, 522, 260, 125, 1287, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3820, 3751, 4216, 4064, 4404, 4150, 4629, 3734] avg=4096.0000 max=4629 max_rank=6 min=3734 min_rank=7 imbalance_ratio=1.130127 max_load_reduction=28139 max_load_reduction_pct=0.858734 moved_to_echo=16917 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1553, 847, 207, 428, 221, 560, 216, 198, 0, 0, 0, 0, 0, 0, 0, 0, 368, 352, 90, 0, 72, 188, 362, 1658, 494, 562, 41, 0, 0, 0, 0, 0, 186, 731, 1107, 0, 58, 557, 870, 42, 0, 0, 0, 0, 0, 0, 0, 0, 487, 235, 208, 1051, 575, 644, 79, 193, 9, 0, 0, 0, 0, 0, 0, 0, 273, 643, 628, 27, 1038, 135, 156, 305, 38, 22, 4, 0, 0, 0, 0, 0, 1031, 573, 690, 52, 1406, 261, 162, 390, 0, 0, 0, 0, 0, 0, 0, 0, 430, 562, 693, 135, 1359, 203, 23, 355, 0, 0, 0, 0, 0, 0, 0, 0, 1092, 25, 1612, 510, 93, 344, 576, 91, 1382, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4230, 4187, 3551, 3481, 3269, 4565, 3760, 5725] avg=4096.0000 max=5725 max_rank=7 min=3269 min_rank=4 imbalance_ratio=1.397705 max_load_reduction=27043 max_load_reduction_pct=0.825287 moved_to_echo=17319 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2425, 688, 150, 280, 249, 300, 133, 31, 0, 0, 0, 0, 0, 0, 0, 0, 280, 275, 119, 12, 80, 556, 161, 1473, 476, 435, 47, 0, 0, 0, 0, 0, 138, 269, 1400, 0, 277, 438, 1435, 240, 0, 0, 0, 0, 0, 0, 0, 0, 943, 275, 181, 1531, 671, 486, 39, 305, 6, 0, 0, 0, 0, 0, 0, 0, 271, 653, 679, 187, 1032, 287, 549, 645, 44, 27, 1, 0, 0, 0, 0, 0, 364, 472, 380, 18, 1596, 499, 69, 457, 0, 0, 0, 0, 0, 0, 0, 0, 505, 733, 657, 126, 1322, 141, 115, 389, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 32, 414, 283, 16, 347, 480, 99, 1068, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4256, 3914, 4197, 4437, 4375, 3855, 3988, 3746] avg=4096.0000 max=4437 max_rank=3 min=3746 min_rank=7 imbalance_ratio=1.083252 max_load_reduction=28331 max_load_reduction_pct=0.864594 moved_to_echo=15964 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1682, 690, 122, 298, 213, 635, 173, 60, 0, 0, 0, 0, 0, 0, 0, 0, 298, 365, 68, 2, 136, 302, 216, 1440, 498, 599, 29, 0, 0, 0, 0, 0, 67, 693, 854, 0, 104, 831, 994, 289, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 195, 151, 1623, 493, 633, 79, 188, 6, 0, 0, 0, 0, 0, 0, 0, 306, 825, 724, 100, 1548, 167, 435, 511, 27, 18, 1, 0, 0, 0, 0, 0, 396, 338, 225, 26, 1419, 825, 73, 653, 0, 0, 0, 0, 0, 0, 0, 0, 801, 486, 441, 127, 1382, 279, 111, 448, 0, 0, 0, 0, 0, 0, 0, 0, 592, 22, 914, 215, 65, 230, 313, 161, 1475, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3873, 3953, 3832, 4431, 4662, 3955, 4075, 3987] avg=4096.0000 max=4662 max_rank=4 min=3832 min_rank=2 imbalance_ratio=1.138184 max_load_reduction=28106 max_load_reduction_pct=0.857727 moved_to_echo=16679 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1545, 1025, 61, 466, 142, 327, 171, 9, 0, 0, 0, 0, 0, 0, 0, 0, 435, 231, 94, 0, 32, 435, 137, 1860, 523, 519, 37, 0, 0, 0, 0, 0, 72, 418, 1094, 0, 95, 995, 1023, 653, 0, 0, 0, 0, 0, 0, 0, 0, 866, 212, 140, 1319, 703, 407, 92, 183, 8, 0, 0, 0, 0, 0, 0, 0, 240, 886, 824, 86, 610, 107, 616, 343, 34, 17, 1, 0, 0, 0, 0, 0, 428, 475, 203, 6, 1935, 697, 144, 728, 0, 0, 0, 0, 0, 0, 0, 0, 621, 393, 882, 19, 1450, 207, 136, 975, 0, 0, 0, 0, 0, 0, 0, 0, 629, 47, 611, 251, 18, 281, 147, 116, 1276, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3746, 4303, 4350, 3930, 3764, 4616, 4683, 3376] avg=4096.0000 max=4683 max_rank=6 min=3376 min_rank=7 imbalance_ratio=1.143311 max_load_reduction=28085 max_load_reduction_pct=0.857086 moved_to_echo=16439 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[3328, 2432, 768, 128, 768, 2944, 2048, 12416, 4224, 4096, 384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[2560, 6272, 5760, 896, 8832, 1408, 3456, 3584, 384, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[15872, 6400, 1024, 2688, 2048, 3584, 1152, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[6016, 256, 6400, 2688, 512, 3328, 3200, 1024, 9856, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 3328, 3200, 256, 12672, 4608, 768, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4480, 4608, 4992, 768, 11648, 2048, 1152, 3584, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1024, 4224, 9472, 0, 1408, 5504, 9344, 2048, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[7296, 1664, 1280, 12032, 4864, 3584, 512, 1920, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[619, 1, 331, 0, 6, 0, 174, 7, 574, 0, 2, 0, 1881, 1, 165, 5, 0, 4, 2, 1, 0, 2962, 0, 813, 0, 1090, 0, 894, 21, 558, 32, 3692, 13, 3, 4, 2844, 5, 0, 3, 3949, 239, 46, 10, 0, 0, 37, 374, 2405, 291, 554, 0, 189, 0, 317, 15, 0, 1568, 0, 28, 27, 2723, 3285, 1, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 1068, 5, 0, 4, 0, 107, 1, 3221, 0, 5, 0, 32, 0, 881, 1, 0, 0, 0, 0, 3, 2563, 0, 207, 3237, 0, 0, 1303, 86, 2042, 1821, 4080, 0, 0, 0, 1063, 10, 2, 0, 3757, 13, 605, 0, 0, 0, 3, 15, 3425, 3, 34, 1, 62, 1, 0, 13, 155, 782, 476, 71, 0, 653, 940, 17, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[21, 114, 738, 0, 55, 0, 2781, 127, 876, 27, 13, 11, 880, 0, 1517, 0, 1, 3, 89, 7, 0, 1497, 0, 2903, 0, 2, 0, 3838, 111, 44, 11, 3643, 0, 97, 127, 1032, 204, 1, 0, 2958, 2144, 25, 199, 7, 0, 1, 13, 1857, 0, 222, 0, 1, 85, 210, 0, 160, 390, 0, 56, 10, 1730, 1928, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[19, 2113, 1452, 20, 0, 1, 812, 1, 815, 1, 6, 0, 835, 7, 67, 4, 55, 0, 1, 3, 0, 1295, 2, 220, 1812, 24, 0, 3443, 1367, 579, 43, 2710, 13, 0, 89, 1232, 1, 3, 8, 3037, 8, 2789, 1660, 0, 1, 0, 165, 3110, 4, 159, 3, 906, 6, 105, 1, 50, 39, 7, 82, 1388, 51, 135, 8, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 20, 0, 0, 2, 181, 593, 1, 1003, 0, 366, 1, 205, 84, 4, 90, 1, 0, 16, 0, 0, 3981, 2, 2827, 1, 11, 0, 3090, 3130, 993, 350, 3214, 0, 25, 0, 1205, 13, 3, 33, 443, 6, 215, 1, 3565, 0, 15, 820, 1852, 64, 1, 8, 33, 36, 262, 0, 169, 2, 3, 8, 255, 391, 3174, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1, 1244, 547, 8, 15, 0, 3465, 2, 691, 7, 16, 213, 1, 1, 37, 3, 0, 0, 7, 0, 0, 3976, 0, 883, 27, 5, 0, 1211, 327, 1064, 203, 2270, 2, 0, 2, 3719, 211, 0, 0, 3327, 155, 19, 220, 251, 0, 0, 388, 3631, 0, 1, 0, 1840, 6, 106, 795, 765, 7, 4, 410, 12, 12, 657, 4, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[2, 16, 927, 0, 0, 1, 820, 0, 2827, 0, 0, 0, 1227, 0, 197, 1, 0, 0, 0, 1, 0, 2646, 16, 61, 1, 1536, 3, 2419, 703, 1611, 13, 3397, 17, 11, 2, 28, 71, 1, 357, 4042, 419, 383, 0, 6, 0, 6, 460, 3065, 2, 56, 14, 0, 0, 1090, 8, 62, 1180, 61, 511, 195, 1286, 1010, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[592, 676, 592, 0, 0, 0, 715, 7, 438, 0, 733, 14, 34, 0, 378, 187, 7, 0, 0, 5, 5, 3156, 1, 1959, 42, 74, 0, 3693, 344, 2819, 133, 3791, 11, 0, 0, 1245, 12, 0, 0, 4062, 58, 14, 10, 0, 0, 0, 44, 1452, 0, 21, 1, 707, 0, 0, 2, 383, 19, 1358, 384, 5, 246, 2248, 91, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[619, 1, 331, 0, 6, 0, 174, 7, 1008, 234, 1, 0, 0, 0, 0, 0, 574, 0, 2, 0, 1881, 1, 165, 5, 367, 552, 0, 0, 0, 0, 0, 0, 0, 4, 2, 1, 0, 2962, 0, 813, 67, 0, 0, 0, 0, 0, 0, 0, 0, 1090, 0, 293, 21, 558, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 3, 4, 2844, 5, 0, 3, 2941, 0, 0, 0, 0, 0, 0, 0, 0, 239, 46, 10, 0, 0, 37, 374, 2010, 0, 0, 0, 0, 0, 0, 0, 0, 291, 554, 0, 189, 0, 317, 15, 0, 3140, 0, 0, 0, 0, 0, 0, 0, 1568, 0, 28, 27, 2723, 3285, 1, 3, 327, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2381, 3547, 3849, 1994, 5813, 2716, 4506, 7962] avg=4096.0000 max=7962 max_rank=7 min=1994 min_rank=3 imbalance_ratio=1.943848 max_load_reduction=24806 max_load_reduction_pct=0.757019 moved_to_echo=20997 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1068, 5, 0, 4, 0, 107, 1, 959, 341, 2, 0, 0, 0, 0, 0, 3221, 0, 5, 0, 32, 0, 881, 1, 535, 610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2563, 0, 207, 96, 0, 0, 0, 0, 0, 0, 0, 3237, 0, 0, 427, 86, 2042, 1821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 10, 2, 0, 2798, 0, 0, 0, 0, 0, 0, 0, 0, 13, 605, 0, 0, 0, 3, 15, 2861, 0, 0, 0, 0, 0, 0, 0, 0, 3, 34, 1, 62, 1, 0, 13, 155, 3470, 0, 0, 0, 0, 0, 0, 0, 782, 476, 71, 0, 653, 940, 17, 0, 466, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2487, 5285, 2869, 7613, 3873, 3497, 3739, 3405] avg=4096.0000 max=7613 max_rank=3 min=2487 min_rank=0 imbalance_ratio=1.858643 max_load_reduction=25155 max_load_reduction_pct=0.767670 moved_to_echo=14514 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 1244, 547, 8, 15, 0, 3465, 2, 849, 317, 2, 0, 0, 0, 0, 0, 691, 7, 16, 213, 1, 1, 37, 3, 497, 339, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 3976, 0, 883, 102, 0, 0, 0, 0, 0, 0, 0, 27, 5, 0, 397, 327, 1064, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 3719, 211, 0, 0, 2478, 0, 0, 0, 0, 0, 0, 0, 0, 155, 19, 220, 251, 0, 0, 388, 3033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1840, 6, 106, 795, 765, 1931, 0, 0, 0, 0, 0, 0, 0, 7, 4, 410, 12, 12, 657, 4, 0, 494, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6450, 1805, 4968, 2023, 6412, 4066, 5444, 1600] avg=4096.0000 max=6450 max_rank=0 min=1600 min_rank=7 imbalance_ratio=1.574707 max_load_reduction=26318 max_load_reduction_pct=0.803162 moved_to_echo=17522 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 16, 927, 0, 0, 1, 820, 0, 1031, 634, 2, 0, 0, 0, 0, 0, 2827, 0, 0, 0, 1227, 0, 197, 1, 994, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2646, 16, 61, 86, 0, 0, 0, 0, 0, 0, 0, 1, 1536, 3, 791, 703, 1611, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 11, 2, 28, 71, 1, 357, 3011, 0, 0, 0, 0, 0, 0, 0, 0, 419, 383, 0, 6, 0, 6, 460, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 2, 56, 14, 0, 0, 1090, 8, 62, 2889, 0, 0, 0, 0, 0, 0, 0, 1180, 61, 511, 195, 1286, 1010, 0, 0, 417, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3433, 5754, 2810, 4658, 3498, 3834, 4121, 4660] avg=4096.0000 max=5754 max_rank=1 min=2810 min_rank=2 imbalance_ratio=1.404785 max_load_reduction=27014 max_load_reduction_pct=0.824402 moved_to_echo=16113 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[592, 676, 592, 0, 0, 0, 715, 7, 1036, 968, 1, 0, 0, 0, 0, 0, 438, 0, 733, 14, 34, 0, 378, 187, 1517, 567, 0, 0, 0, 0, 0, 0, 7, 0, 0, 5, 5, 3156, 1, 1959, 40, 0, 0, 0, 0, 0, 0, 0, 42, 74, 0, 1208, 344, 2819, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 1245, 12, 0, 0, 3026, 0, 0, 0, 0, 0, 0, 0, 0, 58, 14, 10, 0, 0, 0, 44, 1214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 1, 707, 0, 0, 2, 383, 3224, 0, 0, 0, 0, 0, 0, 0, 19, 1358, 384, 5, 246, 2248, 91, 0, 197, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4587, 3868, 5173, 4620, 4294, 1340, 4338, 4548] avg=4096.0000 max=5173 max_rank=2 min=1340 min_rank=5 imbalance_ratio=1.262939 max_load_reduction=27595 max_load_reduction_pct=0.842133 moved_to_echo=14520 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[19, 2113, 1452, 20, 0, 1, 812, 1, 775, 903, 2, 0, 0, 0, 0, 0, 815, 1, 6, 0, 835, 7, 67, 4, 1415, 406, 0, 0, 0, 0, 0, 0, 55, 0, 1, 3, 0, 1295, 2, 220, 87, 0, 0, 0, 0, 0, 0, 0, 1812, 24, 0, 1125, 1367, 579, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 89, 1232, 1, 3, 8, 2262, 0, 0, 0, 0, 0, 0, 0, 0, 8, 2789, 1660, 0, 1, 0, 165, 2598, 0, 0, 0, 0, 0, 0, 0, 0, 4, 159, 3, 906, 6, 105, 1, 50, 2304, 0, 0, 0, 0, 0, 0, 0, 39, 7, 82, 1388, 51, 135, 8, 1, 423, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6098, 3556, 1663, 4950, 3608, 7221, 3538, 2134] avg=4096.0000 max=7221 max_rank=5 min=1663 min_rank=2 imbalance_ratio=1.762939 max_load_reduction=25547 max_load_reduction_pct=0.779633 moved_to_echo=16501 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[21, 114, 738, 0, 55, 0, 2781, 127, 755, 1006, 1, 0, 0, 0, 0, 0, 876, 27, 13, 11, 880, 0, 1517, 0, 1577, 545, 0, 0, 0, 0, 0, 0, 1, 3, 89, 7, 0, 1497, 0, 2903, 52, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1255, 111, 44, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 127, 1032, 204, 1, 0, 2203, 0, 0, 0, 0, 0, 0, 0, 0, 2144, 25, 199, 7, 0, 1, 13, 1551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 222, 0, 1, 85, 210, 0, 160, 3098, 0, 0, 0, 0, 0, 0, 0, 390, 0, 56, 10, 1730, 1928, 2, 0, 253, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5598, 5446, 4552, 1423, 3664, 3940, 3776, 4369] avg=4096.0000 max=5598 max_rank=0 min=1423 min_rank=3 imbalance_ratio=1.366699 max_load_reduction=27170 max_load_reduction_pct=0.829163 moved_to_echo=15749 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 20, 0, 0, 2, 181, 593, 1, 116, 815, 6, 0, 0, 0, 0, 0, 1003, 0, 366, 1, 205, 84, 4, 90, 1273, 481, 0, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 3981, 2, 2827, 57, 0, 0, 0, 0, 0, 0, 0, 1, 11, 0, 1002, 3130, 993, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 1205, 13, 3, 33, 327, 0, 0, 0, 0, 0, 0, 0, 0, 6, 215, 1, 3565, 0, 15, 820, 1532, 0, 0, 0, 0, 0, 0, 0, 0, 64, 1, 8, 33, 36, 262, 0, 169, 2733, 0, 0, 0, 0, 0, 0, 0, 2, 3, 8, 255, 391, 3174, 0, 0, 257, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1734, 3507, 6884, 5487, 1606, 6154, 3306, 4090] avg=4096.0000 max=6884 max_rank=2 min=1606 min_rank=4 imbalance_ratio=1.680664 max_load_reduction=25884 max_load_reduction_pct=0.789917 moved_to_echo=15156 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 1920, 1664, 1920, 7168, 13440, 128, 128, 2944, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[10496, 128, 1152, 256, 5120, 128, 3328, 384, 8192, 4096, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[5120, 2816, 128, 6528, 6144, 9728, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[1280, 5376, 4608, 128, 128, 256, 9472, 256, 6656, 5248, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3072, 4096, 2176, 3840, 128, 128, 2304, 17408, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 256, 256, 12416, 640, 128, 512, 19072, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 22144, 128, 9984, 640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[384, 1152, 128, 3840, 256, 2176, 896, 1792, 22912, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 4, 0, 239, 4, 0, 2050, 0, 0, 2162, 0, 64, 3, 3, 0, 0, 9, 2301, 2, 0, 46, 9, 0, 228, 30, 803, 50, 4018, 20, 0, 3270, 8, 1032, 3917, 6, 6, 0, 427, 312, 81, 0, 1, 3979, 0, 2712, 2, 573, 688, 120, 0, 1723, 0, 0, 12, 0, 25, 0, 0, 0, 3, 1741, 6, 0, 79] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 8, 0, 0, 0, 0, 2, 12, 1220, 124, 1, 1405, 7, 113, 1, 3, 0, 251, 0, 4, 20, 0, 0, 3855, 10, 10, 1, 3054, 202, 0, 730, 0, 1987, 1684, 0, 33, 0, 176, 1178, 0, 0, 0, 95, 48, 3414, 27, 0, 0, 588, 0, 0, 0, 4022, 303, 0, 0, 93, 0, 1, 1, 4077, 4007, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 119, 0, 0, 1, 0, 0, 2582, 9, 0, 0, 0, 0, 0, 0, 0, 0, 214, 0, 0, 2, 602, 0, 4, 0, 4067, 0, 830, 648, 0, 4023, 0, 2753, 0, 0, 0, 3, 1, 14, 488, 281, 0, 3650, 5, 558, 3961, 2, 44, 1705, 0, 4066, 8, 0, 0, 1, 0, 0, 0, 11, 2102, 1, 13] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[89, 619, 1, 3, 1, 0, 95, 0, 0, 3193, 0, 20, 15, 0, 5, 0, 1, 8, 0, 66, 0, 0, 0, 2148, 0, 1, 0, 2922, 0, 4, 4012, 329, 116, 3988, 1, 0, 0, 0, 3418, 0, 0, 1, 185, 13, 3864, 0, 4, 921, 314, 272, 4, 0, 3731, 0, 0, 5, 141, 715, 0, 3, 125, 1381, 0, 34] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[405, 1, 0, 571, 0, 0, 1530, 0, 0, 3669, 3, 0, 22, 1, 0, 10, 0, 21, 1, 282, 0, 0, 0, 898, 489, 176, 0, 4038, 1, 0, 0, 10, 184, 3932, 4, 1, 0, 3285, 907, 1, 0, 951, 1270, 59, 428, 0, 2, 1424, 665, 15, 34, 12, 3602, 0, 0, 2, 1050, 1, 0, 84, 2518, 0, 209, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[103, 1, 3, 0, 0, 1, 769, 0, 510, 2078, 0, 3, 0, 0, 0, 0, 82, 756, 1, 0, 0, 0, 0, 3264, 0, 0, 0, 3280, 5, 61, 456, 0, 2264, 369, 0, 183, 0, 4016, 4095, 0, 0, 0, 444, 13, 99, 1, 0, 0, 1932, 996, 1, 0, 4000, 0, 12, 0, 9, 5, 0, 0, 2560, 393, 2, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[131, 0, 0, 100, 0, 67, 65, 1, 0, 2758, 0, 0, 0, 0, 0, 0, 893, 2, 0, 35, 0, 61, 7, 2259, 0, 21, 4, 1357, 19, 317, 0, 0, 3703, 183, 528, 2, 0, 5, 3973, 0, 1, 0, 3300, 29, 4079, 7, 2588, 889, 1, 0, 95, 0, 455, 0, 0, 94, 39, 145, 0, 0, 3542, 297, 0, 716] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 7, 6, 3064, 3, 133, 221, 3, 2594, 0, 0, 0, 0, 1, 0, 3, 1, 0, 0, 0, 750, 0, 2, 0, 3404, 3, 41, 1789, 0, 3652, 878, 0, 252, 27, 175, 0, 0, 0, 0, 855, 1, 645, 2905, 0, 848, 243, 704, 2629, 23, 4012, 0, 3, 0, 0, 1, 0, 0, 2886, 0, 0, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 239, 4, 0, 2050, 0, 1032, 822, 0, 0, 0, 0, 0, 0, 0, 2162, 0, 64, 3, 3, 0, 0, 1177, 568, 0, 0, 0, 0, 0, 0, 9, 2301, 2, 0, 46, 9, 0, 228, 1123, 1535, 0, 0, 0, 0, 0, 0, 30, 803, 50, 2832, 20, 0, 3270, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1814, 6, 6, 0, 427, 312, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3979, 0, 767, 2, 573, 688, 0, 0, 0, 0, 0, 0, 0, 0, 120, 0, 1723, 0, 0, 12, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1741, 6, 0, 79, 0, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4151, 3977, 5253, 7013, 2646, 6010, 1880, 1838] avg=4096.0000 max=7013 max_rank=3 min=1838 min_rank=7 imbalance_ratio=1.712158 max_load_reduction=25755 max_load_reduction_pct=0.785980 moved_to_echo=12374 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 0, 0, 0, 0, 2, 12, 1987, 1035, 0, 0, 0, 0, 0, 0, 1220, 124, 1, 1405, 7, 113, 1, 3, 895, 244, 0, 0, 0, 0, 0, 0, 0, 251, 0, 4, 20, 0, 0, 3855, 1413, 660, 0, 0, 0, 0, 0, 0, 10, 10, 1, 2152, 202, 0, 730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 780, 0, 33, 0, 176, 1178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 48, 966, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 588, 0, 0, 0, 3399, 303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 1, 1, 4077, 4007, 0, 1, 623, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3044, 4013, 6203, 3105, 2167, 1136, 4290, 8810] avg=4096.0000 max=8810 max_rank=7 min=1136 min_rank=5 imbalance_ratio=2.150879 max_load_reduction=23958 max_load_reduction_pct=0.731140 moved_to_echo=16403 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[89, 619, 1, 3, 1, 0, 95, 0, 116, 1172, 0, 0, 0, 0, 0, 0, 0, 3193, 0, 20, 15, 0, 5, 0, 856, 578, 0, 0, 0, 0, 0, 0, 1, 8, 0, 66, 0, 0, 0, 2148, 1600, 1563, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2059, 0, 4, 4012, 329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1847, 1, 0, 0, 0, 3418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 185, 13, 1092, 0, 4, 921, 0, 0, 0, 0, 0, 0, 0, 0, 314, 272, 4, 0, 3153, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 141, 715, 0, 3, 125, 1381, 0, 34, 578, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2096, 4667, 5386, 6405, 5266, 2216, 3748, 2984] avg=4096.0000 max=6405 max_rank=3 min=2096 min_rank=0 imbalance_ratio=1.563721 max_load_reduction=26363 max_load_reduction_pct=0.804535 moved_to_echo=14214 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 119, 0, 0, 1, 0, 4023, 1107, 0, 0, 0, 0, 0, 0, 0, 2582, 9, 0, 0, 0, 0, 0, 1192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 214, 0, 0, 2, 602, 1511, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2865, 0, 830, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2753, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 14, 488, 281, 0, 1032, 5, 558, 3961, 0, 0, 0, 0, 0, 0, 0, 0, 2, 44, 1705, 0, 3436, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 11, 2102, 1, 13, 630, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5250, 3783, 2329, 4347, 2757, 6339, 5195, 2768] avg=4096.0000 max=6339 max_rank=5 min=2329 min_rank=2 imbalance_ratio=1.547607 max_load_reduction=26429 max_load_reduction_pct=0.806549 moved_to_echo=17059 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[131, 0, 0, 100, 0, 67, 65, 1, 3703, 1237, 0, 0, 0, 0, 0, 0, 0, 2758, 0, 0, 0, 0, 0, 0, 397, 26, 0, 0, 0, 0, 0, 0, 893, 2, 0, 35, 0, 61, 7, 2259, 1689, 71, 0, 0, 0, 0, 0, 0, 0, 21, 4, 957, 19, 317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 528, 2, 0, 5, 3973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3300, 29, 1153, 7, 2588, 889, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 95, 0, 385, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 39, 145, 0, 0, 3542, 297, 0, 716, 70, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5304, 3181, 5017, 1318, 4594, 7967, 575, 4812] avg=4096.0000 max=7967 max_rank=5 min=575 min_rank=6 imbalance_ratio=1.945068 max_load_reduction=24801 max_load_reduction_pct=0.756866 moved_to_echo=17948 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[405, 1, 0, 571, 0, 0, 1530, 0, 184, 129, 0, 0, 0, 0, 0, 0, 0, 3669, 3, 0, 22, 1, 0, 10, 1183, 570, 0, 0, 0, 0, 0, 0, 0, 21, 1, 282, 0, 0, 0, 898, 177, 1541, 0, 0, 0, 0, 0, 0, 489, 176, 0, 2845, 1, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1821, 4, 1, 0, 3285, 907, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 951, 1270, 59, 122, 0, 2, 1424, 0, 0, 0, 0, 0, 0, 0, 0, 665, 15, 34, 12, 3044, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1, 0, 84, 2518, 0, 209, 0, 558, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2820, 5458, 2920, 3521, 6019, 3828, 3772, 4430] avg=4096.0000 max=6019 max_rank=4 min=2820 min_rank=0 imbalance_ratio=1.469482 max_load_reduction=26749 max_load_reduction_pct=0.816315 moved_to_echo=18049 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 7, 6, 3064, 3, 3652, 195, 0, 0, 0, 0, 0, 0, 133, 221, 3, 2594, 0, 0, 0, 0, 997, 127, 0, 0, 0, 0, 0, 0, 1, 0, 3, 1, 0, 0, 0, 750, 267, 344, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2399, 3, 41, 1789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 407, 0, 252, 27, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 855, 1, 183, 2905, 0, 848, 0, 0, 0, 0, 0, 0, 0, 0, 243, 704, 2629, 23, 3390, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2886, 0, 0, 3, 622, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6928, 4075, 1366, 4234, 861, 4792, 6992, 3520] avg=4096.0000 max=6992 max_rank=6 min=861 min_rank=4 imbalance_ratio=1.707031 max_load_reduction=25776 max_load_reduction_pct=0.786621 moved_to_echo=16165 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[103, 1, 3, 0, 0, 1, 769, 0, 2264, 34, 0, 0, 0, 0, 0, 0, 510, 2078, 0, 3, 0, 0, 0, 0, 966, 57, 0, 0, 0, 0, 0, 0, 82, 756, 1, 0, 0, 0, 0, 3264, 43, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2303, 5, 61, 456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 0, 183, 0, 4016, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 444, 13, 22, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1932, 996, 1, 0, 3375, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 5, 0, 0, 2560, 393, 2, 1, 625, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3175, 3614, 4294, 2825, 8458, 480, 6316, 3606] avg=4096.0000 max=8458 max_rank=4 min=480 min_rank=5 imbalance_ratio=2.064941 max_load_reduction=24310 max_load_reduction_pct=0.741882 moved_to_echo=18860 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 1536, 10496, 256, 5376, 3072, 3840, 8832, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 7040, 3328, 512, 128, 8192, 13952, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[640, 1024, 128, 18432, 256, 1280, 11008, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[768, 640, 128, 1152, 128, 128, 7680, 128, 17024, 5760, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3968, 2048, 6272, 128, 20224, 384, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[1024, 3456, 128, 640, 128, 128, 128, 14080, 7936, 5888, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1920, 16896, 128, 4096, 128, 128, 128, 128, 7680, 2176, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1408, 896, 128, 128, 17536, 8192, 256, 896, 3712, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 3979, 4067, 73, 0, 3703, 148, 6, 0, 3501, 13, 4, 1419, 1, 0, 0, 0, 4093, 107, 0, 0, 91, 13, 4, 82, 0, 11, 0, 1, 135, 0, 0, 0, 35, 3881, 0, 0, 0, 0, 0, 60, 0, 703, 0, 1, 19, 1, 0, 0, 0, 15, 15, 2732, 1, 3503, 1, 0, 1, 0, 300, 0, 12, 23, 14] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 4043, 663, 2, 330, 97, 0, 578, 0, 3701, 0, 0, 520, 0, 0, 0, 0, 4096, 0, 3, 0, 618, 0, 0, 0, 0, 0, 1048, 0, 195, 0, 0, 0, 20, 3761, 0, 0, 8, 0, 0, 1, 0, 4051, 0, 126, 0, 4021, 1, 70, 0, 378, 1, 3432, 1, 58, 0, 591, 1, 0, 160, 160, 0, 29, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 212, 3650, 88, 0, 0, 110, 0, 589, 7, 0, 3875, 0, 46, 505, 1, 1714, 1, 0, 0, 9, 0, 0, 0, 0, 4, 1040, 0, 0, 269, 0, 1129, 3465, 1296, 238, 0, 0, 0, 313, 3947, 1, 75, 0, 0, 2, 1392, 1, 18, 8, 0, 522, 3547, 1, 4, 5, 120, 2, 0, 57, 0, 0, 484, 4021] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 917, 235, 1105, 1524, 2401, 0, 0, 0, 4037, 1, 0, 0, 0, 2, 454, 124, 4085, 0, 0, 0, 3184, 0, 0, 1, 0, 72, 0, 0, 44, 3, 0, 533, 26, 3569, 0, 0, 0, 0, 8, 0, 2, 1498, 0, 2, 0, 72, 0, 77, 2, 860, 877, 3743, 0, 1, 0, 2091, 4, 0, 959, 241, 1, 0, 13] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 3627, 182, 7, 50, 210, 0, 0, 1, 1588, 0, 0, 0, 2, 0, 8, 4, 4080, 0, 0, 0, 0, 0, 2, 1, 0, 4, 0, 1, 313, 0, 15, 80, 3, 2282, 0, 2, 90, 2, 0, 0, 7, 2318, 1474, 2, 0, 3728, 0, 3599, 0, 2, 59, 3762, 0, 0, 0, 1551, 0, 0, 114, 0, 61, 1, 3536] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[275, 3492, 116, 1, 0, 0, 0, 0, 0, 4066, 0, 58, 458, 1, 0, 306, 0, 3984, 0, 0, 0, 85, 0, 1, 0, 8, 156, 169, 0, 0, 0, 35, 3, 0, 3230, 2, 0, 0, 0, 0, 1030, 62, 3578, 1, 2, 0, 3242, 0, 46, 32, 16, 0, 4084, 0, 3710, 48, 0, 0, 0, 0, 471, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 22, 0, 147, 0, 0, 0, 2620, 917, 3723, 37, 0, 341, 0, 0, 0, 0, 4090, 48, 0, 0, 810, 33, 0, 0, 0, 0, 1896, 218, 0, 230, 0, 0, 0, 2438, 0, 0, 0, 0, 163, 0, 0, 62, 362, 2320, 1, 3781, 383, 253, 4, 0, 3676, 3977, 0, 0, 0, 0, 0, 0, 0, 176, 0, 0, 40] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 1180, 3740, 22, 46, 3673, 0, 0, 144, 835, 181, 0, 0, 0, 288, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 573, 198, 3, 351, 18, 18, 1351, 4033, 0, 0, 0, 0, 0, 0, 0, 2761, 0, 2313, 1, 106, 61, 0, 44, 1190, 0, 4066, 0, 48, 0, 1, 0, 40, 347, 543, 0, 0, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 621, 4067, 73, 0, 3703, 148, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3501, 13, 4, 1419, 1, 0, 0, 154, 0, 0, 0, 0, 0, 0, 0, 0, 3741, 107, 0, 0, 91, 13, 4, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 11, 0, 1, 135, 0, 0, 1463, 2294, 0, 0, 0, 0, 0, 0, 0, 35, 3881, 0, 0, 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 60, 0, 703, 0, 1, 19, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 15, 1269, 1, 3503, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 300, 0, 12, 23, 14, 0, 1064, 92, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8618, 5092, 3956, 3986, 4022, 784, 4804, 1506] avg=4096.0000 max=8618 max_rank=0 min=784 min_rank=5 imbalance_ratio=2.104004 max_load_reduction=24150 max_load_reduction_pct=0.737000 moved_to_echo=11116 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 631, 663, 2, 330, 97, 0, 578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3701, 0, 0, 520, 0, 0, 0, 154, 0, 0, 0, 0, 0, 0, 0, 0, 3744, 0, 3, 0, 618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 0, 195, 0, 0, 1838, 2331, 0, 0, 0, 0, 0, 0, 0, 20, 3761, 0, 0, 8, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4051, 0, 126, 0, 1364, 1, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 378, 1, 1594, 1, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 591, 1, 0, 160, 160, 0, 29, 4, 2657, 1081, 92, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2301, 4375, 4365, 5412, 3895, 5543, 2102, 4775] avg=4096.0000 max=5543 max_rank=5 min=2102 min_rank=6 imbalance_ratio=1.353271 max_load_reduction=27225 max_load_reduction_pct=0.830841 moved_to_echo=16315 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 144, 235, 1105, 1524, 2401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 1, 0, 0, 0, 2, 454, 153, 0, 0, 0, 0, 0, 0, 0, 124, 3734, 0, 0, 0, 3184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 72, 0, 0, 44, 3, 0, 2005, 528, 0, 0, 0, 0, 0, 0, 533, 26, 3569, 0, 0, 0, 0, 8, 106, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1498, 0, 2, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 2, 860, 877, 1738, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2091, 4, 0, 959, 241, 1, 0, 13, 47, 245, 92, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5409, 4647, 7042, 2653, 4242, 1527, 3555, 3693] avg=4096.0000 max=7042 max_rank=2 min=1527 min_rank=5 imbalance_ratio=1.719238 max_load_reduction=25726 max_load_reduction_pct=0.785095 moved_to_echo=13017 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 566, 182, 7, 50, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1588, 0, 0, 0, 2, 0, 8, 153, 0, 0, 0, 0, 0, 0, 0, 4, 3729, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 1, 313, 0, 15, 2015, 2091, 0, 0, 0, 0, 0, 0, 80, 3, 2282, 0, 2, 90, 2, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2318, 1474, 2, 0, 1264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3599, 0, 2, 59, 1747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1551, 0, 0, 114, 0, 61, 1, 3536, 2464, 970, 92, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1015, 1752, 3735, 4440, 2565, 5065, 5407, 8789] avg=4096.0000 max=8789 max_rank=7 min=1015 min_rank=0 imbalance_ratio=2.145752 max_load_reduction=23979 max_load_reduction_pct=0.731781 moved_to_echo=21826 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 212, 3650, 88, 0, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 589, 7, 0, 3875, 0, 46, 505, 64, 0, 0, 0, 0, 0, 0, 0, 1, 1568, 1, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1040, 0, 0, 269, 0, 1900, 0, 0, 0, 0, 0, 0, 0, 1129, 3465, 1296, 238, 0, 0, 0, 313, 44, 0, 0, 0, 0, 0, 0, 0, 3947, 1, 75, 0, 0, 2, 472, 1, 0, 0, 0, 0, 0, 0, 0, 0, 18, 8, 0, 522, 1647, 1, 4, 5, 0, 0, 0, 0, 0, 0, 0, 0, 120, 2, 0, 57, 0, 0, 484, 4021, 920, 0, 38, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 5086, 1579, 3213, 6485, 4498, 2205, 5642] avg=4096.0000 max=6485 max_rank=4 min=1579 min_rank=2 imbalance_ratio=1.583252 max_load_reduction=26283 max_load_reduction_pct=0.802094 moved_to_echo=18830 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[275, 545, 116, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 58, 458, 1, 0, 306, 150, 0, 0, 0, 0, 0, 0, 0, 0, 3641, 0, 0, 0, 85, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 156, 169, 0, 0, 0, 35, 2187, 2013, 0, 0, 0, 0, 0, 0, 3, 0, 3230, 2, 0, 0, 0, 0, 103, 0, 0, 0, 0, 0, 0, 0, 1030, 62, 3578, 1, 2, 0, 1099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 32, 16, 0, 1897, 0, 3710, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 0, 0, 0, 2143, 934, 90, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[937, 5039, 3727, 4568, 3338, 5772, 5749, 3638] avg=4096.0000 max=5772 max_rank=5 min=937 min_rank=0 imbalance_ratio=1.409180 max_load_reduction=26996 max_load_reduction_pct=0.823853 moved_to_echo=18497 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 180, 3740, 22, 46, 3673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 835, 181, 0, 0, 0, 288, 0, 154, 0, 0, 0, 0, 0, 0, 0, 0, 3744, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 573, 198, 3, 351, 18, 2178, 680, 0, 0, 0, 0, 0, 0, 18, 1351, 4033, 0, 0, 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2761, 0, 2313, 1, 36, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 1190, 0, 1888, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 40, 347, 543, 0, 0, 493, 70, 320, 92, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7661, 1602, 3748, 4001, 5508, 5172, 3170, 1906] avg=4096.0000 max=7661 max_rank=0 min=1602 min_rank=1 imbalance_ratio=1.870361 max_load_reduction=25107 max_load_reduction_pct=0.766205 moved_to_echo=15756 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 147, 0, 0, 0, 2620, 0, 0, 0, 0, 0, 0, 0, 0, 917, 3723, 37, 0, 341, 0, 0, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 3725, 48, 0, 0, 810, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1896, 218, 0, 230, 0, 2134, 17, 0, 0, 0, 0, 0, 0, 0, 0, 2438, 0, 0, 0, 0, 163, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 362, 2320, 1, 1279, 383, 0, 0, 0, 0, 0, 0, 0, 0, 253, 4, 0, 3676, 1843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 0, 40, 2502, 5, 96, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2767, 5175, 4616, 4495, 2713, 4407, 5776, 2819] avg=4096.0000 max=5776 max_rank=6 min=2713 min_rank=4 imbalance_ratio=1.410156 max_load_reduction=26992 max_load_reduction_pct=0.823730 moved_to_echo=15715 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 256, 4736, 512, 768, 896, 128, 15744, 9984, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[384, 2688, 9216, 5120, 2048, 10112, 256, 3328, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[4480, 128, 128, 2048, 1664, 128, 640, 8192, 10880, 4736, 768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[5120, 128, 15104, 1920, 4864, 128, 5632, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 128, 2560, 5248, 13696, 128, 7424, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[256, 27648, 256, 128, 0, 4864, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1152, 22144, 256, 128, 6656, 128, 384, 1280, 1152, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1792, 4992, 24576, 256, 128, 128, 128, 512, 896, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33792 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 3855, 8, 8, 234, 4, 0, 0, 0, 1479, 0, 0, 0, 0, 0, 1680, 1337, 33, 8, 0, 16, 0, 0, 0, 0, 0, 2104, 3603, 2674, 3, 0, 7, 0, 4052, 251, 10, 0, 0, 2239, 3817, 0, 0, 2466, 4, 0, 0, 421, 0, 0, 0, 0, 6, 1, 0, 8, 82, 0, 862, 0, 1403, 0, 93] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[490, 0, 1, 2, 1467, 566, 108, 821, 4027, 1, 0, 1281, 0, 0, 0, 4, 1441, 3490, 239, 0, 0, 0, 2761, 10, 0, 15, 0, 0, 0, 5, 846, 3990, 0, 0, 0, 0, 1, 88, 3, 0, 3028, 0, 0, 2, 0, 0, 4, 1781, 0, 0, 0, 3300, 683, 41, 0, 0, 0, 0, 1857, 19, 0, 326, 0, 70] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[2, 0, 0, 3487, 380, 710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2894, 81, 0, 3698, 768, 3, 1197, 0, 41, 0, 0, 0, 0, 0, 2560, 3854, 0, 2, 2, 1, 0, 1568, 0, 0, 2, 256, 0, 1, 725, 0, 3376, 18, 0, 0, 0, 0, 15, 109, 0, 3, 0, 0, 25, 2791, 113, 4086, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 216, 164, 543, 0, 0, 620, 0, 0, 1, 0, 0, 0, 51, 0, 4089, 79, 3818, 0, 0, 3731, 12, 0, 0, 0, 0, 0, 0, 0, 3744, 0, 4, 0, 3378, 0, 108, 0, 0, 0, 0, 0, 0, 0, 2527, 3937, 389, 0, 211, 0, 509, 2670, 0, 0, 0, 0, 0, 0, 130, 19, 1792, 0, 26] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[22, 0, 190, 3753, 12, 0, 1, 0, 2564, 0, 0, 398, 0, 392, 6, 0, 449, 0, 1, 4016, 0, 94, 272, 1837, 0, 51, 0, 822, 0, 37, 4076, 3147, 0, 0, 1, 2859, 0, 2, 0, 0, 746, 2837, 743, 145, 0, 0, 889, 254, 5, 0, 0, 1537, 100, 0, 2, 1, 0, 0, 20, 0, 0, 2, 476, 9] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[173, 0, 4, 36, 3694, 1135, 0, 0, 23, 1100, 0, 0, 101, 0, 54, 0, 3307, 0, 0, 35, 6, 189, 3659, 0, 0, 0, 776, 1, 0, 1770, 0, 3623, 0, 4, 0, 12, 4, 4, 0, 0, 0, 0, 3974, 42, 0, 221, 4033, 0, 207, 3932, 4, 141, 0, 0, 2, 0, 0, 0, 0, 483, 0, 9, 0, 10] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[13, 0, 0, 3699, 2398, 1741, 393, 0, 1, 35, 0, 4, 0, 0, 12, 0, 3736, 3653, 0, 0, 0, 232, 19, 0, 0, 1, 0, 2, 0, 0, 149, 3074, 0, 0, 0, 164, 11, 3, 0, 163, 1361, 0, 656, 2095, 736, 3178, 3850, 4, 67, 60, 7, 0, 43, 972, 1, 0, 0, 0, 63, 0, 1, 0, 66, 105] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 3221, 124, 0, 101, 0, 0, 0, 3519, 20, 1, 0, 8, 1392, 1572, 311, 1325, 0, 0, 1945, 1, 0, 0, 0, 0, 0, 0, 4066, 3385, 0, 0, 490, 2, 0, 0, 0, 0, 0, 0, 2446, 0, 14, 3728, 8, 0, 28, 0, 0, 0, 0, 32, 0, 0, 11, 0, 405, 520, 1, 4091, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3597, 8, 8, 234, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1479, 0, 0, 0, 0, 1047, 9, 0, 0, 0, 0, 0, 0, 0, 0, 1337, 33, 8, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2104, 3603, 2674, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4052, 251, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2239, 3817, 0, 0, 2466, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 421, 0, 0, 0, 0, 6, 1, 0, 1, 633, 0, 0, 0, 0, 0, 0, 8, 82, 0, 862, 0, 1403, 0, 93, 3, 4, 258, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3851, 2535, 1381, 8383, 4320, 8523, 1062, 2713] avg=4096.0000 max=8523 max_rank=5 min=1062 min_rank=6 imbalance_ratio=2.080811 max_load_reduction=24245 max_load_reduction_pct=0.739899 moved_to_echo=16618 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[490, 0, 1, 2, 1467, 566, 108, 821, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 1, 0, 1281, 0, 0, 0, 4, 2175, 1621, 0, 0, 0, 0, 0, 0, 1441, 0, 239, 0, 0, 0, 304, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 5, 846, 1475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 88, 3, 0, 4, 558, 0, 0, 0, 0, 0, 0, 3028, 0, 0, 2, 0, 0, 0, 1781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3300, 683, 41, 0, 0, 1957, 1315, 0, 0, 0, 0, 0, 0, 0, 0, 1857, 19, 0, 326, 0, 70, 0, 836, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3455, 9109, 1994, 2341, 654, 4811, 7296, 3108] avg=4096.0000 max=9109 max_rank=1 min=654 min_rank=4 imbalance_ratio=2.223877 max_load_reduction=23659 max_load_reduction_pct=0.722015 moved_to_echo=15869 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 3254, 380, 710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 702, 0, 0, 0, 0, 0, 0, 2894, 0, 0, 3698, 768, 3, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 2560, 1425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 1, 0, 1568, 0, 0, 3376, 539, 0, 0, 0, 0, 0, 0, 2, 256, 0, 1, 725, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 109, 0, 3, 1890, 31, 0, 0, 0, 0, 0, 0, 0, 0, 25, 2791, 113, 4086, 0, 0, 0, 362, 233, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4346, 752, 7496, 4026, 5488, 1002, 2048, 7610] avg=4096.0000 max=7610 max_rank=7 min=752 min_rank=1 imbalance_ratio=1.857910 max_load_reduction=25158 max_load_reduction_pct=0.767761 moved_to_echo=16148 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 202, 164, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 620, 0, 0, 1, 0, 0, 0, 51, 2550, 2190, 0, 0, 0, 0, 0, 0, 0, 0, 79, 3818, 0, 0, 412, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3378, 0, 108, 0, 0, 3937, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 0, 389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 211, 0, 509, 2670, 0, 0, 0, 1836, 1539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 19, 1792, 0, 26, 2006, 1129, 14, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[909, 5412, 4321, 1384, 7951, 910, 6765, 5116] avg=4096.0000 max=7951 max_rank=4 min=909 min_rank=0 imbalance_ratio=1.941162 max_load_reduction=24817 max_load_reduction_pct=0.757355 moved_to_echo=20742 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[22, 0, 190, 3502, 12, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2564, 0, 0, 398, 0, 392, 6, 0, 0, 159, 0, 0, 0, 0, 0, 0, 449, 0, 1, 4016, 0, 94, 31, 1837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 822, 0, 37, 4076, 1164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2859, 0, 2, 0, 0, 889, 440, 0, 0, 0, 0, 0, 0, 746, 2837, 743, 145, 0, 0, 0, 254, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 1537, 100, 0, 2, 1, 1543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 2, 476, 9, 0, 82, 251, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3727, 3519, 6428, 6150, 4191, 4725, 3188, 840] avg=4096.0000 max=6428 max_rank=2 min=840 min_rank=7 imbalance_ratio=1.569336 max_load_reduction=26340 max_load_reduction_pct=0.803833 moved_to_echo=12944 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 3221, 124, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3519, 20, 1, 0, 8, 981, 1141, 0, 0, 0, 0, 0, 0, 1392, 0, 311, 1325, 0, 0, 216, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 1251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 2, 0, 0, 0, 0, 8, 474, 0, 0, 0, 0, 0, 0, 0, 0, 2446, 0, 14, 769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 32, 0, 0, 1660, 591, 0, 0, 0, 0, 0, 0, 11, 0, 405, 520, 1, 4091, 0, 0, 2959, 588, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3447, 5670, 3245, 5317, 974, 3229, 2311, 8575] avg=4096.0000 max=8575 max_rank=7 min=974 min_rank=4 imbalance_ratio=2.093506 max_load_reduction=24193 max_load_reduction_pct=0.738312 moved_to_echo=15089 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[13, 0, 0, 3451, 2398, 1741, 393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 35, 0, 4, 0, 0, 12, 0, 2278, 11, 0, 0, 0, 0, 0, 0, 3736, 0, 0, 0, 0, 232, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 149, 1137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 11, 3, 0, 163, 3850, 430, 0, 0, 0, 0, 0, 0, 1361, 0, 656, 2095, 736, 656, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 67, 60, 7, 0, 43, 972, 1, 0, 1507, 1375, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 1, 0, 66, 105, 2522, 5, 248, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7996, 2341, 3971, 1289, 4621, 5508, 4032, 3010] avg=4096.0000 max=7996 max_rank=0 min=1289 min_rank=3 imbalance_ratio=1.952148 max_load_reduction=24772 max_load_reduction_pct=0.755981 moved_to_echo=17171 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[173, 0, 4, 31, 3694, 1135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 1100, 0, 0, 101, 0, 54, 0, 0, 2152, 0, 0, 0, 0, 0, 0, 3307, 0, 0, 35, 6, 189, 395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 776, 1, 0, 1770, 0, 1331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 12, 4, 4, 0, 0, 4033, 512, 0, 0, 0, 0, 0, 0, 0, 0, 3974, 42, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 3932, 4, 141, 0, 0, 2, 0, 1780, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 0, 9, 0, 10, 177, 1112, 5, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5037, 3430, 3932, 3878, 4569, 4060, 6066, 1796] avg=4096.0000 max=6066 max_rank=6 min=1796 min_rank=7 imbalance_ratio=1.480957 max_load_reduction=26702 max_load_reduction_pct=0.814880 moved_to_echo=16491 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[7296, 1152, 0, 6784, 128, 512, 128, 128, 9088, 8064, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 896, 896, 2176, 5504, 14464, 9216, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 2432, 4864, 256, 11776, 640, 384, 7680, 4224, 1024, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[7424, 6912, 7936, 2304, 3968, 2048, 0, 2560, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[768, 4224, 128, 5504, 3584, 1280, 128, 128, 12288, 5504, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[13312, 0, 2048, 12928, 896, 640, 1536, 1920, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[768, 128, 256, 14080, 11392, 4864, 768, 1024, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 512, 10496, 384, 1792, 128, 256, 16128, 3584, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 12, 41, 0, 0, 0, 0, 0, 1823, 0, 0, 0, 0, 0, 3864, 0, 0, 7, 11, 0, 0, 302, 557, 0, 0, 0, 3573, 96, 0, 0, 0, 0, 0, 3985, 2349, 0, 150, 4, 2, 0, 66, 3267, 632, 0, 0, 0, 1, 3704, 0, 0, 0, 0, 976, 0, 4085, 0, 19, 0, 1, 1304, 1937] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[3, 21, 3645, 0, 0, 0, 0, 1403, 20, 1190, 3848, 502, 0, 0, 118, 1, 0, 0, 0, 203, 0, 314, 100, 0, 0, 3402, 4036, 1, 0, 4, 37, 0, 0, 0, 1, 52, 1543, 3382, 0, 0, 0, 0, 0, 235, 1370, 0, 0, 0, 208, 0, 2352, 0, 0, 0, 0, 3744, 638, 34, 0, 0, 0, 0, 0, 361] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 3956, 0, 0, 0, 0, 0, 0, 3313, 0, 0, 0, 0, 0, 0, 1395, 143, 114, 2890, 5, 22, 0, 5, 1, 0, 0, 0, 0, 0, 0, 5, 1, 0, 2, 4095, 2990, 49, 8, 0, 0, 3, 0, 4082, 535, 0, 0, 0, 4, 4079, 0, 0, 0, 0, 3855, 0, 9, 432, 771, 0, 4, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[1, 0, 3, 3758, 0, 0, 0, 0, 0, 40, 3780, 62, 0, 0, 2204, 123, 0, 2201, 5, 3227, 0, 19, 32, 0, 10, 0, 3839, 1, 0, 922, 0, 218, 0, 4, 0, 0, 1111, 3945, 0, 0, 0, 1, 0, 1, 1684, 0, 6, 295, 0, 10, 3484, 0, 0, 0, 0, 1782, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 84, 514, 0, 0, 44, 10, 0, 3561, 0, 0, 0, 0, 888, 0, 2418, 74, 94, 4, 0, 0, 0, 0, 579, 0, 0, 3, 3808, 0, 6, 1, 0, 0, 3831, 3708, 2076, 1, 0, 29, 0, 1763, 0, 4052, 0, 0, 0, 0, 0, 2620, 14, 0, 0, 0, 2567, 0, 9, 4, 0, 0, 0, 1, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[267, 0, 4066, 3435, 0, 0, 0, 210, 0, 0, 3947, 0, 2, 0, 0, 0, 0, 0, 0, 169, 50, 5, 0, 0, 0, 1, 1975, 0, 0, 34, 71, 0, 0, 0, 0, 3920, 294, 2741, 0, 5, 0, 0, 0, 0, 1343, 0, 0, 0, 0, 8, 3915, 0, 0, 1294, 210, 4094, 0, 614, 0, 0, 7, 1, 0, 90] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 2, 0, 3995, 0, 0, 0, 7, 0, 69, 94, 22, 0, 0, 9, 374, 0, 4072, 0, 847, 0, 0, 0, 0, 0, 617, 0, 2, 0, 12, 3, 60, 0, 20, 0, 2, 4081, 4031, 0, 814, 0, 0, 0, 0, 2320, 0, 1, 658, 0, 0, 4096, 0, 3, 2692, 0, 156, 76, 2, 0, 0, 2, 0, 0, 3629] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 24, 20, 3, 0, 0, 0, 0, 0, 3990, 1, 0, 7, 7, 0, 0, 0, 246, 0, 87, 0, 2, 0, 3099, 3680, 0, 1087, 0, 0, 0, 28, 9, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4065, 0, 3, 1, 9, 0, 4096, 0, 0, 0, 0, 4086, 7, 52, 0, 0, 3, 0, 1, 4065] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[3, 21, 3645, 0, 0, 0, 0, 1403, 958, 0, 0, 0, 0, 0, 0, 0, 20, 1190, 3848, 502, 0, 0, 118, 1, 364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 203, 0, 314, 100, 0, 554, 457, 0, 0, 0, 0, 0, 0, 0, 3402, 4036, 1, 0, 4, 37, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 52, 1543, 673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 235, 1370, 0, 0, 0, 838, 0, 0, 0, 0, 0, 0, 0, 208, 0, 587, 0, 0, 0, 0, 3744, 0, 0, 0, 0, 0, 0, 0, 0, 638, 34, 0, 0, 0, 0, 0, 361, 1211, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6030, 6043, 1628, 7572, 2269, 2443, 4539, 2244] avg=4096.0000 max=7572 max_rank=3 min=1628 min_rank=2 imbalance_ratio=1.848633 max_load_reduction=25196 max_load_reduction_pct=0.768921 moved_to_echo=11495 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 12, 41, 0, 0, 665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1823, 0, 0, 0, 0, 253, 0, 0, 0, 0, 0, 0, 0, 0, 3864, 0, 0, 7, 11, 0, 0, 873, 317, 0, 0, 0, 0, 0, 0, 302, 557, 0, 0, 0, 3573, 96, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3985, 468, 0, 150, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 66, 3267, 632, 0, 0, 582, 0, 0, 0, 0, 0, 0, 0, 0, 1, 923, 0, 0, 0, 0, 976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 19, 0, 1, 1304, 1937, 1908, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[718, 2076, 5072, 4592, 4603, 4553, 1900, 9254] avg=4096.0000 max=9254 max_rank=7 min=718 min_rank=0 imbalance_ratio=2.259277 max_load_reduction=23514 max_load_reduction_pct=0.717590 moved_to_echo=20310 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 3, 3758, 0, 0, 0, 0, 1117, 0, 0, 0, 0, 0, 0, 0, 0, 40, 3780, 62, 0, 0, 2204, 123, 425, 0, 0, 0, 0, 0, 0, 0, 0, 2201, 5, 3227, 0, 19, 32, 0, 821, 533, 0, 0, 0, 0, 0, 0, 10, 0, 3839, 1, 0, 922, 0, 218, 107, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1111, 785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1684, 0, 6, 295, 978, 0, 0, 0, 0, 0, 0, 0, 0, 10, 868, 0, 0, 0, 0, 1782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1795, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4879, 6634, 6838, 5097, 1900, 2965, 2660, 1795] avg=4096.0000 max=6838 max_rank=2 min=1795 min_rank=7 imbalance_ratio=1.669434 max_load_reduction=25930 max_load_reduction_pct=0.791321 moved_to_echo=9320 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[267, 0, 4066, 3435, 0, 0, 0, 210, 776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3947, 0, 2, 0, 0, 0, 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 50, 5, 0, 0, 923, 370, 0, 0, 0, 0, 0, 0, 0, 1, 1975, 0, 0, 34, 71, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3920, 294, 547, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1343, 0, 0, 0, 679, 0, 0, 0, 0, 0, 0, 0, 0, 8, 975, 0, 0, 1294, 210, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 614, 0, 0, 7, 1, 0, 90, 2017, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8754, 4244, 1517, 2155, 4766, 2022, 6581, 2729] avg=4096.0000 max=8754 max_rank=0 min=1517 min_rank=2 imbalance_ratio=2.137207 max_load_reduction=24014 max_load_reduction_pct=0.732849 moved_to_echo=16098 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3956, 0, 0, 0, 0, 847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3313, 0, 0, 0, 0, 0, 322, 0, 0, 0, 0, 0, 0, 0, 0, 1395, 143, 114, 2890, 5, 22, 0, 962, 404, 0, 0, 0, 0, 0, 0, 5, 1, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 2, 4095, 595, 49, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4082, 535, 0, 0, 741, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1016, 0, 0, 0, 0, 3855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 432, 771, 0, 4, 0, 0, 2101, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4803, 3635, 5935, 87, 4755, 5361, 4875, 3317] avg=4096.0000 max=5935 max_rank=2 min=87 min_rank=3 imbalance_ratio=1.448975 max_load_reduction=26833 max_load_reduction_pct=0.818878 moved_to_echo=18308 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 84, 514, 0, 0, 44, 588, 0, 0, 0, 0, 0, 0, 0, 10, 0, 3561, 0, 0, 0, 0, 888, 224, 0, 0, 0, 0, 0, 0, 0, 0, 2418, 74, 94, 4, 0, 0, 0, 617, 280, 0, 0, 0, 0, 0, 0, 0, 579, 0, 0, 3, 3808, 0, 6, 56, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3831, 3708, 414, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 1763, 0, 4052, 0, 0, 0, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 654, 14, 0, 0, 0, 2567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4, 0, 0, 0, 1, 5, 1349, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1230, 4683, 3487, 4452, 7955, 6358, 3235, 1368] avg=4096.0000 max=7955 max_rank=4 min=1230 min_rank=0 imbalance_ratio=1.942139 max_load_reduction=24813 max_load_reduction_pct=0.757233 moved_to_echo=18916 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 3995, 0, 0, 0, 7, 1142, 0, 0, 0, 0, 0, 0, 0, 0, 69, 94, 22, 0, 0, 9, 374, 434, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 847, 0, 0, 0, 0, 966, 545, 0, 0, 0, 0, 0, 0, 0, 617, 0, 2, 0, 12, 3, 60, 110, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 2, 4081, 801, 0, 814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2320, 0, 1, 658, 999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 0, 3, 2692, 0, 156, 0, 0, 0, 0, 0, 0, 0, 0, 76, 2, 0, 0, 2, 0, 0, 3629, 2110, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5146, 1002, 6430, 804, 5718, 3978, 3871, 5819] avg=4096.0000 max=6430 max_rank=2 min=804 min_rank=3 imbalance_ratio=1.569824 max_load_reduction=26338 max_load_reduction_pct=0.803772 moved_to_echo=19386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 24, 20, 3, 0, 0, 0, 1161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3990, 1, 0, 7, 7, 0, 446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 0, 87, 0, 2, 0, 970, 556, 0, 0, 0, 0, 0, 0, 3099, 3680, 0, 1087, 0, 0, 0, 28, 115, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 3, 1, 1019, 0, 0, 0, 0, 0, 0, 0, 9, 0, 1012, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 7, 52, 0, 0, 3, 0, 1, 4065, 2114, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1208, 4451, 1861, 8009, 802, 5088, 5107, 6242] avg=4096.0000 max=8009 max_rank=3 min=802 min_rank=4 imbalance_ratio=1.955322 max_load_reduction=24759 max_load_reduction_pct=0.755585 moved_to_echo=17239 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[768, 4864, 512, 896, 128, 128, 1408, 10112, 14720, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 7808, 18944, 5120, 128, 1024, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[384, 128, 7808, 15360, 640, 128, 0, 1664, 7296, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3456, 8960, 9856, 1152, 128, 8448, 256, 384, 768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[256, 128, 7168, 128, 128, 4096, 256, 21376, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 13952, 512, 4736, 3072, 384, 256, 0, 6784, 3584, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 1792, 384, 22272, 1280, 128, 1024, 6400, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 1408, 22656, 2432, 128, 128, 2432, 1408, 2816, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 2, 0, 17, 0, 0, 0, 0, 0, 0, 1143, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 2860, 0, 2594, 3614, 0, 2769, 7, 1, 0, 3497, 0, 0, 0, 0, 4, 1, 273, 1, 3994, 21, 0, 302, 1, 568, 0, 1, 0, 0, 1, 1911, 6, 0, 1, 0, 4094, 0, 1025, 0, 3, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 5, 0, 35, 2, 0, 2759, 0, 0, 0, 339, 0, 531, 0, 0, 0, 0, 0, 4079, 29, 0, 1, 87, 0, 3859, 18, 1929, 0, 0, 3344, 0, 0, 0, 8, 925, 0, 0, 0, 0, 2741, 0, 0, 0, 39, 0, 0, 0, 0, 3673, 0, 0, 0, 0, 171, 3361, 140, 0, 0, 4085, 0, 604] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[222, 0, 1, 0, 0, 3842, 1488, 4, 0, 0, 3, 29, 0, 0, 0, 23, 1, 0, 0, 0, 0, 2655, 292, 0, 4085, 0, 0, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3778, 307, 0, 0, 562, 3320, 249, 0, 0, 0, 0, 0, 0, 2989, 2, 214, 1545, 3, 0, 0, 938, 4013, 0, 1, 2034, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[3, 40, 0, 0, 0, 0, 35, 2254, 0, 633, 0, 0, 0, 1, 0, 0, 484, 0, 0, 119, 19, 0, 0, 0, 219, 8, 3874, 3850, 0, 4032, 0, 0, 525, 0, 0, 0, 0, 1, 0, 0, 0, 256, 3047, 3819, 0, 0, 0, 0, 0, 0, 0, 1, 0, 11, 0, 0, 46, 105, 3388, 0, 1934, 4064, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[126, 4055, 11, 691, 0, 161, 459, 0, 34, 0, 18, 0, 5, 0, 1, 0, 756, 11, 8, 0, 49, 203, 0, 0, 2654, 0, 0, 3806, 0, 4035, 0, 191, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 20, 12, 260, 0, 0, 0, 0, 0, 0, 107, 0, 6, 1, 5, 1, 37, 3589, 3900, 0, 159, 3300, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[19, 2854, 0, 0, 0, 0, 386, 0, 0, 2, 0, 0, 40, 0, 186, 0, 0, 2, 0, 1580, 0, 3895, 0, 0, 0, 0, 350, 4057, 1, 3879, 68, 0, 4091, 0, 52, 0, 1795, 3225, 0, 0, 468, 0, 1906, 0, 3, 7, 0, 0, 0, 0, 28, 0, 0, 1, 0, 1, 9, 193, 257, 22, 0, 3391, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1297, 3627, 0, 0, 0, 33, 0, 1130, 0, 30, 0, 1, 0, 2360, 987, 0, 0, 0, 0, 969, 0, 0, 0, 0, 4, 0, 374, 346, 0, 2714, 0, 0, 1413, 0, 0, 0, 0, 4054, 0, 0, 0, 10, 2978, 10, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4061, 1, 0, 76, 446, 1978, 0, 0, 3863, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 693, 0, 0, 3, 0, 140, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 4, 235, 125, 0, 0, 0, 0, 4030, 1, 2, 0, 0, 4036, 0, 0, 0, 4096, 3513, 0, 0, 0, 0, 2564, 0, 296, 4, 0, 0, 10, 0, 4043, 1, 0, 0, 690, 0, 12, 0, 0, 0, 434, 3766, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[126, 4055, 11, 691, 0, 161, 459, 0, 2094, 23, 0, 0, 0, 0, 0, 0, 34, 0, 18, 0, 5, 0, 1, 0, 3141, 0, 0, 0, 0, 0, 0, 0, 756, 11, 8, 0, 49, 203, 0, 0, 1775, 0, 0, 0, 0, 0, 0, 0, 2654, 0, 0, 0, 0, 3204, 0, 191, 0, 0, 0, 0, 0, 0, 0, 0, 1758, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 12, 260, 0, 0, 0, 0, 831, 244, 0, 0, 0, 0, 0, 0, 0, 0, 107, 0, 6, 1, 5, 1, 1254, 665, 0, 0, 0, 0, 0, 0, 37, 3589, 3900, 0, 159, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7620, 3199, 2802, 6049, 1759, 1367, 2039, 7933] avg=4096.0000 max=7933 max_rank=7 min=1367 min_rank=5 imbalance_ratio=1.936768 max_load_reduction=24835 max_load_reduction_pct=0.757904 moved_to_echo=13098 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3, 40, 0, 0, 0, 0, 35, 2254, 268, 29, 0, 0, 0, 0, 0, 0, 0, 633, 0, 0, 0, 1, 0, 0, 3179, 0, 0, 0, 0, 0, 0, 0, 484, 0, 0, 119, 19, 0, 0, 0, 2186, 0, 0, 0, 0, 0, 0, 0, 219, 8, 3874, 0, 0, 3202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 226, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 3047, 3819, 0, 0, 0, 0, 830, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 11, 0, 0, 1544, 671, 0, 0, 0, 0, 0, 0, 46, 105, 3388, 0, 1934, 305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2629, 3813, 2808, 7303, 227, 7983, 2227, 5778] avg=4096.0000 max=7983 max_rank=5 min=227 min_rank=4 imbalance_ratio=1.948975 max_load_reduction=24785 max_load_reduction_pct=0.756378 moved_to_echo=16215 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 2, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1143, 4052, 2984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2860, 0, 2594, 0, 0, 2199, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3497, 0, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 273, 1, 3994, 21, 0, 302, 1, 568, 570, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1911, 6, 0, 0, 630, 0, 0, 0, 0, 0, 0, 1, 0, 4094, 0, 1025, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20, 8179, 0, 7661, 3502, 5730, 2549, 5127] avg=4096.0000 max=8179 max_rank=1 min=0 min_rank=2 imbalance_ratio=1.996826 max_load_reduction=24589 max_load_reduction_pct=0.750397 moved_to_echo=16908 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[222, 0, 1, 0, 0, 3842, 1488, 4, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 3, 29, 0, 0, 0, 23, 138, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2655, 292, 0, 1094, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3778, 307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 562, 3320, 249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2989, 2, 214, 1545, 3, 0, 773, 30, 0, 0, 0, 0, 0, 0, 0, 938, 4013, 0, 1, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5571, 193, 4042, 4085, 4085, 4131, 5556, 5105] avg=4096.0000 max=5571 max_rank=0 min=193 min_rank=1 imbalance_ratio=1.360107 max_load_reduction=27197 max_load_reduction_pct=0.829987 moved_to_echo=18877 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 5, 0, 35, 2, 1709, 29, 0, 0, 0, 0, 0, 0, 0, 2759, 0, 0, 0, 339, 0, 531, 3186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 29, 0, 2197, 0, 0, 0, 0, 0, 0, 0, 1, 87, 0, 0, 18, 1531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1436, 0, 0, 0, 8, 925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2741, 0, 0, 0, 39, 0, 398, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3673, 0, 0, 0, 0, 1552, 673, 0, 0, 0, 0, 0, 0, 171, 3361, 140, 0, 0, 307, 0, 604, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1784, 6815, 6305, 1637, 2369, 3377, 5898, 4583] avg=4096.0000 max=6815 max_rank=1 min=1637 min_rank=3 imbalance_ratio=1.663818 max_load_reduction=25953 max_load_reduction_pct=0.792023 moved_to_echo=16227 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[19, 2854, 0, 0, 0, 0, 386, 0, 2091, 24, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 40, 0, 186, 0, 3350, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1580, 0, 3895, 0, 0, 1824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 350, 0, 1, 3080, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1757, 0, 52, 0, 1795, 3225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 0, 1906, 0, 3, 7, 0, 0, 799, 243, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 1, 0, 1, 1289, 707, 0, 0, 0, 0, 0, 0, 9, 193, 257, 22, 0, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5374, 3578, 7301, 3499, 6829, 3426, 2026, 735] avg=4096.0000 max=7301 max_rank=2 min=735 min_rank=7 imbalance_ratio=1.782471 max_load_reduction=25467 max_load_reduction_pct=0.777191 moved_to_echo=13016 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1297, 3627, 0, 0, 0, 33, 0, 1130, 722, 28, 0, 0, 0, 0, 0, 0, 0, 30, 0, 1, 0, 2360, 987, 0, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 969, 0, 0, 0, 0, 2078, 0, 0, 0, 0, 0, 0, 0, 4, 0, 374, 0, 0, 2155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 607, 0, 0, 0, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 2978, 10, 0, 0, 4, 0, 559, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 1, 0, 1468, 61, 0, 0, 0, 0, 0, 0, 76, 446, 1978, 0, 0, 289, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6837, 3663, 3047, 2533, 4661, 3645, 5591, 2791] avg=4096.0000 max=6837 max_rank=0 min=2533 min_rank=3 imbalance_ratio=1.669189 max_load_reduction=25931 max_load_reduction_pct=0.791351 moved_to_echo=16688 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 693, 0, 0, 3, 0, 140, 0, 2066, 31, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 4, 235, 125, 0, 2030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1727, 0, 0, 0, 4096, 3513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2564, 0, 296, 4, 0, 0, 2, 243, 0, 0, 0, 0, 0, 0, 10, 0, 4043, 1, 0, 0, 690, 0, 1435, 703, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 434, 270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2933, 3328, 6463, 1, 9336, 3109, 6882, 716] avg=4096.0000 max=9336 max_rank=4 min=1 min_rank=3 imbalance_ratio=2.279297 max_load_reduction=23432 max_load_reduction_pct=0.715088 moved_to_echo=20043 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[1792, 11392, 128, 768, 128, 4096, 2560, 3456, 8960, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[9856, 128, 7296, 0, 128, 15488, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 3456, 128, 128, 128, 2816, 2432, 4608, 19712, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[384, 8704, 17792, 128, 3584, 1920, 128, 640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 7168, 3712, 256, 7552, 768, 128, 9344, 4224, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[768, 896, 20608, 4480, 384, 384, 128, 640, 4096, 1152, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[7552, 3584, 128, 0, 6016, 15616, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1280, 128, 128, 6784, 128, 11136, 512, 0, 13184, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[3, 0, 0, 8, 2941, 3186, 0, 255, 0, 0, 7, 4, 913, 0, 321, 0, 0, 1287, 1, 53, 0, 0, 32, 3287, 0, 0, 1, 0, 760, 101, 3799, 31, 5, 796, 3025, 0, 2189, 0, 15, 1, 0, 0, 91, 0, 5, 691, 0, 0, 3456, 633, 56, 0, 0, 0, 3716, 1091, 0, 2, 6, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 25, 3840, 0, 4071, 6, 3501, 33, 0, 0, 0, 1, 67, 0, 0, 0, 0, 1719, 0, 0, 0, 0, 0, 482, 7, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3448, 1832, 2022, 0, 16, 2491, 0, 0, 0, 0, 28, 74, 0, 0, 2547, 0, 0, 0, 68, 260, 0, 1516, 0, 354, 0, 0, 5, 1, 0, 258] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 3605, 0, 0, 588, 2006, 0, 0, 0, 10, 0, 0, 0, 0, 0, 15, 4076, 0, 0, 11, 147, 0, 20, 4, 0, 0, 0, 0, 0, 4095, 1, 0, 4062, 0, 0, 4030, 0, 34, 65, 0, 0, 0, 0, 0, 1, 0, 0, 3801, 0, 0, 0, 0, 0, 39, 1, 213, 0, 0, 501, 4092, 13, 1338, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 3910, 813, 0, 3555, 4019, 8, 0, 275, 0, 0, 0, 0, 55, 0, 55, 3334, 0, 2, 0, 0, 0, 75, 0, 13, 0, 0, 296, 0, 3874, 0, 5, 0, 3313, 364, 67, 230, 5, 153, 0, 0, 6, 0, 131, 10, 0, 0, 0, 482, 0, 0, 0, 0, 0, 0, 4093, 0, 299, 0, 2, 3324, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 2, 0, 65, 0, 3504, 0, 0, 0, 0, 7, 0, 0, 2, 0, 0, 1, 4095, 0, 951, 0, 2, 0, 0, 3955, 0, 0, 0, 0, 0, 840, 4, 0, 3, 4051, 0, 4096, 0, 0, 7, 0, 0, 0, 0, 0, 1, 0, 0, 4039, 0, 0, 178, 0, 0, 0, 6, 0, 0, 0, 0, 2695, 759, 3505, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[2, 0, 4045, 3224, 0, 1775, 170, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 3591, 0, 0, 62, 0, 0, 0, 0, 0, 0, 15, 0, 0, 140, 0, 783, 607, 4085, 0, 4089, 0, 22, 0, 12, 0, 0, 0, 242, 0, 0, 1, 4094, 0, 0, 0, 2, 0, 0, 2, 2311, 1, 10, 168, 2799, 471, 16, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 5, 1421, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4051, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 57, 2, 0, 4088, 0, 0, 9, 4078, 0, 4052, 0, 48, 0, 0, 0, 0, 0, 6, 0, 36, 1, 4084, 0, 0, 0, 0, 0, 3398, 0, 0, 0, 150, 0, 3176, 0, 0, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8, 3957, 579, 758, 0, 0, 0, 1, 0, 0, 0, 4, 0, 4040, 0, 0, 37, 0, 51, 3197, 0, 0, 0, 0, 486, 23, 506, 46, 0, 0, 3503, 0, 3281, 0, 0, 0, 0, 0, 0, 14, 204, 0, 0, 0, 3049, 1362, 0, 142, 10, 0, 3947, 0, 25, 0, 0, 0, 3532, 5, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 8, 2941, 0, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4, 913, 0, 321, 0, 2088, 1269, 0, 218, 0, 0, 0, 0, 0, 993, 1, 53, 0, 0, 32, 3287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 760, 101, 3799, 31, 239, 76, 0, 0, 0, 0, 0, 0, 5, 796, 0, 0, 1950, 0, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 5, 691, 0, 0, 3025, 1098, 0, 0, 0, 0, 0, 0, 2187, 633, 56, 0, 0, 0, 3716, 1091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3207, 4820, 4366, 5007, 2767, 4910, 7683, 8] avg=4096.0000 max=7683 max_rank=6 min=8 min_rank=7 imbalance_ratio=1.875732 max_load_reduction=25085 max_load_reduction_pct=0.765533 moved_to_echo=15368 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 25, 2209, 0, 4071, 0, 3501, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 67, 0, 0, 0, 3, 935, 1631, 291, 0, 0, 0, 0, 0, 1327, 0, 0, 0, 0, 0, 482, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4096, 0, 0, 221, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1832, 1801, 0, 16, 2491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 74, 0, 0, 3448, 3, 0, 0, 0, 0, 0, 0, 1612, 0, 0, 0, 68, 260, 0, 1516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 354, 0, 0, 5, 1, 0, 258, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9839, 2928, 1809, 4425, 6140, 3553, 3456, 618] avg=4096.0000 max=9839 max_rank=0 min=618 min_rank=7 imbalance_ratio=2.402100 max_load_reduction=22929 max_load_reduction_pct=0.699738 moved_to_echo=13767 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 2327, 3224, 0, 0, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 1164, 1504, 1718, 609, 0, 0, 0, 0, 0, 2770, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 140, 0, 448, 212, 234, 0, 0, 0, 0, 0, 783, 607, 0, 0, 3641, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 242, 0, 0, 1, 4085, 611, 0, 0, 0, 0, 0, 0, 2590, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2311, 1, 10, 168, 2565, 471, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5723, 5024, 2832, 1049, 5053, 4951, 2594, 5542] avg=4096.0000 max=5723 max_rank=0 min=1049 min_rank=3 imbalance_ratio=1.397217 max_load_reduction=27045 max_load_reduction_pct=0.825348 moved_to_echo=18140 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2071, 0, 0, 0, 2006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 386, 1396, 1534, 691, 0, 0, 0, 0, 15, 3144, 0, 0, 11, 147, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4095, 1, 441, 241, 342, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 3589, 0, 34, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 202, 0, 0, 0, 0, 0, 0, 2405, 0, 0, 0, 0, 0, 39, 1, 0, 0, 0, 0, 0, 0, 0, 0, 213, 0, 0, 501, 3750, 13, 1338, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4017, 3337, 5124, 7750, 203, 2445, 5815] avg=4096.0000 max=7750 max_rank=4 min=203 min_rank=5 imbalance_ratio=1.892090 max_load_reduction=25018 max_load_reduction_pct=0.763489 moved_to_echo=16213 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2249, 813, 0, 0, 4019, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 275, 0, 0, 0, 0, 55, 0, 2331, 0, 1661, 565, 0, 0, 0, 0, 55, 2572, 0, 2, 0, 0, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 296, 0, 3874, 0, 7, 197, 0, 0, 0, 0, 0, 0, 5, 0, 0, 364, 60, 230, 5, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 131, 10, 0, 0, 3313, 1224, 0, 0, 0, 0, 0, 0, 0, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 299, 0, 2, 3324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7089, 4887, 2704, 4387, 817, 4684, 482, 7718] avg=4096.0000 max=7718 max_rank=7 min=482 min_rank=6 imbalance_ratio=1.884277 max_load_reduction=25050 max_load_reduction_pct=0.764465 moved_to_echo=13701 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 3, 1500, 0, 687, 0, 0, 0, 0, 0, 3124, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 2, 0, 4088, 0, 444, 240, 265, 0, 0, 0, 0, 0, 0, 9, 0, 0, 3608, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 36, 1, 4078, 2, 0, 0, 0, 0, 0, 0, 2584, 0, 0, 0, 0, 0, 3398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 0, 2911, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1421, 2199, 7213, 5096, 3665, 4123, 5982, 3069] avg=4096.0000 max=7213 max_rank=2 min=1421 min_rank=0 imbalance_ratio=1.760986 max_load_reduction=25555 max_load_reduction_pct=0.779877 moved_to_echo=16839 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8, 0, 579, 758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 2593, 1120, 0, 685, 0, 0, 0, 0, 0, 3116, 0, 0, 37, 0, 51, 3197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 23, 506, 46, 359, 239, 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 204, 0, 0, 0, 3503, 1364, 0, 0, 0, 0, 0, 0, 1929, 1362, 0, 142, 10, 0, 3947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 3237, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1345, 4403, 6401, 1954, 2922, 5085, 7390, 3268] avg=4096.0000 max=7390 max_rank=6 min=1345 min_rank=0 imbalance_ratio=1.804199 max_load_reduction=25378 max_load_reduction_pct=0.774475 moved_to_echo=18665 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 2, 0, 0, 2297, 1487, 0, 697, 0, 0, 0, 0, 1, 3152, 0, 951, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3955, 0, 0, 0, 0, 0, 840, 4, 452, 246, 229, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3644, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4051, 1207, 0, 0, 0, 0, 0, 0, 2552, 0, 0, 178, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2466, 759, 3505, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[67, 4490, 4106, 5726, 3654, 5259, 2736, 6730] avg=4096.0000 max=6730 max_rank=7 min=67 min_rank=0 imbalance_ratio=1.643066 max_load_reduction=26038 max_load_reduction_pct=0.794617 moved_to_echo=18379 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[0, 384, 128, 128, 1024, 128, 512, 128, 10880, 9216, 6656, 4480, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 8960, 4224, 7040, 0, 11776, 1152, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[15872, 2560, 128, 384, 128, 384, 11136, 2688, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[6656, 384, 512, 768, 14976, 4608, 4864, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[896, 5504, 0, 2304, 21248, 256, 256, 2816, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[3968, 128, 128, 128, 1664, 4224, 17408, 128, 2688, 1664, 1408, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 20224, 128, 1024, 128, 256, 128, 11264, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 640, 896, 128, 128, 25600, 5760, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4, 4051, 2231, 83, 330, 0, 0, 0, 0, 0, 4, 117, 3282, 0, 0, 61, 0, 0, 0, 0, 0, 3, 2, 0, 0, 4096, 627, 0, 0, 0, 3958, 0, 3676, 0, 0, 0, 3941, 1, 0, 0, 32, 0, 0, 4, 0, 0, 217, 439, 0, 0, 0, 0, 1, 9, 0, 0, 4014, 30, 1554, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 82, 50, 810, 0, 0, 0, 0, 1, 95, 908, 2, 0, 3599, 0, 1, 0, 3463, 0, 4, 30, 0, 0, 0, 0, 0, 0, 0, 24, 3012, 0, 1, 0, 4096, 0, 0, 0, 4013, 0, 0, 0, 0, 0, 0, 1412, 2328, 631, 0, 0, 0, 0, 18, 4090, 0, 0, 0, 0, 4045, 0, 51, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 7, 0, 4034, 3682, 0, 13, 474, 0, 3, 0, 4036, 0, 0, 0, 0, 14, 0, 35, 0, 0, 0, 0, 0, 0, 0, 1, 3884, 1078, 0, 0, 194, 100, 0, 12, 4074, 0, 8, 17, 4079, 0, 0, 20, 0, 0, 0, 3921, 0, 295, 0, 2786, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 8, 13, 1220, 3355, 0, 37, 3676, 0, 15, 0, 0, 2, 0, 0, 0, 0, 215, 0, 0, 6, 93, 0, 9, 116, 0, 0, 0, 0, 36, 0, 0, 26, 335, 3957, 0, 3428, 0, 2048, 2590, 4096, 0, 0, 0, 0, 0, 0, 129, 0, 149, 0, 68, 0, 0, 0, 3071, 0, 0, 0, 0, 4070, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 50, 1, 0, 244, 2162, 0, 4094, 0, 87, 0, 4096, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 246, 0, 0, 0, 0, 4045, 0, 0, 0, 1002, 14, 0, 2910, 0, 0, 0, 3929, 3083, 0, 0, 30, 0, 0, 83, 110, 3377, 0, 1, 24, 145, 0, 0, 12, 26, 10, 7, 462, 2514, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3415, 3891, 2681, 1, 889, 0, 0, 0, 218, 0, 0, 0, 868, 9, 59, 0, 0, 1340, 552, 0, 5, 1, 0, 0, 51, 0, 52, 1, 0, 3264, 4078, 48, 364, 0, 0, 0, 4072, 0, 1049, 0, 0, 5, 0, 0, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2740, 9, 0, 2938] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 1, 0, 1372, 3907, 941, 4079, 35, 0, 0, 0, 11, 3802, 0, 0, 0, 4091, 0, 4, 0, 0, 0, 10, 11, 0, 0, 635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2794, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 2846, 7, 1, 0, 0, 0, 0, 46, 6, 0, 0, 0, 0, 4079, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 1119, 4079, 4040, 0, 119, 0, 0, 4, 0, 42, 2, 4089, 0, 46, 0, 184, 0, 4049, 0, 0, 0, 37, 0, 1, 0, 0, 3195, 0, 0, 0, 1, 0, 2482, 1475, 58, 0, 0, 92, 0, 0, 829, 0, 0, 0, 3961, 0, 2557, 0, 0, 0, 0, 0, 0, 40, 232, 0, 0, 0, 28, 0, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1119, 0, 3120, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 42, 2, 4089, 0, 30, 0, 0, 0, 0, 0, 0, 0, 46, 0, 184, 0, 4049, 0, 0, 0, 598, 4, 0, 0, 0, 0, 0, 0, 37, 0, 1, 0, 0, 3195, 0, 0, 3161, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2482, 361, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 829, 0, 0, 0, 3961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2557, 0, 0, 0, 0, 0, 0, 516, 918, 0, 0, 0, 0, 0, 0, 40, 232, 0, 0, 0, 28, 0, 7, 920, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4358, 4167, 4881, 6394, 2902, 4842, 3991, 1233] avg=4096.0000 max=6394 max_rank=3 min=1233 min_rank=7 imbalance_ratio=1.561035 max_load_reduction=26374 max_load_reduction_pct=0.804871 moved_to_echo=12968 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 13, 1220, 0, 0, 37, 3676, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 2, 0, 0, 0, 1366, 0, 0, 0, 0, 0, 0, 0, 0, 215, 0, 0, 6, 93, 0, 9, 1391, 211, 0, 0, 0, 0, 0, 0, 116, 0, 0, 0, 0, 36, 0, 0, 2600, 0, 0, 0, 0, 0, 0, 0, 26, 335, 3957, 0, 837, 0, 2048, 2590, 0, 0, 0, 0, 0, 0, 0, 0, 2225, 0, 0, 0, 0, 0, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 0, 68, 0, 0, 0, 3071, 1200, 755, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 294, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4954, 1383, 1925, 2752, 9793, 2354, 5243, 4364] avg=4096.0000 max=9793 max_rank=4 min=1383 min_rank=1 imbalance_ratio=2.390869 max_load_reduction=22975 max_load_reduction_pct=0.701141 moved_to_echo=21754 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 3128, 2231, 83, 0, 0, 0, 0, 0, 0, 0, 0, 330, 0, 0, 0, 0, 0, 4, 117, 1315, 0, 0, 0, 0, 0, 0, 0, 3282, 0, 0, 61, 0, 0, 0, 0, 1492, 203, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 4096, 627, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3958, 0, 897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2140, 1, 0, 0, 32, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 217, 439, 0, 0, 0, 0, 1287, 1, 0, 0, 0, 0, 0, 0, 1, 9, 0, 0, 4014, 30, 1554, 0, 923, 283, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5443, 1766, 5038, 4731, 4855, 2177, 1944, 6814] avg=4096.0000 max=6814 max_rank=7 min=1766 min_rank=1 imbalance_ratio=1.663574 max_load_reduction=25954 max_load_reduction_pct=0.792053 moved_to_echo=15790 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 7, 0, 0, 2843, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 474, 0, 3, 0, 4036, 0, 0, 0, 1361, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 35, 0, 0, 0, 0, 1654, 211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3884, 1078, 0, 0, 3127, 0, 0, 0, 0, 0, 0, 0, 194, 100, 0, 12, 994, 0, 8, 17, 0, 0, 0, 0, 0, 0, 0, 0, 2214, 0, 0, 20, 0, 0, 0, 3921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 295, 0, 2786, 0, 0, 0, 0, 1426, 907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 839, 293, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2863, 5874, 1914, 8090, 1325, 6155, 5414, 1133] avg=4096.0000 max=8090 max_rank=3 min=1133 min_rank=7 imbalance_ratio=1.975098 max_load_reduction=24678 max_load_reduction_pct=0.753113 moved_to_echo=14027 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3005, 2681, 1, 0, 0, 0, 0, 0, 0, 0, 0, 889, 0, 0, 0, 218, 0, 0, 0, 1358, 0, 0, 0, 0, 0, 0, 0, 868, 9, 59, 0, 0, 1340, 552, 0, 147, 210, 0, 0, 0, 0, 0, 0, 5, 1, 0, 0, 51, 0, 52, 1, 2648, 0, 0, 0, 0, 0, 0, 0, 0, 3264, 4078, 48, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2212, 0, 1049, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 0, 0, 0, 0, 0, 0, 0, 127, 767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2740, 9, 0, 2938, 886, 292, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5687, 2465, 3185, 2758, 7480, 3266, 1062, 6865] avg=4096.0000 max=7480 max_rank=4 min=1062 min_rank=6 imbalance_ratio=1.826172 max_load_reduction=25288 max_load_reduction_pct=0.771729 moved_to_echo=18673 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 50, 1, 0, 189, 2162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 87, 0, 4096, 0, 0, 0, 1311, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 1181, 203, 0, 0, 0, 0, 0, 0, 246, 0, 0, 0, 0, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 14, 0, 710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2133, 3083, 0, 0, 30, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 110, 3377, 0, 1, 24, 145, 0, 0, 1019, 0, 0, 0, 0, 0, 0, 0, 12, 26, 10, 7, 462, 2514, 0, 0, 55, 282, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2402, 9588, 1388, 4291, 1726, 5329, 4676, 3368] avg=4096.0000 max=9588 max_rank=1 min=1388 min_rank=2 imbalance_ratio=2.340820 max_load_reduction=23180 max_load_reduction_pct=0.707397 moved_to_echo=15099 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 1372, 0, 727, 4079, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 3802, 0, 0, 0, 1364, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4, 0, 0, 0, 10, 11, 1134, 211, 0, 0, 0, 0, 0, 0, 0, 0, 635, 0, 0, 0, 0, 0, 3029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 682, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2221, 0, 0, 0, 0, 0, 0, 2846, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1, 0, 0, 0, 0, 46, 6, 978, 878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 214, 294, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6214, 5177, 5461, 3664, 682, 5067, 1916, 4587] avg=4096.0000 max=6214 max_rank=0 min=682 min_rank=4 imbalance_ratio=1.517090 max_load_reduction=26554 max_load_reduction_pct=0.810364 moved_to_echo=12252 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 0, 35, 810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 95, 908, 2, 0, 1342, 0, 0, 0, 0, 0, 0, 0, 3599, 0, 1, 0, 3463, 0, 4, 30, 1667, 212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 64, 0, 0, 0, 0, 0, 0, 0, 3012, 0, 1, 0, 992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2166, 0, 0, 0, 0, 0, 0, 1412, 0, 0, 0, 0, 0, 0, 0, 0, 2328, 631, 0, 0, 0, 0, 18, 4090, 1437, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 51, 0, 15, 293, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[847, 2348, 8976, 88, 4005, 3578, 8522, 4404] avg=4096.0000 max=8976 max_rank=2 min=88 min_rank=3 imbalance_ratio=2.191406 max_load_reduction=23792 max_load_reduction_pct=0.726074 moved_to_echo=20509 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[2688, 7040, 256, 3328, 128, 256, 128, 7168, 8064, 4352, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 3840, 0, 13056, 12032, 3968, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[5888, 128, 128, 128, 12416, 1024, 4096, 128, 9472, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3328, 4736, 12032, 2560, 5632, 128, 2176, 2688, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[11904, 256, 256, 128, 7552, 1536, 640, 128, 9344, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 384, 128, 128, 19456, 2688, 1664, 3072, 3968, 2048, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[15488, 3200, 1152, 896, 128, 128, 0, 12416, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[512, 128, 640, 128, 3968, 12544, 768, 128, 14720, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 1, 1188, 0, 457, 0, 3361, 0, 0, 542, 33, 0, 0, 0, 1, 0, 7, 0, 3400, 2613, 0, 0, 0, 0, 3084, 0, 168, 0, 0, 0, 2391, 0, 2499, 0, 0, 0, 0, 3912, 21, 0, 0, 5, 0, 1452, 289, 0, 0, 0, 23, 0, 0, 0, 3318, 4002, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[2995, 0, 4037, 0, 4, 2, 0, 27, 308, 4088, 45, 0, 0, 0, 6, 0, 1953, 105, 0, 0, 0, 0, 0, 0, 1218, 366, 3660, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 1402, 0, 7, 111, 0, 0, 11, 3612, 0, 0, 79, 3717, 3347, 0, 2, 27, 0, 281, 930, 56, 111, 231, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[78, 0, 0, 0, 0, 0, 0, 1242, 0, 0, 88, 0, 4096, 253, 0, 0, 0, 0, 0, 7, 0, 1525, 0, 0, 3, 0, 253, 0, 0, 43, 30, 282, 0, 0, 0, 2, 4096, 2602, 0, 0, 0, 1, 0, 1201, 0, 10, 315, 0, 0, 3944, 0, 0, 0, 0, 4075, 0, 0, 1545, 3097, 3980, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 1991, 0, 0, 0, 1, 0, 0, 1337, 0, 355, 0, 693, 0, 0, 0, 0, 0, 50, 0, 0, 4095, 0, 0, 3639, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 1832, 0, 1, 0, 0, 0, 276, 3738, 0, 353, 677, 365, 130, 301, 0, 0, 0, 1338, 14, 3412, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[3076, 0, 0, 0, 0, 0, 1, 162, 0, 4037, 0, 1289, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 1695, 3211, 624, 1, 89, 0, 4089, 0, 0, 3854, 0, 0, 0, 104, 3, 1188, 292, 0, 70, 70, 0, 0, 0, 70, 4096, 0, 0, 0, 0, 0, 0, 581, 0, 47, 4095, 0, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[6, 1483, 180, 0, 0, 0, 19, 0, 0, 0, 725, 0, 163, 666, 0, 4084, 2, 0, 0, 77, 0, 241, 0, 163, 3499, 0, 3822, 0, 11, 0, 4027, 1, 1340, 5, 434, 5, 574, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 216, 0, 3069, 254, 0, 0, 0, 12, 0, 0, 0, 0, 0, 3612, 3, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[223, 74, 25, 1616, 967, 0, 450, 998, 0, 0, 440, 0, 2, 0, 0, 0, 174, 0, 0, 2, 0, 4089, 1, 24, 3584, 0, 4066, 75, 57, 0, 0, 0, 0, 4095, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 321, 0, 4001, 0, 436, 0, 0, 0, 0, 0, 0, 2914, 3, 4096, 9, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 548, 216, 0, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 3840, 0, 0, 4066, 0, 1114, 0, 0, 0, 2, 0, 3, 0, 0, 656, 1017, 4062, 0, 862, 0, 0, 4, 4095, 3997, 0, 0, 2035, 0, 0, 137, 1, 0, 0, 0, 0, 405, 0, 1760, 3539, 8, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[78, 0, 0, 0, 0, 0, 0, 1242, 1445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 4096, 253, 0, 0, 1437, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 1525, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 0, 0, 43, 30, 282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2575, 2602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1201, 0, 10, 315, 0, 41, 84, 13, 0, 0, 0, 0, 0, 0, 3931, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1545, 1596, 3980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2765, 5930, 1535, 567, 5179, 1665, 8006, 7121] avg=4096.0000 max=8006 max_rank=6 min=567 min_rank=3 imbalance_ratio=1.954590 max_load_reduction=24762 max_load_reduction_pct=0.755676 moved_to_echo=21971 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 1, 1548, 774, 0, 0, 0, 0, 0, 0, 1188, 0, 457, 0, 3361, 0, 0, 542, 839, 60, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 1, 0, 7, 0, 2626, 0, 0, 0, 0, 0, 0, 0, 0, 2613, 0, 0, 0, 0, 3084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 0, 0, 0, 1503, 0, 2499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3912, 21, 0, 0, 5, 0, 49, 4, 0, 0, 0, 0, 0, 0, 1448, 289, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1710, 4002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2324, 6447, 2667, 5697, 4170, 3991, 1760, 5712] avg=4096.0000 max=6447 max_rank=1 min=1760 min_rank=6 imbalance_ratio=1.573975 max_load_reduction=26321 max_load_reduction_pct=0.803253 moved_to_echo=15633 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2995, 0, 4037, 0, 4, 2, 0, 27, 107, 278, 0, 0, 0, 0, 0, 0, 308, 4088, 45, 0, 0, 0, 6, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1953, 105, 0, 0, 0, 0, 0, 0, 940, 0, 0, 0, 0, 0, 0, 0, 0, 366, 3064, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1402, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 111, 0, 0, 11, 3612, 0, 0, 79, 596, 0, 11, 0, 0, 0, 0, 0, 3717, 3336, 0, 2, 27, 0, 281, 930, 0, 0, 0, 0, 0, 0, 0, 0, 56, 111, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7450, 4451, 2998, 3460, 1409, 4420, 8293, 287] avg=4096.0000 max=8293 max_rank=6 min=287 min_rank=7 imbalance_ratio=2.024658 max_load_reduction=24475 max_load_reduction_pct=0.746918 moved_to_echo=14409 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1991, 0, 0, 0, 1, 0, 0, 0, 829, 0, 0, 0, 0, 0, 0, 1337, 0, 355, 0, 693, 0, 0, 0, 1430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 4095, 0, 0, 2810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2562, 1832, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 276, 3738, 0, 353, 677, 667, 84, 0, 0, 0, 0, 0, 0, 365, 130, 301, 0, 0, 0, 1338, 14, 0, 0, 0, 0, 0, 0, 0, 0, 3412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2821, 3815, 6955, 3427, 4395, 5795, 2148, 3412] avg=4096.0000 max=6955 max_rank=2 min=2148 min_rank=6 imbalance_ratio=1.697998 max_load_reduction=25813 max_load_reduction_pct=0.787750 moved_to_echo=15750 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 548, 216, 0, 341, 821, 926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 356, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 3840, 0, 0, 3140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 933, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 656, 640, 4062, 0, 862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4095, 3997, 0, 0, 2035, 181, 21, 0, 0, 0, 0, 0, 0, 0, 0, 137, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 405, 0, 908, 3539, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2852, 387, 7040, 935, 6223, 10333, 138, 4860] avg=4096.0000 max=10333 max_rank=5 min=138 min_rank=6 imbalance_ratio=2.522705 max_load_reduction=22435 max_load_reduction_pct=0.684662 moved_to_echo=21554 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[6, 1483, 180, 0, 0, 0, 19, 0, 0, 798, 0, 0, 0, 0, 0, 0, 0, 0, 725, 0, 163, 666, 0, 4084, 203, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 77, 0, 241, 0, 163, 2701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3199, 0, 11, 0, 4027, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1340, 5, 434, 5, 358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 216, 623, 13, 10, 0, 0, 0, 0, 0, 0, 3059, 254, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3612, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2486, 5841, 3184, 7238, 2142, 4937, 3325, 3615] avg=4096.0000 max=7238 max_rank=3 min=2142 min_rank=4 imbalance_ratio=1.767090 max_load_reduction=25530 max_load_reduction_pct=0.779114 moved_to_echo=14019 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3076, 0, 0, 0, 0, 0, 1, 162, 1911, 386, 0, 0, 0, 0, 0, 0, 0, 4037, 0, 1289, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 1309, 0, 0, 0, 0, 0, 0, 0, 0, 3211, 523, 1, 89, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3854, 0, 0, 0, 104, 3, 1188, 0, 0, 0, 0, 0, 0, 0, 0, 292, 0, 70, 70, 0, 0, 0, 70, 101, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 2110, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5536, 5400, 1332, 7913, 5149, 603, 4677, 2158] avg=4096.0000 max=7913 max_rank=3 min=603 min_rank=5 imbalance_ratio=1.931885 max_load_reduction=24855 max_load_reduction_pct=0.758514 moved_to_echo=12587 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[223, 74, 25, 1616, 967, 0, 450, 998, 1364, 817, 0, 0, 0, 0, 0, 0, 0, 0, 440, 0, 2, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 174, 0, 0, 2, 0, 4089, 1, 24, 2767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3399, 75, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 321, 667, 0, 16, 0, 0, 0, 0, 0, 0, 3985, 0, 436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1495, 3, 4096, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6534, 497, 7057, 3531, 4101, 1024, 4421, 5603] avg=4096.0000 max=7057 max_rank=2 min=497 min_rank=1 imbalance_ratio=1.722900 max_load_reduction=25711 max_load_reduction_pct=0.784637 moved_to_echo=15149 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[512, 128, 128, 13696, 11392, 128, 768, 3456, 2944, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[6400, 3584, 4352, 1664, 1536, 256, 512, 2816, 7296, 4864, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1536, 8064, 512, 768, 7680, 10112, 2560, 2176, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 16000, 1024, 512, 128, 0, 5760, 1536, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[2176, 128, 128, 256, 128, 13824, 128, 256, 16384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[3968, 1792, 8064, 11648, 7808, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 6272, 14848, 128, 256, 128, 11264, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[2944, 8192, 2176, 1408, 8320, 1024, 128, 4736, 4352, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 9, 0, 0, 0, 159, 45, 4034, 0, 0, 1, 0, 0, 0, 42, 0, 0, 3799, 0, 0, 0, 3969, 4, 3622, 279, 0, 216, 4079, 0, 0, 0, 153, 0, 0, 0, 0, 3714, 0, 6, 0, 0, 3963, 0, 476, 0, 0, 0, 0, 0, 0, 0, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[29, 0, 0, 3728, 0, 7, 0, 6, 14, 0, 1305, 30, 0, 293, 0, 0, 0, 1537, 0, 18, 3180, 0, 1093, 0, 1078, 2605, 2899, 0, 0, 0, 3657, 0, 0, 0, 0, 6, 0, 0, 0, 487, 0, 0, 0, 0, 1059, 1418, 37, 0, 3899, 145, 0, 0, 86, 0, 6, 50, 0, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[82, 0, 307, 0, 2465, 0, 0, 0, 0, 0, 0, 0, 264, 531, 1652, 0, 1, 0, 0, 0, 3412, 0, 16, 0, 21, 0, 665, 0, 26, 0, 0, 0, 0, 0, 2653, 0, 0, 0, 2954, 3673, 0, 56, 572, 0, 1376, 0, 1340, 0, 0, 0, 0, 0, 0, 1713, 0, 21, 0, 723, 4070, 0, 3166, 0, 1009, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[10, 1, 2, 2994, 182, 0, 544, 48, 284, 1135, 1, 0, 0, 6, 0, 0, 1, 0, 0, 4008, 0, 0, 0, 0, 237, 0, 3897, 0, 0, 0, 3771, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 3918, 0, 0, 6, 0, 796, 1, 0, 0, 0, 2678, 13, 0, 17, 30, 0, 0, 0, 10, 0, 3, 4091, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 26, 277, 3808, 674, 283, 0, 0, 0, 0, 1521, 31, 0, 6, 0, 0, 144, 1, 0, 0, 3809, 0, 0, 11, 115, 0, 4023, 0, 0, 2343, 3352, 455, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 4083, 0, 0, 1, 0, 68, 0, 3530, 0, 0, 0, 106, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 3185, 3375, 0, 0, 145, 1, 2620, 1, 42, 505, 0, 0, 2, 3, 3799, 12, 0, 443, 0, 0, 281, 4060, 4086, 54, 0, 0, 2240, 0, 458, 0, 0, 0, 108, 0, 0, 0, 0, 0, 2, 8, 0, 582, 0, 0, 4077, 0, 0, 0, 1900, 0, 0, 0, 15, 1, 0, 719, 0, 0, 44, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 266, 14, 4048, 0, 1354, 0, 0, 0, 0, 0, 5, 0, 36, 3400, 0, 45, 700, 0, 58, 4096, 0, 1912, 677, 0, 0, 845, 0, 63, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 8, 764, 0, 8, 3693, 0, 73, 0, 3519, 0, 126, 2682, 268, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[4056, 2029, 0, 265, 3653, 23, 0, 0, 0, 0, 122, 0, 0, 0, 476, 0, 0, 5, 0, 3728, 34, 0, 197, 0, 0, 0, 4096, 1, 0, 3, 4092, 3879, 0, 166, 0, 669, 0, 0, 0, 3864, 0, 91, 0, 13, 0, 0, 34, 7, 839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 426, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2389, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 159, 45, 4034, 0, 0, 1, 0, 0, 1838, 1028, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 2931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3969, 4, 0, 279, 0, 216, 1428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 0, 0, 0, 0, 3714, 414, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 3963, 0, 476, 0, 454, 367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 1340, 813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2594, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2398, 7105, 2973, 5896, 4281, 5266, 2255, 2594] avg=4096.0000 max=7105 max_rank=1 min=2255 min_rank=6 imbalance_ratio=1.734619 max_load_reduction=25663 max_load_reduction_pct=0.783173 moved_to_echo=14396 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[10, 1, 2, 1747, 182, 0, 544, 48, 0, 0, 0, 0, 0, 0, 0, 0, 284, 1135, 1, 0, 0, 6, 0, 0, 1699, 1107, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 237, 0, 0, 0, 0, 0, 1320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3918, 0, 0, 6, 0, 796, 1, 0, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2678, 13, 0, 17, 30, 979, 752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 3, 4091, 8, 2790, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2534, 4232, 4009, 1557, 4076, 4989, 4469, 6902] avg=4096.0000 max=6902 max_rank=7 min=1557 min_rank=3 imbalance_ratio=1.685059 max_load_reduction=25866 max_load_reduction_pct=0.789368 moved_to_echo=20436 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[82, 0, 307, 0, 2465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 264, 531, 1652, 0, 0, 189, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2633, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2653, 0, 0, 0, 2954, 3673, 371, 0, 0, 0, 0, 0, 0, 0, 0, 56, 572, 0, 1376, 0, 1340, 0, 408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1713, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 723, 4070, 0, 3166, 0, 1009, 0, 476, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2854, 2636, 2650, 47, 9651, 3752, 1734, 9444] avg=4096.0000 max=9651 max_rank=4 min=47 min_rank=3 imbalance_ratio=2.356201 max_load_reduction=23117 max_load_reduction_pct=0.705475 moved_to_echo=24581 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[29, 0, 0, 2175, 0, 7, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 1305, 30, 0, 293, 0, 0, 1648, 822, 0, 0, 0, 0, 0, 0, 0, 1537, 0, 18, 2454, 0, 1093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1078, 2605, 0, 0, 0, 0, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 487, 346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1418, 37, 0, 380, 334, 0, 0, 0, 0, 0, 0, 3899, 145, 0, 0, 86, 0, 6, 50, 1219, 729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2077, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2217, 4112, 5102, 4963, 839, 3228, 6134, 6173] avg=4096.0000 max=6173 max_rank=7 min=839 min_rank=4 imbalance_ratio=1.507080 max_load_reduction=26595 max_load_reduction_pct=0.811615 moved_to_echo=16374 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 26, 277, 2222, 674, 283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1521, 31, 0, 6, 0, 0, 1512, 1142, 0, 0, 0, 0, 0, 0, 144, 1, 0, 0, 2938, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 0, 0, 2343, 1170, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 415, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 1, 0, 68, 0, 456, 341, 0, 0, 0, 0, 0, 0, 3530, 0, 0, 0, 106, 1, 0, 0, 1245, 670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2881, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3482, 4212, 3094, 4083, 4515, 4949, 5552, 2881] avg=4096.0000 max=5552 max_rank=6 min=2881 min_rank=7 imbalance_ratio=1.355469 max_load_reduction=27216 max_load_reduction_pct=0.830566 moved_to_echo=17897 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1858, 3375, 0, 0, 145, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2620, 1, 42, 505, 0, 0, 2, 0, 15, 0, 0, 0, 0, 0, 0, 3, 3799, 12, 0, 342, 0, 0, 281, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 4086, 0, 0, 0, 2240, 0, 458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8, 0, 582, 0, 0, 4077, 53, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1900, 0, 0, 0, 15, 1042, 0, 0, 0, 0, 0, 0, 0, 1, 0, 719, 0, 0, 44, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5378, 3186, 4437, 10844, 156, 5007, 2957, 803] avg=4096.0000 max=10844 max_rank=3 min=156 min_rank=4 imbalance_ratio=2.647461 max_load_reduction=21924 max_load_reduction_pct=0.669067 moved_to_echo=8923 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4056, 2029, 0, 156, 3653, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 0, 0, 0, 476, 0, 1844, 1162, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3728, 27, 0, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 1432, 3879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 166, 0, 669, 0, 0, 0, 3864, 3, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 13, 0, 0, 34, 7, 4, 23, 0, 0, 0, 0, 0, 0, 839, 0, 0, 0, 0, 0, 0, 0, 86, 816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 426, 0, 2934, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9917, 3604, 3957, 5315, 4702, 172, 1741, 3360] avg=4096.0000 max=9917 max_rank=0 min=172 min_rank=5 imbalance_ratio=2.421143 max_load_reduction=22851 max_load_reduction_pct=0.697357 moved_to_echo=9975 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 266, 14, 2354, 0, 1354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 36, 3400, 0, 0, 240, 0, 0, 0, 0, 0, 0, 45, 700, 0, 58, 3154, 0, 1912, 677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 449, 0, 0, 0, 0, 0, 0, 0, 8, 764, 0, 8, 3693, 0, 73, 0, 493, 366, 0, 0, 0, 0, 0, 0, 3519, 0, 126, 2682, 268, 3, 0, 0, 1328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 605, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3988, 3681, 6546, 63, 4548, 5405, 7926, 611] avg=4096.0000 max=7926 max_rank=6 min=63 min_rank=3 imbalance_ratio=1.935059 max_load_reduction=24842 max_load_reduction_pct=0.758118 moved_to_echo=18490 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[9600, 6784, 0, 384, 128, 4864, 6656, 4864, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[512, 3840, 7040, 128, 896, 896, 5632, 128, 8576, 5760, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[11904, 256, 128, 7296, 512, 1792, 128, 128, 7296, 3840, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 768, 4864, 128, 3200, 128, 9728, 128, 14464, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 2432, 640, 12928, 10368, 1792, 640, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 256, 2816, 896, 128, 0, 3072, 24064, 2048, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 8960, 640, 128, 10752, 1536, 2944, 4096, 2304, 2048, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[256, 6144, 128, 7936, 14592, 0, 3328, 1024, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[18, 0, 27, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 3, 499, 0, 0, 111, 0, 1, 0, 0, 0, 4, 1837, 810, 1, 4096, 0, 0, 394, 4091, 3385, 0, 0, 0, 0, 0, 0, 3797, 0, 0, 0, 4010, 3884, 0, 1, 60, 9, 0, 3857, 0, 1189, 180, 0, 386, 0, 113, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 2963, 0, 0, 0, 3744, 1, 0, 0, 0, 0, 0, 0, 16, 0, 325, 0, 415, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2619, 0, 0, 0, 3980, 0, 1922, 0, 3233, 0, 0, 2291, 5, 0, 2402, 1234, 4030, 689, 920, 0, 3, 0, 0, 0, 7, 1900, 0, 0, 0, 0, 67, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 80, 0, 0, 0, 3887, 0, 10, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 4041, 0, 0, 0, 0, 3696, 0, 2182, 0, 0, 19, 0, 0, 0, 200, 3240, 0, 0, 163, 2468, 3, 0, 3886, 8, 66, 2, 0, 0, 0, 484, 2, 0, 0, 0, 0, 0, 0, 0, 0, 163] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 3980, 665, 0, 0, 4048, 0, 0, 0, 0, 3318, 4089, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2505, 0, 0, 0, 0, 0, 8, 0, 6, 4, 0, 185, 0, 1, 0, 3773, 6, 0, 55, 0, 0, 955, 4083, 0, 26, 87, 40, 0, 0, 0, 5, 165, 542, 0, 0, 0, 125, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 1162, 209, 0, 0, 0, 0, 4, 0, 3870, 0, 568, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 1317, 0, 25, 0, 4030, 1770, 0, 0, 0, 97, 145, 0, 4050, 2077, 4, 0, 0, 128, 713, 3038, 2380, 0, 2, 0, 0, 0, 48, 1820, 1145, 0, 0, 4085, 0, 0, 0, 0, 72] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 3998, 0, 0, 0, 3756, 0, 0, 0, 0, 0, 52, 0, 4024, 0, 0, 0, 0, 0, 3589, 0, 0, 0, 0, 0, 518, 2, 0, 3, 328, 911, 0, 0, 46, 92, 0, 2, 0, 352, 2428, 0, 331, 4, 49, 30, 4095, 0, 1, 0, 2877, 0, 0, 0, 0, 0, 1241, 0, 0, 2, 5, 0, 0, 4032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 1, 3901, 285, 393, 19, 0, 0, 0, 0, 0, 348, 65, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3809, 0, 0, 0, 0, 0, 2371, 2585, 119, 1, 0, 41, 349, 707, 0, 3, 1008, 6, 1, 3474, 0, 1117, 0, 60, 0, 0, 4028, 0, 0, 0, 0, 347, 0, 3613, 0, 0, 3, 0, 0, 17] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1250, 0, 4081, 0, 53, 0, 0, 0, 0, 14, 0, 0, 4003, 0, 3594, 20, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 2, 0, 4071, 856, 2357, 0, 1, 0, 2977, 0, 0, 1240, 19, 0, 0, 0, 149, 1178, 0, 0, 2207, 0, 2, 4095, 129, 12, 1, 0, 0, 0, 0, 0, 0, 138, 0, 313] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2067, 0, 0, 0, 3744, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 1, 238, 0, 0, 0, 0, 0, 0, 325, 0, 415, 0, 0, 0, 0, 0, 2174, 263, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3980, 0, 1922, 0, 2360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2291, 4, 0, 2402, 0, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 689, 920, 0, 3, 0, 0, 0, 7, 633, 635, 0, 0, 0, 0, 0, 0, 1900, 0, 0, 0, 0, 67, 0, 0, 1234, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5812, 255, 3177, 447, 8262, 8727, 2887, 3201] avg=4096.0000 max=8727 max_rank=5 min=255 min_rank=1 imbalance_ratio=2.130615 max_load_reduction=24041 max_load_reduction_pct=0.733673 moved_to_echo=23077 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[18, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 3, 499, 0, 1098, 280, 0, 0, 0, 0, 0, 0, 0, 111, 0, 1, 0, 0, 0, 4, 3396, 2, 0, 0, 0, 0, 0, 0, 1837, 810, 1, 4096, 0, 0, 394, 695, 0, 0, 0, 0, 0, 0, 0, 0, 3385, 0, 0, 0, 0, 0, 0, 2771, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2912, 3884, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 3857, 0, 1189, 180, 0, 386, 5, 746, 0, 0, 0, 0, 0, 0, 0, 113, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[38, 1885, 3514, 7833, 6156, 6856, 6372, 114] avg=4096.0000 max=7833 max_rank=3 min=38 min_rank=0 imbalance_ratio=1.912354 max_load_reduction=24935 max_load_reduction_pct=0.760956 moved_to_echo=19498 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1250, 0, 2848, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4003, 0, 3594, 20, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 710, 362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4071, 146, 0, 0, 0, 0, 0, 0, 0, 0, 2357, 0, 1, 0, 2977, 0, 0, 906, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 149, 1178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2207, 0, 2, 4095, 129, 12, 1, 0, 871, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 0, 313, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4151, 7722, 1078, 4219, 6241, 1346, 7560, 451] avg=4096.0000 max=7722 max_rank=1 min=451 min_rank=7 imbalance_ratio=1.885254 max_load_reduction=25046 max_load_reduction_pct=0.764343 moved_to_echo=15598 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 56, 0, 0, 0, 3887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4081, 0, 0, 0, 0, 0, 675, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 1811, 7, 0, 0, 0, 0, 0, 0, 4041, 0, 0, 0, 0, 3696, 0, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 200, 2364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 163, 1793, 3, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 66, 2, 0, 0, 0, 484, 2, 0, 17, 637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 163, 3886, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3943, 5005, 5905, 8108, 2583, 1967, 1208, 4049] avg=4096.0000 max=8108 max_rank=3 min=1208 min_rank=6 imbalance_ratio=1.979492 max_load_reduction=24660 max_load_reduction_pct=0.752563 moved_to_echo=9807 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 2722, 285, 393, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 348, 65, 1, 0, 0, 953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2150, 346, 0, 0, 0, 0, 0, 0, 3809, 0, 0, 0, 0, 0, 2371, 435, 0, 0, 0, 0, 0, 0, 0, 0, 119, 1, 0, 41, 349, 707, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 6, 1, 2521, 0, 1117, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4028, 0, 0, 0, 0, 347, 833, 0, 0, 0, 0, 0, 0, 0, 0, 3613, 0, 0, 3, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3420, 1367, 6592, 6615, 1220, 4713, 5208, 3633] avg=4096.0000 max=6615 max_rank=3 min=1220 min_rank=4 imbalance_ratio=1.614990 max_load_reduction=26153 max_load_reduction_pct=0.798126 moved_to_echo=14774 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2789, 0, 0, 0, 3756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 4024, 0, 1, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3589, 0, 0, 0, 756, 355, 0, 0, 0, 0, 0, 0, 0, 0, 518, 2, 0, 3, 328, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 92, 0, 2, 0, 257, 0, 0, 0, 0, 0, 0, 0, 0, 2428, 0, 331, 3, 49, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2877, 0, 0, 0, 0, 0, 854, 69, 0, 0, 0, 0, 0, 0, 1241, 0, 0, 2, 5, 0, 0, 4032, 4095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6545, 4103, 4700, 1006, 397, 2841, 3801, 9375] avg=4096.0000 max=9375 max_rank=7 min=397 min_rank=4 imbalance_ratio=2.288818 max_load_reduction=23393 max_load_reduction_pct=0.713898 moved_to_echo=16414 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2770, 665, 0, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3318, 4089, 1, 0, 0, 0, 282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 357, 0, 0, 0, 0, 0, 0, 2505, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4, 0, 185, 0, 1, 0, 2746, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 55, 0, 0, 955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 87, 40, 0, 0, 0, 5, 165, 853, 745, 0, 0, 0, 0, 0, 0, 542, 0, 0, 0, 125, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7483, 7690, 4453, 2513, 2942, 1016, 1921, 4750] avg=4096.0000 max=7690 max_rank=1 min=1016 min_rank=5 imbalance_ratio=1.877441 max_load_reduction=25078 max_load_reduction_pct=0.765320 moved_to_echo=10629 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 1162, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3870, 0, 568, 0, 0, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3346, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1317, 0, 25, 0, 684, 0, 0, 0, 0, 0, 0, 0, 0, 1770, 0, 0, 0, 97, 145, 0, 2955, 0, 0, 0, 0, 0, 0, 0, 0, 2077, 4, 0, 0, 128, 713, 0, 2380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 48, 1820, 1145, 0, 796, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 72, 3038, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1376, 4741, 3349, 2027, 4967, 5302, 3811, 7195] avg=4096.0000 max=7195 max_rank=7 min=1376 min_rank=0 imbalance_ratio=1.756592 max_load_reduction=25573 max_load_reduction_pct=0.780426 moved_to_echo=21275 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[12288, 896, 640, 5504, 128, 3840, 7296, 2944, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[3072, 1024, 10880, 4224, 1408, 768, 1920, 2176, 4096, 3968, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[7680, 128, 128, 4352, 3456, 2816, 256, 14464, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 4096, 3712, 12160, 128, 8704, 128, 2816, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[5632, 128, 2944, 7296, 4224, 6400, 0, 6656, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1280, 128, 13312, 1024, 1664, 256, 15488, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3712, 3840, 4096, 128, 256, 256, 0, 4608, 16384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[384, 128, 512, 128, 15872, 0, 128, 128, 14464, 1792, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[244, 268, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 2025, 142, 0, 42, 17, 21, 0, 0, 3586, 3417, 0, 0, 0, 0, 0, 0, 2, 939, 0, 2484, 0, 0, 5, 4095, 0, 104, 4007, 4, 0, 0, 0, 1020, 0, 347, 771, 1089, 0, 1, 0, 0, 0, 0, 11, 0, 4, 0, 4078, 764, 0, 3240, 0, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 1, 3490, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 0, 0, 0, 3935, 1638, 0, 22, 11, 0, 0, 0, 0, 0, 0, 0, 2609, 0, 31, 0, 1, 3474, 0, 2, 12, 0, 4056, 0, 554, 386, 3675, 80, 0, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 0, 4092] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1305, 0, 0, 3705, 0, 0, 15, 0, 0, 0, 0, 37, 0, 0, 0, 0, 507, 0, 3, 1991, 71, 4090, 6, 13, 25, 0, 0, 0, 12, 4, 0, 0, 0, 3966, 0, 3405, 0, 247, 4067, 1563, 0, 0, 0, 234, 0, 4096, 0, 2586, 21, 0, 0, 0, 3, 175, 0, 612, 0, 0, 0, 0, 0, 2, 4, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[2953, 2, 0, 4019, 0, 4078, 0, 0, 2694, 0, 0, 885, 0, 1, 0, 0, 26, 0, 4, 0, 0, 4051, 0, 0, 0, 21, 0, 0, 358, 3, 115, 4095, 0, 155, 0, 2855, 0, 0, 0, 22, 146, 300, 0, 1, 0, 0, 0, 1954, 200, 3, 0, 0, 0, 2019, 0, 0, 0, 11, 0, 0, 0, 1796, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[3638, 0, 0, 0, 0, 659, 0, 0, 4016, 3320, 0, 0, 0, 0, 0, 0, 0, 3, 0, 95, 0, 4083, 992, 0, 0, 0, 0, 0, 1834, 0, 0, 3957, 41, 1689, 0, 0, 0, 16, 15, 0, 0, 0, 0, 0, 294, 0, 0, 4059, 0, 494, 0, 0, 0, 0, 0, 0, 2379, 20, 0, 0, 8, 1156, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 3472, 1, 4042, 0, 13, 3583, 0, 0, 0, 0, 0, 150, 240, 779, 0, 0, 115, 0, 2638, 0, 0, 0, 0, 0, 24, 32, 0, 0, 0, 0, 365, 252, 3763, 0, 0, 749, 0, 0, 0, 0, 172, 3607, 52, 0, 3445, 0, 39, 1134, 0, 1, 0, 0, 0, 4084, 0, 0, 14, 2, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 13, 0, 0, 2664, 0, 0, 3958, 0, 164, 892, 0, 0, 76, 0, 0, 0, 0, 48, 1, 4091, 0, 0, 0, 3481, 0, 0, 0, 0, 0, 1286, 0, 111, 1, 4006, 0, 233, 158, 0, 0, 0, 0, 0, 0, 161, 167, 0, 0, 0, 0, 1483, 0, 75, 79, 0, 0, 0, 1485, 0, 0, 4082, 4053, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 304, 101, 3942, 0, 24, 0, 0, 0, 0, 237, 844, 0, 0, 0, 0, 0, 0, 0, 971, 3968, 3976, 0, 0, 110, 0, 0, 1, 0, 0, 0, 2100, 0, 0, 3, 2, 0, 0, 4036, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 197, 0, 0, 0, 0, 0, 3786, 4065, 4, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 2332, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1158, 621, 91, 0, 0, 0, 0, 0, 0, 185, 0, 0, 0, 2305, 1638, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 11, 0, 0, 0, 0, 0, 0, 1673, 552, 0, 0, 0, 0, 0, 0, 0, 2609, 0, 9, 0, 1, 3474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 12, 0, 4056, 0, 554, 386, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 462, 0, 0, 0, 0, 0, 22, 1078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1757, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2334, 1870, 4128, 2258, 6093, 8594, 1642, 5849] avg=4096.0000 max=8594 max_rank=5 min=1642 min_rank=6 imbalance_ratio=2.098145 max_load_reduction=24174 max_load_reduction_pct=0.737732 moved_to_echo=22178 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[244, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 2025, 142, 0, 42, 0, 496, 30, 0, 0, 0, 0, 0, 17, 21, 0, 0, 3586, 2002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 939, 0, 2484, 1338, 479, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1100, 0, 104, 4007, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 0, 347, 771, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 11, 0, 2995, 936, 0, 0, 0, 0, 0, 0, 4, 0, 4078, 764, 0, 1406, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[512, 2772, 5626, 5242, 5220, 3197, 3943, 6256] avg=4096.0000 max=6256 max_rank=7 min=512 min_rank=0 imbalance_ratio=1.527344 max_load_reduction=26512 max_load_reduction_pct=0.809082 moved_to_echo=18616 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1305, 0, 0, 2475, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 1230, 0, 64, 0, 0, 0, 0, 0, 507, 0, 3, 1991, 71, 2395, 6, 13, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 12, 4, 0, 0, 0, 574, 0, 0, 0, 0, 0, 0, 0, 3966, 0, 915, 0, 247, 4067, 1563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 234, 0, 4096, 0, 2522, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 3, 175, 0, 612, 2490, 1121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3795, 1331, 4986, 615, 10758, 6852, 4422, 9] avg=4096.0000 max=10758 max_rank=4 min=9 min_rank=7 imbalance_ratio=2.626465 max_load_reduction=22010 max_load_reduction_pct=0.671692 moved_to_echo=22041 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2953, 2, 0, 2685, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2694, 0, 0, 885, 0, 1, 0, 0, 1334, 275, 48, 0, 0, 0, 0, 0, 26, 0, 4, 0, 0, 2373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 358, 3, 115, 4095, 742, 568, 0, 0, 0, 0, 0, 0, 0, 155, 0, 767, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 146, 300, 0, 1, 0, 0, 0, 1906, 0, 0, 0, 0, 0, 0, 0, 0, 200, 3, 0, 0, 0, 2019, 0, 0, 2088, 1110, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 779, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9718, 5237, 2403, 5902, 944, 2353, 5420, 791] avg=4096.0000 max=9718 max_rank=0 min=791 min_rank=7 imbalance_ratio=2.372559 max_load_reduction=23050 max_load_reduction_pct=0.703430 moved_to_echo=9508 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2320, 1, 4042, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 3583, 0, 0, 0, 0, 0, 150, 240, 1152, 0, 85, 0, 0, 0, 0, 0, 779, 0, 0, 115, 0, 1545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 32, 0, 0, 0, 0, 370, 0, 0, 0, 0, 0, 0, 0, 365, 252, 1011, 0, 0, 749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 3607, 52, 0, 3360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 1134, 0, 1, 0, 0, 0, 2752, 723, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6376, 5210, 2439, 426, 2377, 7191, 4649, 4100] avg=4096.0000 max=7191 max_rank=5 min=426 min_rank=3 imbalance_ratio=1.755615 max_load_reduction=25577 max_load_reduction_pct=0.780548 moved_to_echo=18317 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3638, 0, 0, 0, 0, 659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4016, 3320, 0, 0, 0, 0, 0, 0, 0, 177, 101, 0, 0, 0, 0, 0, 0, 3, 0, 95, 0, 2391, 992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1834, 0, 0, 3957, 477, 573, 0, 0, 0, 0, 0, 0, 41, 1689, 0, 0, 0, 16, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 294, 0, 0, 3958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 0, 0, 0, 0, 0, 0, 0, 1119, 0, 0, 0, 0, 0, 0, 2379, 20, 0, 0, 8, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4297, 7614, 3481, 6841, 1761, 4252, 1613, 2909] avg=4096.0000 max=7614 max_rank=1 min=1613 min_rank=6 imbalance_ratio=1.858887 max_load_reduction=25154 max_load_reduction_pct=0.767639 moved_to_echo=10535 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 13, 0, 0, 2664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3958, 0, 164, 892, 0, 0, 76, 0, 0, 625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 1, 2396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3481, 0, 0, 0, 0, 0, 1286, 1686, 574, 0, 0, 0, 0, 0, 0, 0, 111, 1, 1073, 0, 233, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1483, 0, 75, 79, 0, 2933, 1121, 0, 0, 0, 0, 0, 0, 0, 0, 1485, 0, 0, 1771, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2677, 5715, 2445, 7027, 1576, 328, 5691, 7309] avg=4096.0000 max=7309 max_rank=7 min=328 min_rank=5 imbalance_ratio=1.784424 max_load_reduction=25459 max_load_reduction_pct=0.776947 moved_to_echo=14904 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 304, 101, 2630, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 237, 844, 0, 0, 0, 0, 1312, 626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 971, 3968, 2321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 0, 1, 0, 0, 0, 2100, 1684, 562, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 197, 0, 2, 1093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3786, 1755, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3059, 3019, 7260, 4457, 4039, 1, 5388, 5545] avg=4096.0000 max=7260 max_rank=2 min=1 min_rank=5 imbalance_ratio=1.772461 max_load_reduction=25508 max_load_reduction_pct=0.778442 moved_to_echo=14973 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[256, 3584, 0, 128, 2304, 1024, 128, 13952, 7680, 4352, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[256, 384, 128, 5504, 3968, 5248, 1408, 16512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[6528, 128, 5632, 896, 3840, 8064, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 8960, 384, 4992, 0, 640, 16512, 1664, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[14336, 3456, 512, 2688, 2048, 256, 256, 384, 6272, 2944, 512, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1408, 256, 128, 3328, 7680, 17792, 2688, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[384, 640, 1664, 1536, 128, 6400, 384, 640, 13312, 8320, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8192, 640, 128, 12544, 128, 11520, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 781, 1354, 0, 0, 0, 0, 84, 0, 0, 0, 3370, 1935, 278, 0, 0, 4073, 1, 0, 3600, 4095, 1, 0, 0, 0, 0, 0, 3085, 38, 1541, 3950, 0, 0, 2, 0, 0, 1, 0, 0, 3133, 35, 65, 0, 0, 0, 0, 535, 0, 11, 0, 0, 0, 0, 0, 0, 7, 455, 0, 5, 0, 0, 2, 331, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[1, 0, 0, 4096, 63, 0, 0, 0, 0, 0, 0, 2658, 0, 231, 13, 1221, 0, 0, 0, 0, 0, 2846, 23, 0, 0, 0, 3321, 623, 2767, 0, 1481, 0, 101, 517, 0, 25, 0, 50, 0, 3816, 1022, 3, 0, 0, 1, 332, 0, 118, 0, 0, 0, 35, 0, 9, 0, 3309, 4086, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[1651, 0, 0, 4006, 7, 0, 0, 0, 49, 0, 12, 1933, 924, 0, 1, 0, 6, 0, 2730, 3, 23, 2587, 0, 0, 0, 0, 2487, 163, 1736, 0, 0, 0, 649, 158, 0, 0, 0, 3531, 0, 3241, 0, 0, 0, 0, 0, 642, 51, 0, 0, 44, 0, 0, 0, 4, 1578, 844, 3708, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[664, 0, 0, 7, 238, 1117, 0, 0, 0, 0, 0, 1, 3488, 946, 0, 4069, 0, 0, 0, 3585, 0, 0, 1, 62, 434, 2, 852, 0, 3953, 1, 1, 23, 700, 65, 0, 0, 0, 160, 3, 910, 0, 367, 0, 3988, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 1, 0, 0, 19, 0, 3034, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[21, 0, 5, 3587, 2936, 3362, 0, 287, 0, 0, 0, 3571, 52, 29, 0, 505, 0, 1, 1, 0, 7, 13, 0, 0, 0, 1, 0, 0, 4022, 105, 0, 2, 5, 170, 0, 0, 48, 0, 114, 4086, 0, 0, 0, 8, 0, 0, 0, 2503, 26, 367, 0, 35, 0, 617, 0, 1330, 4093, 0, 0, 3, 0, 0, 856, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 40, 116, 0, 0, 0, 0, 263, 7, 0, 0, 0, 0, 476, 0, 2261, 3828, 31, 3842, 28, 0, 4038, 4, 3030, 61, 4, 1, 40, 3611, 48, 0, 0, 1781, 0, 7, 0, 0, 3, 0, 3928, 474, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 974, 827, 0, 0, 1294, 467, 0, 574, 185, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[549, 0, 0, 22, 0, 2724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 1494, 154, 0, 3938, 0, 0, 277, 14, 0, 0, 4007, 0, 0, 1, 0, 0, 2195, 0, 4, 0, 0, 4038, 0, 0, 0, 4080, 0, 0, 0, 0, 21, 1172, 3, 437, 0, 0, 0, 320, 4, 0, 0, 1, 3227, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[304, 0, 0, 267, 2, 52, 100, 5, 0, 0, 6, 0, 0, 3328, 0, 1, 0, 2, 0, 3770, 3824, 1813, 0, 0, 0, 0, 0, 130, 3397, 0, 0, 6, 260, 0, 3478, 0, 0, 0, 0, 3811, 0, 0, 0, 0, 0, 1291, 246, 24, 0, 147, 0, 0, 0, 15, 0, 0, 2581, 102, 0, 0, 3806, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 781, 1354, 0, 0, 0, 0, 84, 1311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3370, 1935, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 1, 0, 1840, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3085, 23, 1541, 3950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 2178, 0, 0, 0, 0, 0, 0, 0, 0, 35, 65, 0, 0, 0, 0, 535, 0, 955, 13, 45, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 7, 1, 2, 0, 0, 0, 0, 0, 0, 455, 0, 5, 0, 0, 2, 331, 0, 0, 404, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3530, 5583, 10009, 8599, 2181, 1648, 21, 1197] avg=4096.0000 max=10009 max_rank=2 min=21 min_rank=6 imbalance_ratio=2.443604 max_load_reduction=22759 max_load_reduction_pct=0.694550 moved_to_echo=5047 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 40, 116, 0, 0, 0, 0, 263, 10, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 476, 0, 1762, 0, 0, 0, 0, 0, 0, 0, 0, 3828, 31, 3842, 15, 0, 0, 4, 3030, 0, 0, 0, 0, 0, 0, 0, 0, 61, 4, 1, 40, 2166, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1781, 0, 7, 0, 0, 3, 0, 2731, 0, 0, 0, 0, 0, 0, 0, 0, 474, 525, 0, 0, 0, 0, 0, 0, 1197, 1242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 974, 827, 0, 4038, 203, 0, 0, 0, 0, 0, 0, 0, 1294, 467, 0, 574, 185, 0, 0, 499, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[429, 2245, 10750, 2320, 4522, 3438, 6042, 3022] avg=4096.0000 max=10750 max_rank=2 min=429 min_rank=0 imbalance_ratio=2.624512 max_load_reduction=22018 max_load_reduction_pct=0.671936 moved_to_echo=17024 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 4096, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2658, 0, 231, 13, 952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3321, 623, 1661, 0, 1481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 517, 0, 25, 0, 50, 0, 2653, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 3, 0, 0, 1, 332, 0, 118, 1163, 951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 9, 0, 3309, 2846, 155, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 269, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4160, 3854, 23, 7086, 3346, 3590, 6354, 4355] avg=4096.0000 max=7086 max_rank=3 min=23 min_rank=2 imbalance_ratio=1.729980 max_load_reduction=25682 max_load_reduction_pct=0.783752 moved_to_echo=17645 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1651, 0, 0, 4006, 7, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 49, 0, 12, 1933, 924, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2730, 2, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2487, 163, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 649, 158, 0, 0, 0, 3531, 0, 2253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 642, 51, 0, 988, 597, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 4, 1578, 844, 2587, 97, 0, 0, 0, 0, 0, 0, 3708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5665, 2919, 2761, 3692, 6591, 2278, 5154, 3708] avg=4096.0000 max=6591 max_rank=4 min=2278 min_rank=5 imbalance_ratio=1.609131 max_load_reduction=26177 max_load_reduction_pct=0.798859 moved_to_echo=17731 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[21, 0, 5, 3587, 2936, 3362, 0, 287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3571, 52, 29, 0, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2413, 105, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 5, 170, 0, 0, 48, 0, 114, 2841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 2503, 1245, 1383, 0, 0, 0, 0, 0, 0, 26, 367, 0, 35, 0, 617, 0, 1330, 13, 226, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 3, 0, 0, 856, 0, 111, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10198, 4046, 9, 2521, 3178, 5139, 2614, 5063] avg=4096.0000 max=10198 max_rank=0 min=9 min_rank=2 imbalance_ratio=2.489746 max_load_reduction=22570 max_load_reduction_pct=0.688782 moved_to_echo=15994 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[664, 0, 0, 7, 238, 1117, 0, 0, 1305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3488, 946, 0, 3170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1833, 0, 0, 1, 62, 0, 0, 0, 0, 0, 0, 0, 0, 434, 2, 852, 0, 2372, 1, 1, 23, 0, 0, 0, 0, 0, 0, 0, 0, 700, 65, 0, 0, 0, 160, 3, 633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 367, 0, 3988, 0, 0, 0, 0, 277, 1359, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 1, 0, 222, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 3034, 0, 1, 0, 899, 402, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3331, 7605, 1896, 3685, 1561, 6036, 4299, 4355] avg=4096.0000 max=7605 max_rank=1 min=1561 min_rank=4 imbalance_ratio=1.856689 max_load_reduction=25163 max_load_reduction_pct=0.767914 moved_to_echo=16251 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[304, 0, 0, 267, 2, 52, 100, 5, 1372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 3328, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1927, 3824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 2039, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 260, 0, 3478, 0, 0, 0, 0, 2649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1291, 246, 24, 1162, 1168, 48, 0, 0, 0, 0, 0, 0, 147, 0, 0, 0, 15, 0, 0, 1813, 190, 0, 0, 0, 0, 0, 0, 2581, 102, 0, 0, 3806, 0, 0, 0, 0, 423, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2102, 3335, 5753, 2175, 6387, 3939, 2165, 6912] avg=4096.0000 max=6912 max_rank=7 min=2102 min_rank=0 imbalance_ratio=1.687500 max_load_reduction=25856 max_load_reduction_pct=0.789062 moved_to_echo=19403 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[549, 0, 0, 22, 0, 2724, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1494, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 277, 14, 0, 0, 2398, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2195, 0, 4, 0, 0, 2803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 1235, 1381, 4, 0, 0, 0, 0, 0, 21, 1172, 3, 437, 0, 0, 0, 320, 3938, 228, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1, 3227, 0, 0, 0, 905, 19, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3353, 3181, 1567, 2690, 5002, 6700, 6119, 4156] avg=4096.0000 max=6700 max_rank=5 min=1567 min_rank=2 imbalance_ratio=1.635742 max_load_reduction=26068 max_load_reduction_pct=0.795532 moved_to_echo=21977 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[7936, 128, 8192, 5760, 8064, 0, 128, 3200, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[1536, 1024, 0, 8192, 128, 2304, 896, 2688, 8320, 8192, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[896, 128, 6784, 4096, 14208, 1792, 5504, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3200, 896, 1536, 12032, 3328, 7296, 128, 640, 4096, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 1792, 128, 512, 4096, 1664, 2432, 5888, 15360, 1408, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3584, 1024, 5760, 128, 128, 3840, 128, 18816, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 11648, 6400, 5376, 128, 9472, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[14976, 1408, 512, 128, 10752, 256, 1280, 0, 2688, 1280, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 2696, 1492, 0, 270, 0, 0, 0, 0, 0, 0, 0, 2377, 4096, 0, 41, 26, 0, 0, 2971, 108, 0, 11, 782, 0, 0, 208, 0, 0, 3168, 0, 0, 130, 0, 3, 62, 0, 604, 0, 25, 0, 3495, 8, 0, 0, 3988, 4093, 2110, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[78, 97, 0, 0, 0, 141, 1764, 3862, 0, 0, 16, 0, 0, 2653, 0, 0, 0, 0, 0, 3489, 4096, 0, 0, 117, 0, 5, 0, 0, 2, 0, 1, 2438, 0, 1, 0, 0, 15, 0, 0, 0, 0, 0, 7, 4090, 0, 3561, 3946, 0, 0, 1, 0, 0, 0, 0, 1099, 0, 0, 0, 0, 0, 0, 1289, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 81, 0, 251, 0, 10, 0, 4067, 2051, 0, 0, 0, 662, 1, 0, 0, 40, 3992, 7, 4096, 1240, 148, 3285, 15, 0, 0, 1, 0, 0, 11, 0, 0, 0, 0, 0, 0, 1110, 0, 1057, 0, 0, 1888, 0, 0, 0, 9, 4066, 0, 0, 0, 0, 9, 0, 0, 750, 0, 119, 0, 0, 0, 728, 3074, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 3144, 1709, 0, 17, 0, 0, 0, 3197, 0, 0, 0, 45, 0, 0, 3, 0, 0, 91, 0, 4013, 0, 876, 0, 21, 3902, 0, 1, 37, 68, 468, 0, 0, 0, 31, 0, 0, 507, 0, 1813, 0, 0, 9, 0, 0, 4064, 2927, 0, 0, 1138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 601, 0, 4085, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2703, 0, 0, 1, 198, 53, 0, 113, 3331, 0, 10, 6, 1, 0, 753, 0, 13, 37, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 3775, 0, 0, 0, 0, 307, 0, 0, 0, 378, 2392, 0, 0, 0, 2941, 0, 0, 2254, 21, 3786, 0, 626, 0, 0, 4071, 0, 920, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[40, 3990, 0, 106, 2182, 0, 0, 0, 0, 1926, 9, 0, 0, 461, 0, 0, 0, 5, 0, 0, 0, 4095, 0, 1630, 731, 0, 4062, 0, 0, 1439, 0, 53, 0, 0, 0, 0, 257, 0, 0, 23, 0, 4095, 0, 0, 0, 3816, 134, 9, 2, 187, 0, 16, 0, 0, 0, 48, 0, 0, 0, 0, 195, 3257, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[67, 0, 1449, 0, 0, 0, 0, 3, 0, 0, 29, 0, 4011, 0, 0, 0, 0, 3934, 0, 4092, 0, 1, 0, 3708, 0, 0, 4088, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 33, 0, 0, 451, 0, 0, 0, 67, 0, 125, 7, 11, 0, 3, 2730, 0, 0, 481, 2228, 0, 37, 0, 0, 1273, 0, 3936] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 19, 3971, 0, 719, 2, 1, 2, 3740, 0, 0, 0, 483, 1, 146, 4093, 0, 0, 3421, 0, 0, 22, 0, 0, 21, 0, 2431, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 3754, 999, 113, 122, 1949, 1, 0, 3416, 0, 0, 0, 0, 0, 0, 0, 0, 214, 0, 2788] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[78, 97, 0, 0, 0, 141, 1764, 3395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 2653, 0, 0, 467, 329, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1605, 0, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 2, 0, 1, 2438, 1158, 394, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 15, 0, 0, 0, 3561, 314, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4090, 0, 0, 3829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1099, 0, 2781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1289, 0, 0, 1004, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5475, 3582, 1722, 3998, 3891, 7926, 3881, 2293] avg=4096.0000 max=7926 max_rank=5 min=1722 min_rank=2 imbalance_ratio=1.935059 max_load_reduction=24842 max_load_reduction_pct=0.758118 moved_to_echo=17991 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 2696, 1492, 0, 270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2377, 4096, 0, 41, 0, 8, 122, 0, 0, 0, 0, 0, 26, 0, 0, 0, 44, 0, 11, 782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 208, 0, 0, 3168, 0, 0, 30, 335, 0, 0, 0, 0, 0, 0, 130, 0, 3, 62, 0, 604, 0, 25, 3988, 268, 0, 0, 0, 0, 0, 0, 0, 3495, 8, 0, 0, 0, 3971, 2110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 2368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4458, 6644, 863, 3741, 5080, 9584, 2372, 26] avg=4096.0000 max=9584 max_rank=5 min=26 min_rank=7 imbalance_ratio=2.339844 max_load_reduction=23184 max_load_reduction_pct=0.707520 moved_to_echo=17062 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 3144, 1709, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3197, 0, 0, 0, 45, 0, 0, 3, 0, 322, 87, 0, 0, 0, 0, 0, 0, 0, 91, 0, 1572, 0, 876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 3902, 0, 1, 37, 68, 468, 0, 1135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 507, 0, 1813, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 2840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 601, 0, 4085, 0, 1, 984, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4870, 3654, 2539, 5632, 6415, 2849, 1138, 5671] avg=4096.0000 max=6415 max_rank=4 min=1138 min_rank=6 imbalance_ratio=1.566162 max_load_reduction=26353 max_load_reduction_pct=0.804230 moved_to_echo=16073 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 81, 0, 251, 0, 10, 0, 3575, 0, 0, 0, 0, 0, 0, 0, 0, 2051, 0, 0, 0, 662, 1, 0, 0, 492, 99, 121, 0, 0, 0, 0, 0, 40, 3992, 7, 0, 487, 148, 3285, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 11, 0, 0, 350, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1110, 0, 1057, 0, 9, 368, 0, 0, 0, 0, 0, 0, 0, 1888, 0, 0, 0, 0, 3945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 750, 0, 3266, 0, 0, 0, 0, 0, 0, 0, 119, 0, 0, 0, 728, 3074, 0, 0, 304, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3917, 3426, 7974, 824, 2544, 5833, 4025, 4225] avg=4096.0000 max=7974 max_rank=2 min=824 min_rank=3 imbalance_ratio=1.946777 max_load_reduction=24794 max_load_reduction_pct=0.756653 moved_to_echo=16627 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2703, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 198, 53, 0, 113, 3331, 0, 10, 6, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 753, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3775, 0, 0, 0, 0, 2392, 0, 0, 0, 0, 0, 0, 0, 307, 0, 0, 0, 378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2941, 0, 0, 2254, 21, 3786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 626, 0, 0, 4071, 0, 920, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2704, 3715, 791, 4084, 6167, 685, 9002, 5620] avg=4096.0000 max=9002 max_rank=6 min=685 min_rank=5 imbalance_ratio=2.197754 max_load_reduction=23766 max_load_reduction_pct=0.725281 moved_to_echo=21474 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[40, 3990, 0, 106, 2182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1926, 9, 0, 0, 461, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4095, 0, 1630, 0, 0, 0, 0, 0, 0, 0, 0, 731, 0, 4062, 0, 0, 1439, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 257, 0, 0, 23, 3816, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 128, 9, 0, 0, 0, 0, 0, 0, 0, 0, 2, 187, 0, 16, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 195, 3257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6318, 2402, 5730, 6285, 4096, 4232, 253, 3452] avg=4096.0000 max=6318 max_rank=0 min=253 min_rank=6 imbalance_ratio=1.542480 max_load_reduction=26450 max_load_reduction_pct=0.807190 moved_to_echo=12033 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 19, 3490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 719, 2, 1, 2, 3740, 0, 0, 481, 329, 29, 0, 0, 0, 0, 0, 0, 483, 1, 0, 1601, 0, 0, 3421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 21, 0, 2431, 1158, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 340, 0, 0, 0, 3754, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 970, 113, 0, 0, 0, 0, 0, 0, 0, 0, 122, 1949, 1, 0, 3416, 0, 0, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 214, 0, 2788, 1005, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3509, 5303, 5506, 3649, 4107, 1083, 5604, 4007] avg=4096.0000 max=5604 max_rank=6 min=1083 min_rank=5 imbalance_ratio=1.368164 max_load_reduction=27164 max_load_reduction_pct=0.828979 moved_to_echo=14801 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[67, 0, 1449, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 4011, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3934, 0, 0, 0, 1, 0, 3708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 3, 0, 0, 0, 0, 0, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 33, 0, 67, 367, 0, 0, 0, 0, 0, 0, 0, 451, 0, 0, 0, 0, 0, 125, 0, 0, 0, 0, 0, 0, 0, 0, 7, 11, 0, 3, 2730, 0, 0, 481, 3261, 0, 0, 0, 0, 0, 0, 0, 2228, 0, 37, 0, 0, 1273, 0, 3936, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1517, 4042, 7643, 4555, 468, 576, 6493, 7474] avg=4096.0000 max=7643 max_rank=2 min=468 min_rank=4 imbalance_ratio=1.865967 max_load_reduction=25125 max_load_reduction_pct=0.766754 moved_to_echo=15011 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[384, 9984, 128, 4096, 384, 0, 15744, 2432, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 8448, 896, 0, 5376, 4352, 4224, 9728, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[256, 6272, 128, 128, 8448, 128, 5760, 640, 11904, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3072, 0, 128, 4736, 1024, 14208, 0, 6784, 3328, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[768, 3968, 12544, 128, 128, 4736, 512, 4992, 3840, 1792, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[256, 128, 128, 3840, 1792, 1152, 1152, 1920, 21760, 1408, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 10112, 4736, 384, 5248, 256, 1792, 10496, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[5504, 2816, 128, 128, 10496, 11008, 128, 128, 1536, 1152, 512, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 95, 6, 0, 0, 7, 4000, 0, 0, 262, 4093, 125, 2884, 2460, 0, 0, 0, 0, 0, 438, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 1020, 0, 4095, 0, 0, 2, 0, 34, 13, 41, 4, 0, 0, 0, 3515, 0, 0, 9, 0, 0, 0, 1480, 4081, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 177, 0, 0, 0, 375, 3765, 0, 0, 42, 0, 0, 0, 1, 1808, 3337, 0, 0, 1987, 465, 3712, 0, 0, 0, 1, 267, 1521, 0, 10, 0, 0, 0, 0, 3190, 0, 6, 6, 0, 12, 0, 4064, 0, 154, 0, 4, 0, 0, 241, 0, 0, 7, 0, 3732, 0, 0, 0, 3476, 390, 0, 0, 18, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[8, 0, 0, 627, 3807, 0, 3918, 214, 1112, 186, 65, 2887, 2000, 0, 0, 0, 721, 0, 0, 0, 2168, 32, 4096, 1450, 0, 0, 1551, 0, 333, 8, 0, 0, 0, 18, 256, 2049, 0, 0, 0, 0, 0, 3, 1, 0, 0, 2, 0, 0, 1, 0, 0, 95, 1, 0, 1614, 0, 0, 344, 0, 2, 0, 341, 2127, 731] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1, 1216, 62, 0, 3713, 0, 0, 38, 0, 5, 3117, 16, 0, 0, 0, 3493, 4095, 615, 1128, 0, 461, 0, 14, 0, 0, 6, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2514, 0, 0, 3783, 74, 0, 3429, 0, 536, 0, 0, 0, 0, 0, 335, 0, 21, 0, 0, 0, 0, 0, 0, 0, 3997, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4077, 2, 4, 0, 0, 2, 4094, 0, 0, 0, 4094, 0, 0, 0, 365, 977, 54, 45, 0, 0, 3072, 18, 0, 0, 0, 0, 0, 0, 1, 2837, 0, 0, 33, 1064, 1674, 0, 3818, 0, 0, 1, 0, 0, 0, 57, 2380, 0, 0, 7, 4089, 0, 0, 0, 1, 0, 0, 0, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1, 17, 84, 1, 0, 1, 2, 8, 0, 46, 0, 0, 1367, 0, 1, 0, 246, 19, 0, 0, 0, 0, 1579, 4092, 0, 2, 0, 3, 2202, 0, 0, 0, 0, 0, 0, 0, 606, 3459, 502, 51, 287, 0, 4085, 0, 0, 7, 3899, 4086, 0, 0, 0, 3840, 0, 0, 0, 0, 2213, 0, 0, 0, 46, 0, 0, 16] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 36, 0, 0, 0, 0, 0, 10, 3425, 0, 1885, 0, 0, 0, 2, 0, 0, 0, 0, 602, 60, 0, 4088, 143, 0, 0, 434, 0, 0, 0, 3830, 0, 194, 0, 0, 868, 0, 0, 0, 2, 3447, 0, 4096, 0, 64, 0, 1, 1815, 3447, 0, 0, 41, 0, 0, 0, 0, 4095, 0, 0, 0, 183, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 1, 330, 0, 0, 0, 3615, 4047, 909, 0, 0, 3977, 1, 0, 1, 0, 1878, 0, 319, 202, 0, 0, 2420, 3172, 0, 0, 0, 2, 0, 1472, 0, 0, 0, 0, 0, 1, 0, 377, 565, 7, 0, 0, 1309, 0, 0, 0, 2227, 0, 4046, 0, 0, 390, 0, 0, 0, 0, 67, 0, 0, 0, 0, 0, 1433, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 0, 627, 3807, 0, 3918, 214, 0, 0, 0, 0, 0, 0, 0, 0, 1112, 186, 65, 2887, 2000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 0, 0, 0, 2168, 32, 0, 1450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1551, 0, 333, 8, 0, 0, 3803, 0, 0, 0, 0, 0, 0, 0, 0, 18, 256, 2049, 0, 0, 0, 0, 386, 293, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 95, 1, 0, 1614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 344, 0, 2, 0, 341, 2127, 731, 0, 155, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8574, 6250, 3830, 5695, 3002, 6, 1711, 3700] avg=4096.0000 max=8574 max_rank=0 min=6 min_rank=5 imbalance_ratio=2.093262 max_load_reduction=24194 max_load_reduction_pct=0.738342 moved_to_echo=8419 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 177, 0, 0, 0, 375, 372, 0, 0, 0, 0, 0, 0, 0, 3765, 0, 0, 42, 0, 0, 0, 1, 41, 0, 0, 0, 0, 0, 0, 0, 450, 3337, 0, 0, 1987, 465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 267, 1521, 0, 10, 0, 3447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3190, 0, 6, 6, 0, 969, 265, 0, 0, 0, 0, 0, 0, 12, 0, 2194, 0, 154, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 241, 0, 0, 7, 0, 3732, 0, 791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3476, 390, 0, 0, 18, 0, 666, 389, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[924, 3849, 6239, 5246, 4436, 2364, 4771, 4939] avg=4096.0000 max=6239 max_rank=2 min=924 min_rank=0 imbalance_ratio=1.523193 max_load_reduction=26529 max_load_reduction_pct=0.809601 moved_to_echo=16510 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 6, 0, 0, 7, 42, 0, 0, 0, 0, 0, 0, 0, 996, 0, 0, 262, 4093, 125, 0, 2460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 438, 0, 0, 2677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 2143, 207, 0, 0, 0, 0, 0, 0, 1020, 0, 2209, 0, 0, 2, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 13, 41, 4, 0, 0, 0, 3515, 0, 797, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 1480, 4081, 4, 672, 861, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4471, 150, 7936, 3115, 2354, 3265, 4370, 7107] avg=4096.0000 max=7936 max_rank=2 min=150 min_rank=1 imbalance_ratio=1.937500 max_load_reduction=24832 max_load_reduction_pct=0.757812 moved_to_echo=17096 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 1216, 62, 0, 3713, 0, 0, 38, 314, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3117, 16, 0, 0, 0, 3493, 35, 0, 0, 0, 0, 0, 0, 0, 1020, 615, 1128, 0, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 99, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2514, 0, 0, 3783, 2194, 1, 0, 0, 0, 0, 0, 0, 74, 0, 1850, 0, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 335, 0, 21, 0, 0, 0, 668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3997, 0, 0, 0, 562, 881, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5344, 6666, 3224, 118, 8492, 2460, 1024, 5440] avg=4096.0000 max=8492 max_rank=4 min=118 min_rank=3 imbalance_ratio=2.073242 max_load_reduction=24276 max_load_reduction_pct=0.740845 moved_to_echo=17416 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4077, 2, 350, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 2, 4094, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 1019, 0, 0, 0, 365, 977, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3072, 18, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2837, 0, 0, 33, 1064, 2194, 4, 0, 0, 0, 0, 0, 0, 1674, 0, 2060, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 2380, 0, 0, 7, 4089, 0, 743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 626, 881, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4429, 4139, 2406, 3140, 6133, 3735, 7276, 1510] avg=4096.0000 max=7276 max_rank=6 min=1510 min_rank=7 imbalance_ratio=1.776367 max_load_reduction=25492 max_load_reduction_pct=0.777954 moved_to_echo=18654 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 330, 0, 0, 0, 3615, 4047, 120, 0, 0, 0, 0, 0, 0, 0, 909, 0, 0, 3977, 1, 0, 1, 0, 13, 0, 0, 0, 0, 0, 0, 0, 461, 0, 319, 202, 0, 0, 0, 3172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1472, 0, 0, 2247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 377, 565, 7, 1010, 173, 0, 0, 0, 0, 0, 0, 0, 0, 707, 0, 0, 0, 2227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 0, 0, 390, 0, 0, 0, 0, 255, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 0, 0, 1433, 0, 214, 407, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8113, 4901, 4154, 3721, 2133, 2934, 4691, 2121] avg=4096.0000 max=8113 max_rank=0 min=2121 min_rank=7 imbalance_ratio=1.980713 max_load_reduction=24655 max_load_reduction_pct=0.752411 moved_to_echo=11879 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 17, 84, 1, 0, 1, 2, 8, 374, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 1367, 0, 1, 0, 42, 0, 0, 0, 0, 0, 0, 0, 63, 19, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 2202, 0, 0, 0, 1466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 606, 3459, 502, 51, 131, 113, 0, 0, 0, 0, 0, 0, 287, 0, 2204, 0, 0, 7, 3899, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3840, 0, 0, 0, 0, 795, 0, 0, 0, 0, 0, 0, 0, 2213, 0, 0, 0, 46, 0, 0, 16, 670, 52, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[488, 1456, 4174, 3673, 4862, 10483, 4635, 2997] avg=4096.0000 max=10483 max_rank=5 min=488 min_rank=0 imbalance_ratio=2.559326 max_load_reduction=22285 max_load_reduction_pct=0.680084 moved_to_echo=22977 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 36, 0, 0, 0, 0, 0, 10, 379, 0, 0, 0, 0, 0, 0, 0, 3425, 0, 1885, 0, 0, 0, 2, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 602, 60, 0, 0, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 434, 0, 0, 0, 3830, 0, 3796, 0, 0, 0, 0, 0, 0, 0, 194, 0, 0, 868, 0, 0, 0, 2, 0, 292, 0, 0, 0, 0, 0, 0, 3447, 0, 2194, 0, 64, 0, 1, 1815, 0, 0, 0, 0, 0, 0, 0, 0, 3447, 0, 0, 41, 0, 0, 0, 0, 802, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 183, 0, 0, 0, 676, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[425, 5357, 805, 8060, 1356, 7521, 4290, 4954] avg=4096.0000 max=8060 max_rank=3 min=425 min_rank=0 imbalance_ratio=1.967773 max_load_reduction=24708 max_load_reduction_pct=0.754028 moved_to_echo=18121 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[6528, 128, 13440, 0, 768, 128, 6144, 6016, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 1280, 512, 896, 7552, 128, 15744, 4736, 2304, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 5248, 384, 4096, 1920, 3840, 0, 17536, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[256, 128, 384, 8960, 3200, 3968, 1152, 4992, 9088, 1408, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 1536, 1152, 9216, 1664, 0, 11392, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[6400, 384, 3584, 512, 4352, 1920, 7680, 768, 4096, 3712, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[9216, 256, 5120, 7040, 7552, 0, 128, 3584, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[7552, 384, 2816, 4480, 128, 128, 13056, 0, 4864, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True + [2026-09-21 08:25:36] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 425.8 | throughput per GPU (TFLOP/s/GPU): 75.7 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.120955E+01 | loss scale: 1.0 | grad norm: 36.282 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1966, 947, 59, 227, 167, 346, 89, 49, 436, 315, 63, 1, 29, 575, 218, 1900, 77, 516, 1293, 2014, 102, 732, 1557, 540, 1029, 55, 374, 1227, 646, 242, 123, 304, 155, 928, 655, 129, 626, 343, 522, 437, 264, 337, 415, 7, 1656, 630, 65, 837, 631, 501, 373, 35, 1687, 224, 203, 857, 437, 102, 582, 130, 7, 271, 274, 230] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[2682, 580, 148, 229, 345, 415, 90, 131, 424, 265, 65, 4, 75, 352, 300, 2017, 131, 1004, 1244, 1341, 139, 345, 506, 230, 727, 173, 186, 1429, 338, 591, 59, 251, 256, 468, 454, 57, 686, 188, 343, 392, 575, 485, 1126, 20, 1818, 853, 53, 415, 437, 898, 972, 33, 1920, 457, 85, 138, 771, 15, 719, 378, 19, 354, 455, 112] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[2114, 839, 153, 444, 255, 538, 129, 106, 345, 248, 49, 2, 20, 358, 143, 2045, 40, 553, 1000, 1787, 121, 934, 937, 295, 1190, 126, 225, 1298, 427, 533, 85, 268, 220, 524, 675, 118, 584, 329, 477, 377, 521, 357, 426, 15, 1504, 764, 79, 542, 563, 726, 759, 57, 1856, 419, 260, 783, 522, 32, 627, 204, 63, 217, 429, 132] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[2037, 822, 90, 195, 305, 377, 254, 102, 422, 204, 88, 1, 63, 327, 185, 1890, 188, 380, 927, 1687, 115, 873, 835, 513, 1246, 156, 95, 1181, 718, 484, 49, 221, 411, 650, 980, 130, 663, 199, 744, 506, 486, 223, 611, 16, 1511, 711, 30, 750, 727, 604, 851, 68, 1444, 569, 264, 268, 505, 21, 664, 214, 77, 305, 347, 189] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[2054, 810, 153, 305, 218, 489, 250, 78, 289, 426, 114, 4, 172, 282, 209, 1389, 112, 666, 804, 1484, 165, 407, 842, 259, 816, 140, 57, 991, 469, 562, 87, 434, 287, 596, 816, 157, 1312, 153, 455, 390, 631, 776, 857, 3, 1767, 429, 38, 509, 344, 1503, 497, 120, 1833, 313, 128, 228, 842, 29, 812, 359, 64, 379, 477, 127] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[2092, 518, 167, 191, 208, 373, 141, 122, 259, 348, 44, 6, 81, 298, 236, 1280, 138, 774, 1343, 1784, 265, 707, 1077, 254, 756, 221, 242, 1350, 556, 413, 59, 261, 526, 742, 518, 119, 460, 280, 442, 568, 436, 640, 853, 22, 1487, 994, 90, 451, 345, 402, 576, 30, 2499, 442, 78, 476, 507, 18, 794, 200, 63, 497, 525, 124] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[2303, 692, 195, 171, 303, 396, 32, 75, 246, 286, 24, 0, 75, 227, 173, 1714, 134, 1072, 1456, 1969, 76, 442, 811, 144, 972, 170, 89, 1362, 449, 971, 139, 321, 297, 480, 992, 76, 666, 188, 336, 498, 292, 746, 773, 37, 1715, 673, 54, 288, 270, 468, 790, 48, 2219, 582, 95, 112, 580, 4, 534, 202, 4, 388, 794, 78] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1527, 916, 105, 359, 113, 365, 25, 57, 142, 278, 81, 1, 24, 405, 86, 1578, 123, 345, 1413, 1636, 158, 940, 1187, 260, 932, 209, 136, 1574, 255, 498, 132, 219, 208, 820, 780, 158, 798, 315, 441, 513, 483, 639, 614, 32, 1662, 1015, 44, 444, 937, 713, 487, 105, 1409, 544, 207, 567, 630, 12, 1215, 263, 42, 204, 209, 179] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2022, 810, 153, 305, 218, 489, 250, 78, 0, 0, 0, 0, 0, 0, 0, 0, 289, 426, 114, 4, 172, 282, 209, 1389, 772, 132, 143, 0, 0, 0, 0, 0, 112, 666, 804, 30, 165, 407, 842, 259, 0, 0, 0, 0, 0, 0, 0, 0, 816, 140, 57, 991, 469, 562, 87, 434, 81, 32, 0, 0, 0, 0, 0, 0, 287, 596, 816, 157, 1312, 153, 455, 390, 424, 0, 0, 0, 0, 0, 0, 0, 631, 776, 857, 3, 1119, 429, 38, 509, 0, 0, 0, 0, 0, 0, 0, 0, 344, 1503, 497, 120, 1061, 313, 128, 228, 0, 0, 0, 0, 0, 0, 0, 0, 842, 29, 812, 359, 64, 379, 477, 127, 1322, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4325, 3932, 3285, 3669, 4590, 4362, 4194, 4411] avg=4096.0000 max=4590 max_rank=4 min=3285 min_rank=2 imbalance_ratio=1.120605 max_load_reduction=28178 max_load_reduction_pct=0.859924 moved_to_echo=17557 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1935, 947, 59, 227, 167, 346, 89, 49, 0, 0, 0, 0, 0, 0, 0, 0, 436, 315, 63, 1, 29, 575, 218, 1900, 711, 180, 134, 0, 0, 0, 0, 0, 77, 516, 1293, 40, 102, 732, 1557, 540, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 55, 374, 1227, 646, 242, 123, 304, 76, 31, 0, 0, 0, 0, 0, 0, 155, 928, 655, 129, 626, 343, 522, 437, 398, 0, 0, 0, 0, 0, 0, 0, 264, 337, 415, 7, 1048, 630, 65, 837, 0, 0, 0, 0, 0, 0, 0, 0, 631, 501, 373, 35, 976, 224, 203, 857, 0, 0, 0, 0, 0, 0, 0, 0, 437, 102, 582, 130, 7, 271, 274, 230, 1794, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3819, 4562, 4857, 4107, 4193, 3603, 3800, 3827] avg=4096.0000 max=4857 max_rank=2 min=3603 min_rank=5 imbalance_ratio=1.185791 max_load_reduction=27911 max_load_reduction_pct=0.851776 moved_to_echo=15423 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2640, 580, 148, 229, 345, 415, 90, 131, 0, 0, 0, 0, 0, 0, 0, 0, 424, 265, 65, 4, 75, 352, 300, 2017, 809, 119, 147, 0, 0, 0, 0, 0, 131, 1004, 1244, 28, 139, 345, 506, 230, 0, 0, 0, 0, 0, 0, 0, 0, 727, 173, 186, 1429, 338, 591, 59, 251, 83, 42, 0, 0, 0, 0, 0, 0, 256, 468, 454, 57, 686, 188, 343, 392, 437, 0, 0, 0, 0, 0, 0, 0, 575, 485, 1126, 20, 1151, 853, 53, 415, 0, 0, 0, 0, 0, 0, 0, 0, 437, 898, 972, 33, 1111, 457, 85, 138, 0, 0, 0, 0, 0, 0, 0, 0, 771, 15, 719, 378, 19, 354, 455, 112, 1194, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4578, 4577, 3627, 3879, 3281, 4678, 4131, 4017] avg=4096.0000 max=4678 max_rank=5 min=3281 min_rank=4 imbalance_ratio=1.142090 max_load_reduction=28090 max_load_reduction_pct=0.857239 moved_to_echo=16107 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2081, 839, 153, 444, 255, 538, 129, 106, 0, 0, 0, 0, 0, 0, 0, 0, 345, 248, 49, 2, 20, 358, 143, 2045, 782, 159, 122, 0, 0, 0, 0, 0, 40, 553, 1000, 36, 121, 934, 937, 295, 0, 0, 0, 0, 0, 0, 0, 0, 1190, 126, 225, 1298, 427, 533, 85, 268, 69, 33, 0, 0, 0, 0, 0, 0, 220, 524, 675, 118, 584, 329, 477, 377, 361, 0, 0, 0, 0, 0, 0, 0, 521, 357, 426, 15, 952, 764, 79, 542, 0, 0, 0, 0, 0, 0, 0, 0, 563, 726, 759, 57, 1074, 419, 260, 783, 0, 0, 0, 0, 0, 0, 0, 0, 522, 32, 627, 204, 63, 217, 429, 132, 1592, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4545, 4273, 3916, 4254, 3665, 3656, 4641, 3818] avg=4096.0000 max=4641 max_rank=6 min=3656 min_rank=5 imbalance_ratio=1.133057 max_load_reduction=28127 max_load_reduction_pct=0.858368 moved_to_echo=15780 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2005, 822, 90, 195, 305, 377, 254, 102, 0, 0, 0, 0, 0, 0, 0, 0, 422, 204, 88, 1, 63, 327, 185, 1890, 608, 150, 122, 0, 0, 0, 0, 0, 188, 380, 927, 34, 115, 873, 835, 513, 0, 0, 0, 0, 0, 0, 0, 0, 1246, 156, 95, 1181, 718, 484, 49, 221, 69, 32, 0, 0, 0, 0, 0, 0, 411, 650, 980, 130, 663, 199, 744, 506, 363, 0, 0, 0, 0, 0, 0, 0, 486, 223, 611, 16, 957, 711, 30, 750, 0, 0, 0, 0, 0, 0, 0, 0, 727, 604, 851, 68, 836, 569, 264, 268, 0, 0, 0, 0, 0, 0, 0, 0, 505, 21, 664, 214, 77, 305, 347, 189, 1503, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4150, 4060, 3865, 4251, 4646, 3784, 4187, 3825] avg=4096.0000 max=4646 max_rank=4 min=3784 min_rank=5 imbalance_ratio=1.134277 max_load_reduction=28122 max_load_reduction_pct=0.858215 moved_to_echo=16442 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2059, 518, 167, 191, 208, 373, 141, 122, 0, 0, 0, 0, 0, 0, 0, 0, 259, 348, 44, 6, 81, 298, 236, 1280, 1053, 159, 121, 0, 0, 0, 0, 0, 138, 774, 1343, 36, 265, 707, 1077, 254, 0, 0, 0, 0, 0, 0, 0, 0, 756, 221, 242, 1350, 556, 413, 59, 261, 68, 33, 0, 0, 0, 0, 0, 0, 526, 742, 518, 119, 460, 280, 442, 568, 357, 0, 0, 0, 0, 0, 0, 0, 436, 640, 853, 22, 941, 994, 90, 451, 0, 0, 0, 0, 0, 0, 0, 0, 345, 402, 576, 30, 1446, 442, 78, 476, 0, 0, 0, 0, 0, 0, 0, 0, 507, 18, 794, 200, 63, 497, 525, 124, 1589, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3779, 3885, 4594, 3959, 4012, 4427, 3795, 4317] avg=4096.0000 max=4594 max_rank=2 min=3779 min_rank=0 imbalance_ratio=1.121582 max_load_reduction=28174 max_load_reduction_pct=0.859802 moved_to_echo=16551 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2267, 692, 195, 171, 303, 396, 32, 75, 0, 0, 0, 0, 0, 0, 0, 0, 246, 286, 24, 0, 75, 227, 173, 1714, 935, 176, 139, 0, 0, 0, 0, 0, 134, 1072, 1456, 39, 76, 442, 811, 144, 0, 0, 0, 0, 0, 0, 0, 0, 972, 170, 89, 1362, 449, 971, 139, 321, 79, 36, 0, 0, 0, 0, 0, 0, 297, 480, 992, 76, 666, 188, 336, 498, 412, 0, 0, 0, 0, 0, 0, 0, 292, 746, 773, 37, 1085, 673, 54, 288, 0, 0, 0, 0, 0, 0, 0, 0, 270, 468, 790, 48, 1284, 582, 95, 112, 0, 0, 0, 0, 0, 0, 0, 0, 580, 4, 534, 202, 4, 388, 794, 78, 1754, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4131, 3995, 4174, 4588, 3945, 3948, 3649, 4338] avg=4096.0000 max=4588 max_rank=3 min=3649 min_rank=6 imbalance_ratio=1.120117 max_load_reduction=28180 max_load_reduction_pct=0.859985 moved_to_echo=15880 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1501, 916, 105, 359, 113, 365, 25, 57, 0, 0, 0, 0, 0, 0, 0, 0, 142, 278, 81, 1, 24, 405, 86, 1578, 598, 151, 140, 0, 0, 0, 0, 0, 123, 345, 1413, 24, 158, 940, 1187, 260, 0, 0, 0, 0, 0, 0, 0, 0, 932, 209, 136, 1574, 255, 498, 132, 219, 80, 26, 0, 0, 0, 0, 0, 0, 208, 820, 780, 158, 798, 315, 441, 513, 403, 0, 0, 0, 0, 0, 0, 0, 483, 639, 614, 32, 1039, 1015, 44, 444, 0, 0, 0, 0, 0, 0, 0, 0, 937, 713, 487, 105, 811, 544, 207, 567, 0, 0, 0, 0, 0, 0, 0, 0, 630, 12, 1215, 263, 42, 204, 209, 179, 1461, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3441, 3484, 4450, 4061, 4436, 4310, 4371, 4215] avg=4096.0000 max=4450 max_rank=2 min=3441 min_rank=0 imbalance_ratio=1.086426 max_load_reduction=28318 max_load_reduction_pct=0.864197 moved_to_echo=17332 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[16512, 6144, 1152, 2176, 1920, 3328, 1024, 768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1024, 5376, 9600, 384, 1152, 5504, 7808, 2560, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[2688, 2432, 640, 128, 640, 2944, 1664, 13824, 6272, 1280, 1152, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[2432, 5248, 5888, 1024, 5888, 2048, 3840, 3712, 3200, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3712, 4224, 5760, 256, 8320, 6144, 512, 4352, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[7680, 1280, 1408, 10496, 3968, 4352, 768, 2304, 640, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4864, 256, 6016, 2048, 384, 2688, 3584, 1280, 12288, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4352, 5888, 5376, 512, 8704, 3584, 1408, 3456, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[58, 1507, 1607, 0, 0, 0, 655, 0, 497, 18, 3, 4, 21, 0, 685, 0, 4, 0, 0, 0, 0, 3025, 0, 2610, 0, 1, 1, 1656, 10, 1181, 23, 2399, 290, 0, 593, 24, 0, 2, 4, 3691, 259, 42, 2820, 2, 0, 0, 948, 1831, 338, 1539, 0, 720, 5, 0, 0, 1612, 972, 160, 60, 0, 883, 0, 8, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[3, 2846, 363, 0, 7, 13, 391, 0, 2585, 0, 10, 97, 311, 18, 32, 5, 4, 25, 3, 1, 3, 3884, 0, 2119, 146, 29, 0, 4, 360, 29, 175, 442, 11, 0, 2, 0, 0, 3, 358, 3940, 2, 3793, 2463, 5, 240, 0, 1270, 1255, 795, 510, 0, 801, 0, 1, 8, 2725, 60, 195, 8, 0, 353, 0, 65, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 581, 668, 0, 0, 0, 1508, 0, 219, 0, 31, 141, 439, 0, 1348, 0, 160, 1, 1, 0, 0, 1763, 1, 4039, 1, 1062, 0, 712, 1, 85, 7, 3738, 107, 0, 2, 11, 0, 0, 0, 1016, 24, 382, 913, 0, 0, 2, 32, 3205, 1789, 2502, 4, 83, 0, 24, 16, 17, 2071, 80, 2, 47, 3927, 0, 6, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 2378, 58, 1, 6, 1, 149, 9, 1985, 0, 10, 67, 7, 2, 67, 1, 0, 0, 1, 0, 1, 4091, 0, 3969, 49, 80, 0, 1941, 647, 215, 269, 1979, 1, 0, 458, 415, 10, 0, 18, 2060, 6, 718, 1370, 0, 9, 2, 41, 2590, 1469, 1212, 0, 1457, 0, 29, 62, 115, 2, 374, 70, 5, 2260, 28, 1, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 1717, 103, 0, 0, 0, 1048, 0, 1173, 0, 340, 2, 4, 0, 61, 27, 7, 0, 0, 0, 0, 3479, 0, 3643, 27, 8, 0, 19, 705, 1071, 1, 653, 511, 11, 76, 19, 1, 0, 1, 4002, 20, 137, 1733, 8, 0, 0, 0, 2980, 121, 2810, 0, 549, 6, 0, 5, 2318, 2, 59, 1, 0, 3153, 157, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 3730, 144, 0, 0, 101, 1409, 1, 1689, 0, 744, 26, 74, 16, 182, 0, 0, 0, 1, 1, 1, 3938, 1, 3739, 1, 483, 0, 291, 198, 112, 100, 679, 34, 0, 2802, 1577, 0, 0, 0, 3454, 865, 105, 92, 186, 32, 0, 12, 1719, 0, 2869, 0, 60, 0, 6, 1, 985, 83, 2, 7, 6, 209, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[5, 2684, 123, 0, 0, 16, 167, 0, 2338, 0, 51, 33, 6, 1, 1, 0, 0, 1, 1, 0, 0, 3724, 1, 3579, 1, 34, 2, 2, 261, 2920, 757, 911, 8, 4, 55, 66, 24, 1, 10, 3940, 0, 771, 334, 0, 0, 0, 2, 1805, 1, 1862, 0, 914, 0, 7, 0, 2103, 2, 0, 17, 2, 2679, 161, 381, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[7, 1178, 66, 0, 0, 0, 3268, 3, 3865, 7, 2351, 58, 13, 6, 6, 0, 0, 68, 2, 0, 0, 4022, 0, 4041, 0, 435, 0, 351, 111, 33, 30, 89, 77, 0, 82, 25, 0, 0, 0, 949, 5, 167, 3179, 6, 0, 0, 107, 2982, 120, 391, 0, 133, 1, 0, 43, 2767, 4, 2, 4, 1, 1624, 58, 25, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 2378, 58, 1, 6, 1, 149, 9, 433, 98, 0, 0, 0, 0, 0, 0, 1985, 0, 10, 67, 7, 2, 67, 1, 1568, 54, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 695, 0, 3969, 0, 0, 0, 0, 0, 0, 0, 0, 49, 80, 0, 1941, 647, 215, 269, 1979, 739, 0, 0, 0, 0, 0, 0, 0, 1, 0, 458, 415, 10, 0, 18, 2060, 179, 0, 0, 0, 0, 0, 0, 0, 6, 718, 1370, 0, 9, 2, 41, 1364, 0, 0, 0, 0, 0, 0, 0, 0, 1469, 935, 0, 1457, 0, 29, 62, 115, 0, 0, 0, 0, 0, 0, 0, 0, 2, 374, 70, 5, 2260, 28, 1, 3, 1828, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3133, 3761, 4666, 5919, 3141, 3510, 4067, 4571] avg=4096.0000 max=5919 max_rank=3 min=3133 min_rank=0 imbalance_ratio=1.445068 max_load_reduction=26849 max_load_reduction_pct=0.819366 moved_to_echo=15289 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[7, 1178, 66, 0, 0, 0, 3268, 3, 499, 31, 0, 0, 0, 0, 0, 0, 3865, 7, 2351, 58, 13, 6, 6, 0, 1541, 62, 0, 0, 0, 0, 0, 0, 0, 68, 2, 0, 0, 684, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 435, 0, 351, 111, 33, 30, 89, 850, 0, 0, 0, 0, 0, 0, 0, 77, 0, 82, 25, 0, 0, 0, 949, 57, 0, 0, 0, 0, 0, 0, 0, 5, 167, 3179, 6, 0, 0, 107, 1571, 0, 0, 0, 0, 0, 0, 0, 0, 120, 303, 0, 133, 1, 0, 43, 2767, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 4, 1, 1624, 58, 25, 6, 1797, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5052, 7909, 4795, 1899, 1190, 5035, 3367, 3521] avg=4096.0000 max=7909 max_rank=1 min=1190 min_rank=4 imbalance_ratio=1.930908 max_load_reduction=24859 max_load_reduction_pct=0.758636 moved_to_echo=13113 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 3730, 144, 0, 0, 101, 1409, 1, 291, 237, 0, 0, 0, 0, 0, 0, 1689, 0, 744, 26, 74, 16, 182, 0, 1513, 40, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 662, 1, 3739, 0, 0, 0, 0, 0, 0, 0, 0, 1, 483, 0, 291, 198, 112, 100, 679, 493, 0, 0, 0, 0, 0, 0, 0, 34, 0, 2802, 1577, 0, 0, 0, 3454, 426, 0, 0, 0, 0, 0, 0, 0, 865, 105, 92, 186, 32, 0, 12, 895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2206, 0, 60, 0, 6, 1, 985, 0, 0, 0, 0, 0, 0, 0, 0, 83, 2, 7, 6, 209, 1, 0, 0, 1763, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5913, 4284, 4405, 2357, 8293, 2187, 3258, 2071] avg=4096.0000 max=8293 max_rank=4 min=2071 min_rank=7 imbalance_ratio=2.024658 max_load_reduction=24475 max_load_reduction_pct=0.746918 moved_to_echo=15809 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[58, 1507, 1607, 0, 0, 0, 655, 0, 306, 125, 0, 0, 0, 0, 0, 0, 497, 18, 3, 4, 21, 0, 685, 0, 1159, 38, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 514, 0, 2610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1656, 10, 1181, 23, 2399, 522, 0, 0, 0, 0, 0, 0, 0, 290, 0, 593, 24, 0, 2, 4, 3691, 227, 0, 0, 0, 0, 0, 0, 0, 259, 42, 2820, 2, 0, 0, 948, 965, 0, 0, 0, 0, 0, 0, 0, 0, 338, 1187, 0, 720, 5, 0, 0, 1612, 0, 0, 0, 0, 0, 0, 0, 0, 972, 160, 60, 0, 883, 0, 8, 0, 1352, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4258, 2425, 3128, 5793, 4831, 5036, 3862, 3435] avg=4096.0000 max=5793 max_rank=3 min=2425 min_rank=1 imbalance_ratio=1.414307 max_load_reduction=26975 max_load_reduction_pct=0.823212 moved_to_echo=17164 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[5, 2684, 123, 0, 0, 16, 167, 0, 302, 151, 0, 0, 0, 0, 0, 0, 2338, 0, 51, 33, 6, 1, 1, 0, 1427, 38, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 633, 1, 3579, 0, 0, 0, 0, 0, 0, 0, 0, 1, 34, 2, 2, 261, 2920, 757, 911, 515, 0, 0, 0, 0, 0, 0, 0, 8, 4, 55, 66, 24, 1, 10, 3940, 275, 0, 0, 0, 0, 0, 0, 0, 0, 771, 334, 0, 0, 0, 2, 950, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1436, 0, 914, 0, 7, 0, 2103, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 17, 2, 2679, 161, 381, 0, 1664, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3448, 3895, 4215, 5403, 4383, 2057, 4461, 4906] avg=4096.0000 max=5403 max_rank=3 min=2057 min_rank=5 imbalance_ratio=1.319092 max_load_reduction=27365 max_load_reduction_pct=0.835114 moved_to_echo=15807 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1717, 103, 0, 0, 0, 1048, 0, 498, 228, 0, 0, 0, 0, 0, 0, 1173, 0, 340, 2, 4, 0, 61, 27, 1333, 62, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 592, 0, 3643, 0, 0, 0, 0, 0, 0, 0, 0, 27, 8, 0, 19, 705, 1071, 1, 653, 850, 0, 0, 0, 0, 0, 0, 0, 511, 11, 76, 19, 1, 0, 1, 4002, 415, 0, 0, 0, 0, 0, 0, 0, 20, 137, 1733, 8, 0, 0, 0, 1570, 0, 0, 0, 0, 0, 0, 0, 0, 121, 2167, 0, 549, 6, 0, 5, 2318, 0, 0, 0, 0, 0, 0, 0, 0, 2, 59, 1, 0, 3153, 157, 0, 0, 1554, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3594, 3002, 4242, 3334, 5036, 3468, 5166, 4926] avg=4096.0000 max=5166 max_rank=6 min=3002 min_rank=1 imbalance_ratio=1.261230 max_load_reduction=27602 max_load_reduction_pct=0.842346 moved_to_echo=18596 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3, 2846, 363, 0, 7, 13, 391, 0, 210, 41, 0, 0, 0, 0, 0, 0, 2585, 0, 10, 97, 311, 18, 32, 5, 1488, 26, 0, 0, 0, 0, 0, 0, 4, 25, 3, 1, 3, 660, 0, 2119, 0, 0, 0, 0, 0, 0, 0, 0, 146, 29, 0, 4, 360, 29, 175, 442, 358, 0, 0, 0, 0, 0, 0, 0, 11, 0, 2, 0, 0, 3, 358, 3940, 75, 0, 0, 0, 0, 0, 0, 0, 2, 3793, 2463, 5, 240, 0, 1270, 661, 0, 0, 0, 0, 0, 0, 0, 0, 795, 394, 0, 801, 0, 1, 8, 2725, 0, 0, 0, 0, 0, 0, 0, 0, 60, 195, 8, 0, 353, 0, 65, 0, 1736, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3874, 4572, 2815, 1543, 4389, 8434, 4724, 2417] avg=4096.0000 max=8434 max_rank=5 min=1543 min_rank=3 imbalance_ratio=2.059082 max_load_reduction=24334 max_load_reduction_pct=0.742615 moved_to_echo=19964 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 581, 668, 0, 0, 0, 1508, 0, 536, 203, 0, 0, 0, 0, 0, 0, 219, 0, 31, 141, 439, 0, 1348, 0, 675, 67, 0, 0, 0, 0, 0, 0, 160, 1, 1, 0, 0, 300, 1, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1062, 0, 712, 1, 85, 7, 3738, 914, 0, 0, 0, 0, 0, 0, 0, 107, 0, 2, 11, 0, 0, 0, 1016, 369, 0, 0, 0, 0, 0, 0, 0, 24, 382, 913, 0, 0, 2, 32, 1688, 0, 0, 0, 0, 0, 0, 0, 0, 1789, 1930, 4, 83, 0, 24, 16, 17, 0, 0, 0, 0, 0, 0, 0, 0, 2071, 80, 2, 47, 3927, 0, 6, 0, 788, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3496, 2920, 4502, 6520, 1505, 3041, 3863, 6921] avg=4096.0000 max=6921 max_rank=7 min=1505 min_rank=4 imbalance_ratio=1.689697 max_load_reduction=25847 max_load_reduction_pct=0.788788 moved_to_echo=15330 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[14464, 128, 3584, 512, 896, 128, 2432, 128, 10752, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 2176, 128, 4992, 2304, 5760, 1408, 11008, 5248, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1280, 6144, 12928, 256, 384, 128, 2432, 9728, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3200, 896, 256, 128, 15104, 512, 512, 128, 12544, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4736, 10624, 128, 4736, 128, 128, 256, 12672, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 16640, 3200, 128, 128, 256, 8704, 128, 3200, 1152, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[1152, 128, 4096, 2176, 128, 128, 512, 23168, 2048, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[256, 128, 128, 128, 128, 4864, 128, 27776, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[30, 4, 0, 0, 0, 0, 0, 1, 0, 2050, 6, 259, 79, 5, 4, 0, 5, 1162, 79, 57, 0, 1, 0, 0, 9, 2, 19, 0, 2265, 0, 0, 9, 3715, 80, 20, 9, 2354, 0, 1415, 0, 0, 0, 0, 0, 3962, 496, 0, 28, 3989, 122, 0, 0, 3526, 6, 0, 3926, 160, 0, 2, 2, 1033, 1877, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[5, 0, 0, 1797, 6, 0, 2090, 0, 0, 1401, 1887, 0, 0, 0, 10, 263, 7, 1416, 0, 2, 16, 0, 0, 0, 0, 0, 5, 264, 878, 4074, 0, 0, 1457, 213, 28, 16, 0, 537, 2973, 0, 0, 0, 0, 20, 132, 533, 3226, 1, 4, 58, 3784, 0, 61, 0, 0, 3480, 164, 0, 0, 0, 14, 0, 29, 1917] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[11, 0, 6, 3, 0, 2, 0, 0, 750, 0, 2, 1769, 0, 0, 64, 3, 493, 2570, 4063, 0, 0, 5, 0, 0, 0, 0, 0, 1343, 753, 2660, 0, 0, 3798, 2, 252, 0, 34, 5, 1083, 0, 0, 0, 0, 59, 2481, 0, 160, 5, 1, 1591, 0, 0, 599, 0, 0, 4057, 4089, 0, 0, 0, 0, 2, 27, 26] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[11, 0, 0, 0, 0, 0, 20, 0, 16, 170, 17, 25, 0, 0, 0, 0, 474, 3834, 706, 26, 4, 0, 0, 0, 0, 0, 0, 2409, 19, 4, 0, 268, 3783, 7, 3193, 234, 448, 0, 3794, 0, 0, 2, 0, 3, 581, 0, 0, 0, 4054, 68, 18, 2, 227, 0, 263, 3940, 3096, 4, 0, 0, 960, 35, 50, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[392, 24, 0, 0, 4, 0, 0, 6, 0, 0, 6, 4061, 0, 0, 0, 1, 2, 2783, 1761, 0, 31, 0, 0, 7, 0, 0, 0, 3, 0, 7, 0, 0, 2758, 190, 65, 11, 0, 2, 74, 0, 0, 0, 0, 3279, 3831, 0, 0, 0, 4072, 3637, 0, 0, 4070, 0, 0, 1521, 0, 0, 0, 0, 157, 6, 7, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[157, 0, 0, 0, 2357, 17, 0, 68, 550, 3780, 0, 0, 1, 0, 0, 0, 366, 3731, 635, 0, 11, 1, 0, 0, 78, 0, 0, 0, 0, 0, 190, 17, 3697, 0, 75, 49, 2, 2261, 3880, 0, 0, 0, 0, 0, 4031, 2805, 0, 1, 132, 243, 232, 0, 3214, 1, 23, 149, 11, 0, 1, 0, 0, 0, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1, 97, 0, 1, 0, 0, 1, 0, 46, 2571, 0, 2745, 30, 0, 2, 0, 0, 1503, 6, 15, 0, 0, 0, 21, 31, 2, 6, 70, 75, 2674, 15, 0, 1618, 260, 1, 13, 0, 3, 39, 1, 0, 98, 0, 1558, 3702, 0, 3108, 243, 131, 1062, 834, 0, 1785, 0, 0, 4024, 3993, 0, 0, 0, 1, 40, 342, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[344, 0, 0, 0, 0, 1, 0, 0, 0, 2, 43, 770, 0, 0, 1, 0, 2, 4088, 1677, 7, 0, 4, 0, 0, 0, 0, 0, 22, 1, 9, 0, 1, 3854, 7, 30, 14, 5, 0, 3871, 0, 0, 0, 46, 60, 3717, 0, 19, 46, 1201, 877, 0, 0, 1123, 0, 0, 3732, 2992, 0, 0, 1, 4048, 8, 10, 135] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[30, 4, 0, 0, 0, 0, 0, 1, 3926, 72, 0, 0, 0, 0, 0, 0, 0, 2050, 6, 259, 79, 5, 4, 0, 1203, 661, 0, 0, 0, 0, 0, 0, 5, 1162, 79, 57, 0, 1, 0, 0, 211, 0, 0, 0, 0, 0, 0, 0, 9, 2, 19, 0, 2265, 0, 0, 9, 2195, 0, 0, 0, 0, 0, 0, 0, 787, 80, 20, 9, 2354, 0, 1415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2998, 496, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 3989, 122, 0, 0, 1537, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 2, 2, 1033, 1877, 0, 0, 753, 786, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4033, 4267, 1515, 4499, 4665, 3522, 5654, 4613] avg=4096.0000 max=5654 max_rank=6 min=1515 min_rank=2 imbalance_ratio=1.380371 max_load_reduction=27114 max_load_reduction_pct=0.827454 moved_to_echo=18454 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 0, 1797, 6, 0, 2090, 0, 3480, 28, 0, 0, 0, 0, 0, 0, 0, 1401, 1887, 0, 0, 0, 10, 263, 20, 259, 0, 0, 0, 0, 0, 0, 7, 1416, 0, 2, 16, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 264, 878, 4074, 0, 0, 861, 0, 0, 0, 0, 0, 0, 0, 309, 213, 28, 16, 0, 537, 2973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 100, 533, 3226, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4, 58, 3784, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 0, 0, 0, 14, 0, 29, 1917, 25, 13, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7406, 3840, 1448, 6082, 4076, 3880, 3874, 2162] avg=4096.0000 max=7406 max_rank=0 min=1448 min_rank=2 imbalance_ratio=1.808105 max_load_reduction=25362 max_load_reduction_pct=0.773987 moved_to_echo=13992 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[157, 0, 0, 0, 2357, 17, 0, 68, 149, 72, 0, 0, 0, 0, 0, 0, 550, 3780, 0, 0, 1, 0, 0, 0, 1097, 658, 0, 0, 0, 0, 0, 0, 366, 3731, 635, 0, 11, 1, 0, 0, 215, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 0, 190, 17, 2184, 0, 0, 0, 0, 0, 0, 0, 783, 0, 75, 49, 2, 2261, 3880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3050, 2805, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 132, 243, 232, 0, 1401, 1, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 1, 0, 0, 0, 2, 0, 766, 716, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2820, 6086, 4959, 2469, 7050, 5856, 2032, 1496] avg=4096.0000 max=7050 max_rank=4 min=1496 min_rank=7 imbalance_ratio=1.721191 max_load_reduction=25718 max_load_reduction_pct=0.784851 moved_to_echo=16434 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[392, 24, 0, 0, 4, 0, 0, 6, 1521, 53, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4061, 0, 0, 0, 1, 1389, 490, 0, 0, 0, 0, 0, 0, 2, 2783, 1761, 0, 31, 0, 0, 7, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 7, 0, 0, 1629, 0, 0, 0, 0, 0, 0, 0, 586, 190, 65, 11, 0, 2, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3279, 2899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 3637, 0, 0, 1774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 6, 7, 0, 728, 907, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2000, 5947, 4788, 1639, 928, 6178, 9483, 1805] avg=4096.0000 max=9483 max_rank=6 min=928 min_rank=4 imbalance_ratio=2.315186 max_load_reduction=23285 max_load_reduction_pct=0.710602 moved_to_echo=18394 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[11, 0, 6, 3, 0, 2, 0, 0, 4057, 74, 0, 0, 0, 0, 0, 0, 750, 0, 2, 1769, 0, 0, 64, 3, 204, 676, 0, 0, 0, 0, 0, 0, 493, 2570, 4063, 0, 0, 5, 0, 0, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1343, 753, 2660, 0, 0, 2244, 0, 0, 0, 0, 0, 0, 0, 804, 2, 252, 0, 34, 5, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 1878, 0, 160, 5, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1591, 0, 0, 262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 2, 27, 26, 471, 133, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4153, 3468, 7263, 7000, 2180, 2102, 1854, 4748] avg=4096.0000 max=7263 max_rank=2 min=1854 min_rank=6 imbalance_ratio=1.773193 max_load_reduction=25505 max_load_reduction_pct=0.778351 moved_to_echo=10884 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 97, 0, 1, 0, 0, 1, 0, 4024, 31, 0, 0, 0, 0, 0, 0, 46, 2571, 0, 2745, 30, 0, 2, 0, 609, 288, 0, 0, 0, 0, 0, 0, 0, 1503, 6, 15, 0, 0, 0, 21, 197, 0, 0, 0, 0, 0, 0, 0, 31, 2, 6, 70, 75, 2674, 15, 0, 956, 0, 0, 0, 0, 0, 0, 0, 343, 260, 1, 13, 0, 3, 39, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 1558, 2801, 0, 3108, 243, 0, 0, 0, 0, 0, 0, 0, 0, 131, 1062, 834, 0, 779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3993, 0, 0, 0, 1, 40, 342, 0, 704, 397, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4155, 6291, 1742, 3829, 660, 7808, 2806, 5477] avg=4096.0000 max=7808 max_rank=5 min=660 min_rank=4 imbalance_ratio=1.906250 max_load_reduction=24960 max_load_reduction_pct=0.761719 moved_to_echo=16751 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[344, 0, 0, 0, 0, 1, 0, 0, 3732, 75, 0, 0, 0, 0, 0, 0, 0, 2, 43, 770, 0, 0, 1, 0, 383, 686, 0, 0, 0, 0, 0, 0, 2, 4088, 1677, 7, 0, 4, 0, 0, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 1, 9, 0, 1, 2277, 0, 0, 0, 0, 0, 0, 0, 816, 7, 30, 14, 5, 0, 3871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 60, 2812, 0, 19, 46, 0, 0, 0, 0, 0, 0, 0, 0, 1201, 877, 0, 0, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2992, 0, 0, 1, 4048, 8, 10, 135, 707, 250, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4152, 1885, 5976, 2310, 4743, 2983, 2568, 8151] avg=4096.0000 max=8151 max_rank=7 min=1885 min_rank=1 imbalance_ratio=1.989990 max_load_reduction=24617 max_load_reduction_pct=0.751251 moved_to_echo=18445 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[11, 0, 0, 0, 0, 0, 20, 0, 3940, 78, 0, 0, 0, 0, 0, 0, 16, 170, 17, 25, 0, 0, 0, 0, 81, 675, 0, 0, 0, 0, 0, 0, 474, 3834, 706, 26, 4, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2409, 19, 4, 0, 268, 2240, 0, 0, 0, 0, 0, 0, 0, 790, 7, 3193, 234, 448, 0, 3794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 68, 18, 2, 92, 0, 263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3096, 4, 0, 0, 960, 35, 50, 3, 114, 54, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4049, 984, 5077, 4940, 8466, 439, 4497, 4316] avg=4096.0000 max=8466 max_rank=4 min=439 min_rank=5 imbalance_ratio=2.066895 max_load_reduction=24302 max_load_reduction_pct=0.741638 moved_to_echo=17718 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[5248, 768, 3712, 384, 2944, 2816, 17152, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[13696, 7680, 4992, 128, 6400, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1408, 9984, 2048, 9728, 128, 128, 128, 384, 4992, 4480, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 4224, 4096, 9472, 256, 384, 14592, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 4992, 17024, 3840, 6528, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[14592, 128, 128, 128, 6272, 2048, 512, 2176, 4352, 3328, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1408, 21120, 8960, 128, 128, 128, 0, 128, 1280, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1024, 128, 128, 1920, 2432, 128, 2176, 128, 24832, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 3702, 0, 3940, 0, 4003, 0, 0, 0, 17, 2, 116, 0, 0, 23, 0, 1, 36, 0, 8, 0, 2, 124, 4050, 0, 289, 0, 98, 3, 3982, 0, 4038, 0, 0, 0, 0, 0, 6, 2531, 3922, 4, 0, 0, 0, 38, 0, 15, 0, 387, 2, 3, 133, 1, 0, 0, 915, 94, 7, 0, 272] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 2234, 3768, 0, 3880, 10, 685, 112, 1, 0, 21, 0, 0, 102, 191, 48, 0, 0, 1, 0, 0, 1, 0, 118, 3577, 0, 3532, 0, 0, 17, 3932, 3494, 4082, 0, 800, 0, 362, 2, 0, 0, 0, 0, 6, 629, 0, 0, 0, 0, 583, 492, 0, 0, 76, 0, 0, 0, 0, 4, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3305, 3967, 0, 3980, 0, 2467, 0, 7, 6, 0, 0, 3348, 0, 245, 1, 0, 2, 0, 0, 0, 0, 0, 0, 3957, 0, 0, 2, 0, 3855, 4079, 447, 664, 0, 0, 0, 141, 60, 0, 151, 3, 0, 0, 344, 73, 34, 0, 0, 23, 1292, 0, 155, 0, 0, 0, 0, 158, 1, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 2649, 0, 2637, 4079, 2, 4094, 0, 0, 0, 0, 16, 0, 0, 1685, 0, 159, 13, 148, 0, 0, 1, 24, 0, 0, 148, 2472, 0, 3937, 4, 0, 0, 3980, 20, 4093, 0, 264, 0, 7, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 19, 1, 0, 0, 0, 0, 0, 0, 1504, 0, 0, 0, 811] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3257, 4067, 0, 3754, 0, 7, 0, 0, 13, 0, 4, 2, 841, 1, 0, 0, 230, 9, 0, 0, 0, 0, 0, 3572, 0, 2, 10, 0, 4021, 4096, 2462, 4081, 0, 0, 0, 159, 0, 0, 0, 0, 3, 0, 27, 0, 0, 0, 0, 0, 1996, 0, 7, 0, 0, 0, 0, 3, 0, 0, 0, 144] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8, 4, 0, 4077, 0, 134, 3, 55, 0, 50, 1, 3902, 0, 1433, 468, 95, 0, 141, 0, 3, 0, 0, 0, 2969, 7, 3893, 2188, 12, 0, 2763, 470, 73, 15, 0, 0, 2444, 66, 1, 805, 0, 22, 0, 2, 0, 280, 3, 0, 0, 0, 3, 21, 0, 0, 1, 56, 898, 2609, 2, 0, 2791] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 670, 0, 3030, 3901, 0, 2558, 0, 3310, 0, 0, 2, 0, 13, 4079, 23, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 15, 1279, 4096, 785, 2109, 0, 0, 0, 63, 0, 0, 0, 1, 0, 0, 1512, 211, 0, 0, 0, 293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 719, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4090, 3802, 0, 4092, 0, 3895, 0, 0, 4, 0, 0, 1794, 438, 1, 149, 6, 3, 0, 2, 0, 9, 0, 0, 2061, 0, 0, 11, 0, 348, 3652, 0, 4050, 0, 1, 0, 5, 55, 0, 36, 1, 0, 0, 0, 4, 0, 0, 0, 274, 3895, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 1476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4003, 0, 0, 0, 17, 2, 116, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 1, 36, 0, 8, 3635, 0, 0, 0, 0, 0, 0, 0, 0, 2, 124, 2782, 0, 289, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2531, 3922, 4, 0, 0, 0, 1627, 1335, 0, 0, 0, 0, 0, 0, 38, 0, 15, 0, 387, 2, 3, 133, 2605, 347, 0, 0, 0, 0, 0, 0, 1, 0, 0, 915, 94, 7, 0, 272, 2069, 814, 599, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1480, 4592, 3703, 3295, 1972, 9425, 3530, 4771] avg=4096.0000 max=9425 max_rank=5 min=1480 min_rank=0 imbalance_ratio=2.301025 max_load_reduction=23343 max_load_reduction_pct=0.712372 moved_to_echo=19698 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 2234, 1502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 685, 112, 1, 0, 21, 0, 0, 401, 0, 0, 0, 0, 0, 0, 0, 102, 191, 48, 0, 0, 1, 0, 0, 3589, 0, 0, 0, 0, 0, 0, 0, 1, 0, 118, 2457, 0, 3532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 3494, 1991, 0, 800, 0, 362, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 6, 629, 0, 1656, 1315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 583, 492, 0, 0, 76, 2565, 343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 2091, 719, 610, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3744, 1230, 3931, 6108, 6664, 3608, 4059, 3424] avg=4096.0000 max=6664 max_rank=4 min=1230 min_rank=1 imbalance_ratio=1.626953 max_load_reduction=26104 max_load_reduction_pct=0.796631 moved_to_echo=17755 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2649, 0, 2637, 1627, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 1685, 277, 0, 0, 0, 0, 0, 0, 0, 0, 159, 13, 148, 0, 0, 1, 24, 3633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 1698, 0, 3937, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 1996, 0, 264, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1792, 1387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 1, 0, 0, 0, 2707, 347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1504, 0, 0, 0, 811, 2097, 497, 660, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6915, 1978, 3978, 5787, 2287, 3180, 3074, 5569] avg=4096.0000 max=6915 max_rank=0 min=1978 min_rank=1 imbalance_ratio=1.688232 max_load_reduction=25853 max_load_reduction_pct=0.788971 moved_to_echo=14110 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3305, 1582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2467, 0, 7, 6, 0, 0, 3348, 444, 0, 0, 0, 0, 0, 0, 0, 0, 245, 1, 0, 2, 0, 0, 0, 3724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2718, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3855, 0, 447, 324, 0, 0, 0, 141, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 151, 3, 0, 0, 344, 73, 1743, 1350, 0, 0, 0, 0, 0, 0, 34, 0, 0, 23, 1292, 0, 155, 0, 2630, 355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 1, 0, 0, 1, 340, 795, 642, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4887, 6272, 3972, 2720, 4767, 3724, 4489, 1937] avg=4096.0000 max=6272 max_rank=1 min=1937 min_rank=7 imbalance_ratio=1.531250 max_load_reduction=26496 max_load_reduction_pct=0.808594 moved_to_echo=14917 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3257, 1621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 13, 0, 4, 2, 401, 0, 0, 0, 0, 0, 0, 0, 841, 1, 0, 0, 230, 9, 0, 0, 3739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2453, 0, 2, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4021, 0, 2462, 1990, 0, 0, 0, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 27, 0, 1787, 1272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1996, 0, 7, 0, 2482, 357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 144, 2091, 718, 659, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4878, 427, 4820, 2465, 8632, 3089, 4842, 3615] avg=4096.0000 max=8632 max_rank=4 min=427 min_rank=1 imbalance_ratio=2.107422 max_load_reduction=24136 max_load_reduction_pct=0.736572 moved_to_echo=20178 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 3, 55, 0, 50, 1, 3902, 333, 0, 0, 0, 0, 0, 0, 0, 0, 1433, 468, 95, 0, 141, 0, 3, 2523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2039, 7, 3893, 2188, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 36, 15, 0, 0, 2444, 0, 0, 0, 0, 0, 0, 0, 0, 66, 1, 805, 0, 22, 0, 2, 0, 1, 1382, 0, 0, 0, 0, 0, 0, 280, 3, 0, 0, 0, 3, 21, 0, 2695, 240, 0, 0, 0, 0, 0, 0, 0, 1, 56, 898, 2609, 2, 0, 2791, 37, 597, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11, 4478, 4663, 8139, 2965, 2279, 3242, 6991] avg=4096.0000 max=8139 max_rank=3 min=11 min_rank=0 imbalance_ratio=1.987061 max_load_reduction=24629 max_load_reduction_pct=0.751617 moved_to_echo=15477 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 670, 0, 3030, 1555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3310, 0, 0, 2, 0, 13, 4079, 459, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 6, 0, 0, 3739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2811, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 1279, 0, 785, 1029, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1512, 211, 1714, 867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 0, 0, 0, 0, 1691, 357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 719, 0, 0, 1080, 823, 632, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5255, 7863, 3768, 2826, 3156, 4305, 2341, 3254] avg=4096.0000 max=7863 max_rank=1 min=2341 min_rank=6 imbalance_ratio=1.919678 max_load_reduction=24905 max_load_reduction_pct=0.760040 moved_to_echo=13056 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4090, 1508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3895, 0, 0, 4, 0, 0, 1794, 235, 0, 0, 0, 0, 0, 0, 0, 438, 1, 149, 6, 3, 0, 2, 0, 3334, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 1408, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 348, 0, 0, 1971, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 36, 1, 0, 0, 0, 4, 1674, 1388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 274, 3895, 0, 0, 0, 2704, 318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 2, 2079, 418, 620, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5598, 5928, 3933, 1428, 2325, 3158, 7191, 3207] avg=4096.0000 max=7191 max_rank=6 min=1428 min_rank=3 imbalance_ratio=1.755615 max_load_reduction=25577 max_load_reduction_pct=0.780548 moved_to_echo=15881 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[256, 128, 3584, 3968, 128, 128, 2560, 384, 12032, 10368, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 3584, 2816, 768, 0, 4096, 11904, 5504, 4480, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 3328, 128, 18688, 10880, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 14592, 128, 128, 128, 128, 128, 14976, 3072, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[9600, 0, 7680, 11392, 128, 1152, 0, 3200, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 512, 18432, 128, 11776, 2304, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[384, 128, 128, 1280, 8064, 128, 256, 256, 20096, 2688, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1408, 2048, 768, 256, 256, 256, 128, 128, 28032, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 38, 4026, 2, 5, 2, 0, 5, 0, 10, 0, 0, 106, 0, 132, 0, 639, 3863, 4092, 160, 3, 3070, 0, 0, 0, 53, 3464, 0, 0, 0, 4011, 0, 0, 0, 352, 0, 0, 4, 0, 5, 0, 74, 0, 213, 0, 3563, 4, 762, 2, 29, 0, 0, 2, 0, 0, 0, 0, 4075, 0, 0, 2, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[15, 0, 0, 14, 35, 45, 0, 0, 108, 54, 0, 0, 0, 0, 0, 62, 1, 191, 1855, 3967, 2114, 0, 2815, 0, 0, 2, 71, 2655, 0, 44, 12, 2976, 0, 0, 273, 0, 7, 0, 0, 0, 799, 0, 35, 699, 10, 0, 3994, 45, 761, 225, 84, 928, 18, 0, 32, 0, 465, 0, 3741, 0, 0, 3615, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[36, 0, 1, 1, 2, 0, 0, 0, 0, 0, 58, 13, 0, 0, 0, 4011, 0, 221, 4020, 4095, 0, 207, 4096, 0, 0, 0, 667, 3806, 0, 0, 0, 1267, 0, 0, 0, 0, 0, 0, 0, 1, 3075, 3, 40, 1, 0, 0, 159, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2418, 0, 0, 13, 471, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 3723, 4062, 0, 0, 0, 3, 0, 6, 0, 0, 0, 0, 30, 0, 0, 0, 1, 2933, 4091, 946, 3785, 632, 0, 0, 0, 4, 3973, 0, 0, 0, 219, 0, 0, 1, 8, 3324, 0, 39, 0, 4, 0, 0, 85, 0, 0, 35, 4, 239, 20, 1, 260, 6, 0, 0, 0, 0, 0, 4084, 0, 0, 250, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[18, 5, 4093, 149, 0, 0, 0, 0, 1540, 0, 1808, 14, 0, 0, 0, 5, 7, 681, 5, 4096, 0, 3416, 1517, 0, 0, 0, 1, 3629, 0, 0, 15, 10, 0, 2, 0, 82, 0, 0, 0, 1, 3991, 5, 0, 38, 0, 0, 6, 1070, 109, 15, 0, 2339, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 6, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[343, 0, 3539, 8, 0, 13, 25, 45, 2179, 0, 0, 135, 0, 0, 0, 0, 0, 962, 160, 3232, 0, 216, 4058, 0, 0, 0, 0, 1020, 0, 0, 0, 4015, 38, 0, 0, 0, 256, 123, 0, 4, 0, 2, 4, 21, 0, 0, 3458, 0, 0, 6, 1, 294, 818, 1, 9, 0, 0, 0, 3767, 0, 0, 3944, 0, 72] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[7, 2, 8, 0, 0, 822, 0, 0, 3069, 0, 0, 123, 0, 0, 0, 0, 1, 1225, 267, 4091, 4, 3354, 2649, 141, 0, 0, 2, 3905, 0, 0, 0, 2409, 0, 0, 35, 12, 0, 0, 9, 0, 1090, 0, 0, 0, 0, 0, 15, 93, 0, 109, 1, 1308, 321, 0, 0, 2, 0, 0, 3710, 1, 0, 3983, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 6, 3654, 0, 3987, 60, 0, 637, 3678, 207, 12, 0, 0, 14, 0, 0, 470, 0, 29, 4040, 0, 16, 3462, 0, 0, 0, 295, 1854, 0, 0, 10, 3527, 0, 0, 14, 0, 0, 1976, 3, 0, 66, 0, 403, 0, 1, 0, 0, 274, 0, 497, 0, 39, 0, 62, 280, 0, 0, 0, 0, 101, 0, 3094, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 38, 4026, 2, 5, 2, 0, 525, 0, 0, 0, 0, 0, 0, 0, 5, 0, 10, 0, 0, 106, 0, 132, 1588, 514, 79, 0, 0, 0, 0, 0, 0, 639, 3863, 0, 160, 3, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 1876, 0, 0, 0, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 352, 0, 0, 4, 0, 3382, 0, 0, 0, 0, 0, 0, 0, 5, 0, 74, 0, 213, 0, 3563, 4, 833, 0, 0, 0, 0, 0, 0, 0, 762, 2, 29, 0, 0, 2, 0, 0, 2435, 710, 0, 0, 0, 0, 0, 0, 0, 0, 2638, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4598, 2434, 4786, 5940, 3738, 4692, 3940, 2640] avg=4096.0000 max=5940 max_rank=3 min=2434 min_rank=1 imbalance_ratio=1.450195 max_load_reduction=26828 max_load_reduction_pct=0.818726 moved_to_echo=15010 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[15, 0, 0, 14, 35, 45, 0, 0, 482, 0, 0, 0, 0, 0, 0, 0, 108, 54, 0, 0, 0, 0, 0, 62, 1217, 471, 72, 0, 0, 0, 0, 0, 1, 191, 1855, 0, 2114, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 71, 1438, 0, 44, 12, 2976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 273, 0, 7, 0, 0, 0, 3278, 0, 0, 0, 0, 0, 0, 0, 799, 0, 35, 699, 10, 0, 3994, 45, 765, 0, 0, 0, 0, 0, 0, 0, 761, 225, 84, 928, 18, 0, 32, 0, 2232, 689, 0, 0, 0, 0, 0, 0, 465, 0, 2422, 0, 0, 3615, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[591, 1984, 4273, 4543, 3558, 6347, 4969, 6503] avg=4096.0000 max=6503 max_rank=7 min=591 min_rank=0 imbalance_ratio=1.587646 max_load_reduction=26265 max_load_reduction_pct=0.801544 moved_to_echo=21377 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[343, 0, 3539, 8, 0, 13, 25, 45, 485, 0, 0, 0, 0, 0, 0, 0, 2179, 0, 0, 135, 0, 0, 0, 0, 467, 679, 73, 0, 0, 0, 0, 0, 0, 962, 160, 0, 0, 216, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 0, 0, 0, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 256, 123, 0, 4, 2671, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 21, 0, 0, 3458, 0, 770, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 294, 818, 1, 9, 0, 3218, 561, 0, 0, 0, 0, 0, 0, 0, 0, 2439, 0, 0, 3944, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4458, 3533, 1499, 4568, 3092, 4255, 4908, 6455] avg=4096.0000 max=6455 max_rank=7 min=1499 min_rank=2 imbalance_ratio=1.575928 max_load_reduction=26313 max_load_reduction_pct=0.803009 moved_to_echo=18710 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[36, 0, 1, 1, 2, 0, 0, 0, 311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 13, 0, 0, 0, 4011, 1745, 686, 47, 0, 0, 0, 0, 0, 0, 221, 4020, 0, 0, 207, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 667, 2061, 0, 0, 0, 1267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3384, 0, 0, 0, 0, 0, 0, 0, 3075, 3, 40, 1, 0, 0, 159, 4086, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3249, 711, 0, 0, 0, 0, 0, 0, 0, 0, 1566, 0, 0, 13, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[351, 6560, 4609, 3995, 3385, 7858, 3960, 2050] avg=4096.0000 max=7858 max_rank=5 min=351 min_rank=0 imbalance_ratio=1.918457 max_load_reduction=24910 max_load_reduction_pct=0.760193 moved_to_echo=17253 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[18, 5, 4093, 149, 0, 0, 0, 0, 527, 0, 0, 0, 0, 0, 0, 0, 1540, 0, 1808, 14, 0, 0, 0, 5, 1664, 254, 79, 0, 0, 0, 0, 0, 7, 681, 5, 0, 0, 3416, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1965, 0, 0, 15, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 82, 0, 0, 0, 1, 3385, 0, 0, 0, 0, 0, 0, 0, 3991, 5, 0, 38, 0, 0, 6, 1070, 837, 0, 0, 0, 0, 0, 0, 0, 109, 15, 0, 2339, 0, 0, 0, 0, 1203, 711, 0, 0, 0, 0, 0, 0, 0, 0, 2650, 0, 0, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4792, 5364, 4169, 1991, 3470, 5947, 4377, 2658] avg=4096.0000 max=5947 max_rank=5 min=1991 min_rank=3 imbalance_ratio=1.451904 max_load_reduction=26821 max_load_reduction_pct=0.818512 moved_to_echo=16452 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[7, 2, 8, 0, 0, 822, 0, 0, 478, 0, 0, 0, 0, 0, 0, 0, 3069, 0, 0, 123, 0, 0, 0, 0, 1791, 443, 72, 0, 0, 0, 0, 0, 1, 1225, 267, 0, 4, 3354, 105, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2114, 0, 0, 0, 2409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 12, 0, 0, 9, 0, 3382, 0, 0, 0, 0, 0, 0, 0, 1090, 0, 0, 0, 0, 0, 15, 93, 759, 0, 0, 0, 0, 0, 0, 0, 0, 109, 1, 1308, 321, 0, 0, 2, 2101, 709, 0, 0, 0, 0, 0, 0, 0, 0, 2401, 1, 0, 3983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1317, 5498, 5097, 4525, 3438, 1957, 4551, 6385] avg=4096.0000 max=6385 max_rank=7 min=1317 min_rank=0 imbalance_ratio=1.558838 max_load_reduction=26383 max_load_reduction_pct=0.805145 moved_to_echo=16331 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 3654, 0, 3987, 60, 0, 637, 0, 0, 0, 0, 0, 0, 0, 0, 3678, 207, 12, 0, 0, 14, 0, 0, 850, 580, 0, 0, 0, 0, 0, 0, 470, 0, 29, 0, 0, 16, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 295, 1004, 0, 0, 10, 3527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 1976, 3, 0, 3340, 0, 0, 0, 0, 0, 0, 0, 66, 0, 403, 0, 1, 0, 0, 274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 0, 39, 0, 62, 280, 0, 2746, 700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 3094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8344, 5341, 651, 4836, 5333, 744, 4324, 3195] avg=4096.0000 max=8344 max_rank=0 min=651 min_rank=2 imbalance_ratio=2.037109 max_load_reduction=24424 max_load_reduction_pct=0.745361 moved_to_echo=13596 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 3723, 4062, 0, 0, 0, 3, 0, 529, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 30, 0, 0, 1826, 109, 83, 0, 0, 0, 0, 0, 0, 1, 2933, 0, 946, 3785, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2147, 0, 0, 0, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8, 3324, 0, 39, 0, 3382, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 85, 0, 0, 35, 4, 840, 0, 0, 0, 0, 0, 0, 0, 239, 20, 1, 260, 6, 0, 0, 0, 504, 709, 0, 0, 0, 0, 0, 0, 0, 0, 2632, 0, 0, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8317, 2054, 7684, 2370, 6754, 968, 1739, 2882] avg=4096.0000 max=8317 max_rank=0 min=968 min_rank=5 imbalance_ratio=2.030518 max_load_reduction=24451 max_load_reduction_pct=0.746185 moved_to_echo=12343 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[512, 3840, 15488, 4224, 4096, 1024, 128, 768, 3456, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[10624, 384, 1920, 384, 0, 256, 0, 4224, 11264, 3840, 512, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[9088, 128, 640, 896, 256, 0, 11264, 5632, 5376, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[512, 0, 16768, 128, 0, 14976, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 1152, 13184, 0, 128, 128, 18560, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[512, 3968, 13184, 0, 3328, 11008, 896, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1920, 896, 128, 5248, 1280, 128, 384, 128, 17792, 5504, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 384, 512, 3712, 2176, 128, 128, 26240, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 2896, 3862, 64, 0, 2728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 2165, 18, 0, 0, 0, 664, 1, 4094, 0, 0, 0, 0, 353, 0, 0, 0, 0, 3655, 0, 0, 2, 0, 0, 0, 0, 154, 0, 0, 0, 3344, 0, 4067, 0, 0, 0, 0, 63, 6, 57, 237, 208, 194, 0, 3, 3919] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 45, 0, 4062, 0, 0, 2769, 0, 0, 16, 1, 0, 0, 0, 10, 0, 1, 0, 4, 1, 0, 37, 248, 0, 1459, 3808, 4085, 1, 66, 0, 0, 0, 0, 59, 0, 245, 3774, 135, 0, 62, 0, 8, 1, 0, 161, 0, 0, 0, 3947, 0, 1769, 0, 0, 6, 0, 1588, 0, 195, 56, 109, 0, 0, 0, 4040] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 3, 0, 3919, 2, 0, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3811, 1, 0, 1429, 0, 0, 0, 0, 3429, 1096, 0, 3467, 2835, 0, 3, 0, 0, 0, 398, 4090, 96, 0, 0, 0, 0, 2, 10, 1369, 1, 0, 0, 2, 0, 547, 29, 0, 0, 0, 0, 0, 296, 0, 0, 0, 0, 0, 1911] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 2, 3556, 0, 0, 4084, 2, 0, 49, 9, 0, 3, 0, 17, 0, 13, 363, 1, 3885, 0, 0, 0, 0, 618, 142, 37, 0, 3, 1951, 0, 44, 0, 0, 0, 0, 3578, 2977, 0, 0, 0, 35, 0, 3, 0, 0, 4, 2522, 4031, 0, 0, 444, 0, 0, 5, 2435, 3, 9, 0, 0, 0, 0, 0, 1943] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 7, 861, 3936, 0, 0, 4020, 0, 20, 0, 0, 0, 0, 0, 0, 199, 0, 2, 213, 4054, 204, 0, 0, 0, 0, 1, 0, 0, 10, 635, 0, 2414, 0, 0, 0, 1458, 4096, 0, 0, 0, 0, 17, 163, 0, 630, 363, 0, 1433, 4005, 40, 0, 13, 0, 0, 16, 0, 0, 7, 0, 3, 0, 0, 0, 3948] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 30, 3781, 3884, 0, 0, 4093, 0, 6, 3, 0, 0, 8, 0, 528, 0, 0, 2798, 0, 4060, 0, 0, 1478, 8, 60, 3, 88, 0, 0, 0, 0, 642, 2, 0, 0, 0, 810, 3565, 0, 0, 13, 1, 0, 35, 1766, 1, 0, 1, 8, 0, 1, 1, 4079, 0, 0, 194, 0, 396, 0, 0, 0, 0, 425, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 108, 3622, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 2, 525, 4066, 2470, 0, 1, 0, 0, 4064, 306, 0, 7, 1, 0, 32, 0, 0, 0, 0, 4058, 0, 0, 15, 0, 33, 0, 6, 3274, 0, 0, 0, 4058, 0, 2150, 22, 0, 0, 0, 3746, 3, 1, 4, 14, 47, 0, 0, 27] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 4080, 1, 0, 3940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 120, 0, 0, 12, 0, 275, 4094, 0, 4068, 10, 0, 1, 0, 0, 153, 3983, 3992, 34, 0, 425, 0, 0, 0, 582, 0, 0, 0, 0, 4096, 0, 173, 1, 0, 0, 0, 114, 11, 222, 0, 15, 68, 0, 0, 2086] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 45, 0, 0, 0, 0, 2695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 1, 0, 0, 0, 10, 0, 4062, 164, 0, 0, 0, 0, 0, 0, 1, 0, 4, 1, 0, 37, 248, 0, 48, 0, 0, 0, 0, 0, 0, 0, 1459, 3808, 504, 1, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 245, 2578, 135, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 0, 161, 0, 0, 0, 1889, 2644, 0, 0, 0, 0, 0, 0, 1894, 0, 1769, 0, 0, 6, 0, 1588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 195, 56, 109, 0, 0, 0, 4040, 1196, 937, 26, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2740, 4253, 339, 5838, 3079, 4703, 5257, 6559] avg=4096.0000 max=6559 max_rank=7 min=339 min_rank=2 imbalance_ratio=1.601318 max_load_reduction=26209 max_load_reduction_pct=0.799835 moved_to_echo=19598 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 3975, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 9, 0, 3, 0, 17, 0, 3556, 167, 0, 0, 0, 0, 0, 0, 13, 363, 1, 3885, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 618, 142, 6, 0, 3, 1951, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2444, 2977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 3, 0, 0, 4, 2522, 1930, 23, 0, 0, 0, 0, 0, 0, 1934, 0, 0, 444, 0, 0, 5, 2435, 0, 0, 0, 0, 0, 0, 0, 0, 3, 9, 0, 0, 0, 0, 0, 1943, 1134, 8, 38, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3979, 3801, 4333, 2764, 5421, 4517, 4818, 3135] avg=4096.0000 max=5421 max_rank=4 min=2764 min_rank=3 imbalance_ratio=1.323486 max_load_reduction=27347 max_load_reduction_pct=0.834564 moved_to_echo=17891 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2896, 0, 64, 0, 2656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3862, 139, 0, 0, 0, 0, 0, 0, 0, 0, 14, 2165, 18, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 664, 1, 506, 0, 0, 0, 0, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2497, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 0, 0, 0, 1601, 2649, 0, 0, 0, 0, 0, 0, 1604, 0, 4067, 0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 6, 57, 237, 208, 194, 0, 3, 3919, 1158, 939, 25, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5616, 4001, 2244, 1524, 2499, 4404, 5734, 6746] avg=4096.0000 max=6746 max_rank=7 min=1524 min_rank=3 imbalance_ratio=1.646973 max_load_reduction=26022 max_load_reduction_pct=0.794128 moved_to_echo=19383 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 30, 3781, 0, 0, 0, 3984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 0, 0, 8, 0, 528, 0, 3884, 0, 0, 0, 0, 0, 0, 0, 0, 2798, 0, 4060, 0, 0, 1478, 8, 71, 0, 0, 0, 0, 0, 0, 0, 60, 3, 12, 0, 0, 0, 0, 642, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 554, 3565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1, 0, 35, 1766, 1, 0, 1, 3, 56, 0, 0, 0, 0, 0, 0, 5, 0, 1, 1, 4079, 0, 0, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 396, 0, 0, 0, 0, 425, 0, 256, 20, 38, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7795, 4429, 8415, 717, 4121, 1876, 4280, 1135] avg=4096.0000 max=8415 max_rank=2 min=717 min_rank=3 imbalance_ratio=2.054443 max_load_reduction=24353 max_load_reduction_pct=0.743195 moved_to_echo=11412 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 2, 0, 3914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3919, 0, 0, 0, 0, 0, 0, 0, 0, 3811, 1, 0, 1429, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 3429, 136, 0, 3467, 2835, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 398, 2794, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 10, 1369, 1, 0, 0, 0, 709, 0, 0, 0, 0, 0, 0, 2, 0, 547, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 296, 0, 0, 0, 0, 0, 1911, 1296, 251, 38, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3919, 3919, 5311, 9870, 3288, 2091, 578, 3792] avg=4096.0000 max=9870 max_rank=3 min=578 min_rank=6 imbalance_ratio=2.409668 max_load_reduction=22898 max_load_reduction_pct=0.698792 moved_to_echo=9749 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 108, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3622, 168, 0, 0, 0, 0, 0, 0, 103, 2, 525, 4066, 2470, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 38, 0, 7, 1, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2772, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 6, 3274, 0, 0, 0, 1943, 198, 0, 0, 0, 0, 0, 0, 1947, 0, 2150, 22, 0, 0, 0, 3746, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 4, 14, 47, 0, 0, 27, 1286, 70, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[111, 3790, 7167, 4142, 2787, 5454, 7865, 1452] avg=4096.0000 max=7865 max_rank=6 min=111 min_rank=0 imbalance_ratio=1.920166 max_load_reduction=24903 max_load_reduction_pct=0.759979 moved_to_echo=17558 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 861, 0, 0, 0, 3912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 199, 3936, 166, 0, 0, 0, 0, 0, 0, 0, 2, 213, 4054, 204, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 10, 635, 0, 2414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1458, 2798, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 163, 0, 630, 363, 0, 1433, 1917, 0, 0, 0, 0, 0, 0, 0, 1922, 40, 0, 13, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 3, 0, 0, 0, 3948, 1298, 0, 38, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4780, 4321, 4543, 3060, 4256, 4523, 1991, 5294] avg=4096.0000 max=5294 max_rank=7 min=1991 min_rank=6 imbalance_ratio=1.292480 max_load_reduction=27474 max_load_reduction_pct=0.838440 moved_to_echo=16064 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 3827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 120, 0, 0, 12, 72, 0, 0, 0, 0, 0, 0, 0, 0, 275, 499, 0, 4068, 10, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 3983, 2722, 34, 0, 425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 582, 0, 0, 0, 0, 1965, 2653, 0, 0, 0, 0, 0, 0, 1957, 0, 173, 1, 0, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 11, 222, 0, 15, 68, 0, 0, 2086, 1270, 942, 41, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3828, 4254, 416, 4853, 7317, 5200, 2245, 4655] avg=4096.0000 max=7317 max_rank=4 min=416 min_rank=2 imbalance_ratio=1.786377 max_load_reduction=25451 max_load_reduction_pct=0.776703 moved_to_echo=19417 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 256, 640, 7424, 384, 128, 3968, 11264, 8960, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 7040, 768, 18560, 4352, 128, 1792, 128, 512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 1280, 384, 384, 384, 0, 512, 17920, 8960, 3200, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[11392, 128, 8832, 512, 4096, 128, 128, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[2816, 11776, 1792, 128, 7680, 5504, 0, 3584, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 256, 6144, 19200, 6912, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 7680, 0, 128, 128, 25088, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 128, 0, 640, 256, 30976, 1024, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[10, 1259, 0, 0, 0, 0, 0, 0, 232, 8, 53, 0, 0, 4034, 24, 4062, 18, 0, 0, 0, 0, 6, 0, 0, 0, 1, 2688, 17, 0, 4087, 756, 0, 2, 0, 0, 0, 3535, 13, 0, 130, 1981, 0, 2498, 5, 0, 2005, 159, 0, 8, 1, 0, 0, 15, 7, 4041, 0, 0, 0, 118, 0, 0, 995, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 3238, 0, 0, 0, 8, 0, 0, 0, 289, 5, 0, 50, 3971, 20, 4096, 0, 0, 0, 0, 0, 0, 109, 0, 1, 4, 133, 225, 0, 4073, 25, 0, 0, 0, 0, 0, 176, 0, 0, 0, 4057, 0, 2, 0, 7, 1268, 0, 0, 0, 0, 3936, 0, 0, 994, 2884, 0, 0, 0, 0, 0, 0, 3197, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 2597, 0, 0, 0, 70, 0, 0, 0, 0, 1, 0, 3, 0, 3, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 4, 1, 3911, 1, 0, 4078, 34, 0, 17, 655, 0, 51, 106, 0, 5, 922, 3894, 7, 25, 26, 109, 4038, 0, 0, 0, 0, 4087, 0, 0, 0, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 3648, 0, 0, 0, 383, 0, 0, 0, 39, 0, 0, 3, 3756, 2, 4077, 0, 0, 126, 18, 9, 0, 49, 0, 110, 0, 547, 0, 0, 4096, 426, 0, 0, 245, 3, 0, 4082, 0, 0, 62, 2, 36, 3944, 2, 0, 11, 0, 11, 0, 0, 3971, 0, 0, 201, 2882, 0, 15, 9, 0, 0, 3, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 3735, 224, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 1982, 1318, 4094, 0, 0, 0, 0, 0, 3337, 0, 0, 0, 0, 4091, 0, 0, 2464, 962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3981, 610, 184, 2, 1, 3, 0, 0, 0, 0, 2658, 0, 0, 2450, 161, 0, 0, 8, 469, 0, 0, 0, 0, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 74, 0, 0, 0, 0, 0, 275, 0, 315, 0, 0, 1, 4032, 1, 864, 0, 4, 3, 0, 75, 0, 89, 0, 0, 54, 3910, 33, 0, 3884, 0, 0, 314, 1, 0, 0, 4005, 115, 0, 0, 2922, 0, 3894, 0, 3036, 0, 0, 563, 0, 0, 0, 2, 27, 9, 4093, 0, 0, 1, 160, 0, 12, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 4011, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3855, 14, 0, 4079, 74, 0, 0, 0, 0, 0, 152, 0, 0, 123, 4047, 24, 34, 2, 1, 299, 0, 0, 0, 0, 4083, 0, 0, 3, 3746, 0, 1, 0, 4, 0, 10, 0, 0, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[120, 4092, 0, 0, 0, 12, 0, 0, 0, 0, 1, 0, 26, 3554, 10, 4093, 240, 0, 0, 0, 0, 175, 0, 0, 1, 7, 4053, 123, 0, 3975, 0, 0, 12, 0, 0, 0, 43, 0, 0, 2570, 288, 0, 3687, 0, 11, 903, 0, 0, 0, 0, 154, 10, 0, 3413, 1195, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[10, 1259, 0, 0, 0, 0, 0, 0, 837, 0, 0, 0, 0, 0, 0, 0, 232, 8, 53, 0, 0, 4034, 24, 684, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 6, 0, 0, 3378, 531, 0, 0, 0, 0, 0, 0, 0, 1, 2688, 17, 0, 892, 756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3535, 13, 0, 130, 1504, 641, 0, 0, 0, 0, 0, 0, 503, 0, 2498, 5, 0, 2005, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 0, 0, 15, 7, 1183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 0, 0, 995, 0, 0, 2664, 1354, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2106, 5035, 3933, 4354, 5825, 5170, 1214, 5131] avg=4096.0000 max=5825 max_rank=4 min=1214 min_rank=6 imbalance_ratio=1.422119 max_load_reduction=26943 max_load_reduction_pct=0.822235 moved_to_echo=17340 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3238, 0, 0, 0, 8, 0, 0, 1715, 0, 0, 0, 0, 0, 0, 0, 0, 289, 5, 0, 50, 3971, 20, 689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 0, 3407, 529, 0, 0, 0, 0, 0, 0, 1, 4, 133, 225, 0, 889, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 0, 0, 1073, 1313, 0, 0, 0, 0, 0, 0, 1029, 0, 2, 0, 7, 1268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3936, 0, 0, 994, 845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3197, 0, 0, 2655, 966, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4961, 5024, 4045, 1277, 2562, 2306, 5775, 6818] avg=4096.0000 max=6818 max_rank=7 min=1277 min_rank=3 imbalance_ratio=1.664551 max_load_reduction=25950 max_load_reduction_pct=0.791931 moved_to_echo=17461 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3648, 0, 0, 0, 383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 3, 3756, 2, 686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 18, 9, 0, 49, 0, 3391, 532, 0, 0, 0, 0, 0, 0, 110, 0, 547, 0, 0, 894, 426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, 3, 0, 4082, 0, 0, 62, 1072, 0, 0, 0, 0, 0, 0, 0, 2, 36, 3944, 2, 0, 11, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3971, 0, 0, 201, 845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 9, 0, 0, 3, 0, 0, 0, 2670, 965, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4031, 4486, 4125, 1977, 5464, 4006, 5017, 3662] avg=4096.0000 max=5464 max_rank=4 min=1977 min_rank=3 imbalance_ratio=1.333984 max_load_reduction=27304 max_load_reduction_pct=0.833252 moved_to_echo=18149 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 74, 0, 0, 0, 0, 0, 275, 1235, 0, 0, 0, 0, 0, 0, 0, 0, 315, 0, 0, 1, 4032, 1, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 0, 75, 0, 89, 0, 718, 505, 0, 0, 0, 0, 0, 0, 0, 54, 3910, 33, 0, 847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 314, 1, 0, 0, 4005, 115, 0, 0, 1523, 946, 0, 0, 0, 0, 0, 0, 741, 0, 3894, 0, 3036, 0, 0, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 27, 9, 1199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 160, 0, 12, 0, 0, 0, 2532, 1371, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1584, 4495, 1394, 4844, 6904, 8234, 1237, 4076] avg=4096.0000 max=8234 max_rank=5 min=1237 min_rank=6 imbalance_ratio=2.010254 max_load_reduction=24534 max_load_reduction_pct=0.748718 moved_to_echo=20451 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3735, 224, 0, 0, 30, 0, 0, 1683, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1982, 1318, 689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3337, 0, 0, 3405, 320, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 538, 962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 1289, 0, 0, 0, 0, 0, 0, 1009, 610, 184, 2, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2658, 0, 0, 2450, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 469, 0, 0, 0, 0, 4, 1606, 53, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5672, 3989, 7062, 5591, 1348, 1809, 5157, 2140] avg=4096.0000 max=7062 max_rank=2 min=1348 min_rank=4 imbalance_ratio=1.724121 max_load_reduction=25706 max_load_reduction_pct=0.784485 moved_to_echo=10454 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[120, 4092, 0, 0, 0, 12, 0, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 26, 3554, 10, 689, 0, 0, 0, 0, 0, 0, 0, 0, 240, 0, 0, 0, 0, 175, 0, 0, 3404, 516, 0, 0, 0, 0, 0, 0, 1, 7, 4053, 123, 0, 868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 43, 0, 0, 2570, 444, 93, 0, 0, 0, 0, 0, 0, 74, 0, 3687, 0, 11, 903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 10, 0, 3413, 351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2591, 400, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4345, 4280, 4335, 5052, 3162, 4675, 3928, 2991] avg=4096.0000 max=5052 max_rank=3 min=2991 min_rank=7 imbalance_ratio=1.233398 max_load_reduction=27716 max_load_reduction_pct=0.845825 moved_to_echo=14756 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4011, 0, 0, 0, 28, 0, 0, 1711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3398, 530, 0, 0, 0, 0, 0, 0, 0, 7, 3855, 14, 0, 890, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 0, 0, 123, 1394, 1310, 0, 0, 0, 0, 0, 0, 1026, 24, 34, 2, 1, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 3, 1097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 10, 0, 0, 6, 2659, 1255, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5750, 4767, 3928, 4840, 2979, 1386, 5183, 3935] avg=4096.0000 max=5750 max_rank=0 min=1386 min_rank=5 imbalance_ratio=1.403809 max_load_reduction=27018 max_load_reduction_pct=0.824524 moved_to_echo=13483 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 2597, 0, 0, 0, 70, 0, 0, 1652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 3, 685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3411, 534, 0, 0, 0, 0, 0, 0, 4, 1, 3911, 1, 0, 882, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 655, 0, 51, 106, 0, 5, 922, 1503, 1265, 0, 0, 0, 0, 0, 0, 977, 7, 25, 26, 109, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 1170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2662, 1353, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4319, 692, 3946, 4833, 4524, 5182, 5257, 4015] avg=4096.0000 max=5257 max_rank=6 min=692 min_rank=1 imbalance_ratio=1.283447 max_load_reduction=27511 max_load_reduction_pct=0.839569 moved_to_echo=18978 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 768, 0, 128, 4224, 0, 128, 20096, 7808, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[384, 1024, 128, 128, 12160, 128, 128, 3840, 8576, 6912, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[384, 128, 256, 128, 128, 3584, 256, 0, 24576, 4096, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[5376, 768, 14336, 128, 3200, 8576, 256, 640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 18944, 128, 128, 7168, 6784, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 22656, 256, 0, 0, 640, 0, 384, 8960, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 768, 128, 0, 128, 25472, 1408, 4992, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 23296, 512, 0, 6784, 2304, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[3, 0, 0, 2, 3615, 754, 1589, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 38, 0, 1, 4064, 1235, 0, 0, 0, 59, 2, 0, 1354, 0, 54, 3760, 0, 15, 4, 0, 165, 0, 0, 0, 0, 0, 242, 3, 0, 0, 0, 0, 4089, 0, 0, 0, 9, 4090, 0, 0, 0, 0, 3664, 3924, 0, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4, 71, 2, 0, 4095, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4092, 3551, 1, 0, 0, 10, 0, 905, 3646, 0, 0, 3965, 83, 0, 0, 0, 95, 0, 0, 0, 324, 0, 1183, 0, 0, 0, 0, 3876, 2632, 0, 0, 0, 0, 6, 0, 0, 0, 21, 4004, 52, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 151, 19, 0, 0, 0, 52, 5, 0, 0, 1, 0, 0, 2, 1, 0, 0, 0, 2501, 4096, 0, 0, 0, 462, 0, 0, 78, 0, 0, 3965, 89, 2, 0, 0, 0, 1075, 0, 0, 163, 3, 0, 0, 0, 0, 6, 1814, 979, 0, 0, 0, 0, 1702, 3, 0, 2, 28, 3973, 4083, 3418, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 54, 348, 8, 0, 503, 3193, 0, 0, 0, 0, 255, 0, 0, 0, 0, 0, 55, 331, 0, 2, 0, 3800, 1472, 0, 0, 0, 0, 447, 4011, 1, 0, 0, 4007, 0, 0, 0, 0, 6, 288, 0, 0, 142, 0, 0, 0, 28, 0, 0, 0, 3920, 1, 0, 0, 0, 1274, 3121, 0, 0, 0, 3959, 1534, 0, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 287, 1881, 7, 0, 4029, 241, 0, 0, 0, 0, 0, 0, 946, 0, 0, 1621, 172, 262, 3, 0, 2375, 671, 47, 0, 0, 80, 0, 2, 101, 0, 0, 3104, 2, 0, 1266, 0, 5, 0, 0, 0, 328, 0, 23, 0, 0, 5, 0, 0, 3940, 0, 2, 0, 0, 3974, 0, 0, 0, 29, 4016, 1980, 1369, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 3935, 147, 27, 2, 212, 0, 0, 19, 86, 2, 0, 0, 73, 0, 0, 656, 0, 0, 0, 0, 82, 4090, 0, 0, 0, 1, 33, 3850, 3478, 0, 0, 2616, 4, 3, 0, 0, 0, 1795, 0, 0, 7, 0, 0, 0, 97, 0, 0, 4019, 0, 0, 0, 0, 0, 85, 46, 0, 54, 0, 4091, 3256, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 87, 114, 0, 0, 4020, 3, 1, 221, 2, 0, 188, 0, 0, 0, 0, 3766, 0, 0, 0, 0, 3993, 86, 3820, 0, 339, 0, 8, 3, 1009, 0, 3263, 525, 8, 0, 0, 0, 3, 57, 0, 0, 3682, 0, 0, 4, 0, 0, 0, 0, 2170, 0, 63, 0, 4, 104, 12, 0, 1119, 0, 4091, 0, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 10, 3629, 3284, 0, 727, 349, 0, 0, 0, 0, 0, 0, 122, 20, 0, 2, 0, 0, 0, 0, 504, 3974, 12, 0, 0, 0, 3730, 0, 257, 0, 0, 3760, 2, 0, 0, 0, 629, 0, 0, 0, 2963, 0, 285, 0, 0, 0, 515, 0, 406, 0, 46, 0, 0, 3443, 0, 0, 0, 0, 4089, 10, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 54, 348, 8, 0, 503, 1604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 0, 0, 0, 0, 2879, 1345, 0, 0, 0, 0, 0, 0, 0, 55, 331, 0, 2, 0, 1186, 1472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 447, 4011, 1, 0, 566, 287, 0, 0, 0, 0, 0, 0, 0, 3720, 0, 0, 0, 0, 6, 288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 0, 0, 0, 28, 0, 2248, 1269, 1023, 0, 0, 0, 0, 0, 0, 0, 1672, 1, 0, 0, 0, 1274, 0, 0, 0, 0, 0, 0, 0, 0, 3121, 0, 0, 0, 1080, 1534, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2517, 4479, 3046, 5312, 4014, 4710, 2947, 5743] avg=4096.0000 max=5743 max_rank=7 min=2517 min_rank=0 imbalance_ratio=1.402100 max_load_reduction=27025 max_load_reduction_pct=0.824738 moved_to_echo=17414 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 151, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 5, 0, 0, 1, 0, 2890, 885, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 781, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 0, 0, 78, 0, 3, 284, 0, 0, 0, 0, 0, 0, 0, 3681, 89, 2, 0, 0, 0, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 163, 3, 0, 0, 0, 0, 561, 835, 6, 0, 0, 0, 0, 0, 6, 1814, 418, 0, 0, 0, 0, 1702, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 28, 1083, 4083, 3418, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4256, 3833, 4880, 827, 4847, 1568, 3940, 8617] avg=4096.0000 max=8617 max_rank=7 min=827 min_rank=3 imbalance_ratio=2.103760 max_load_reduction=24151 max_load_reduction_pct=0.737030 moved_to_echo=18972 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 2, 3615, 754, 798, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2665, 1438, 0, 0, 0, 0, 0, 0, 0, 1, 0, 38, 0, 1, 1269, 1235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 2, 0, 1354, 0, 282, 270, 0, 0, 0, 0, 0, 0, 54, 3490, 0, 15, 4, 0, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 242, 3, 0, 0, 2345, 1357, 509, 0, 0, 0, 0, 0, 0, 0, 1744, 0, 0, 0, 9, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 999, 3924, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5201, 4103, 2544, 1967, 3728, 4456, 5843, 4926] avg=4096.0000 max=5843 max_rank=6 min=1967 min_rank=3 imbalance_ratio=1.426514 max_load_reduction=26925 max_load_reduction_pct=0.821686 moved_to_echo=18953 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 71, 2, 0, 2057, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2912, 1448, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1277, 3551, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 10, 0, 905, 3646, 0, 726, 284, 0, 0, 0, 0, 0, 0, 0, 3681, 83, 0, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 324, 0, 1183, 0, 0, 0, 1509, 1367, 1312, 0, 0, 0, 0, 0, 0, 3876, 1123, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 1092, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2282, 4360, 4830, 5572, 3859, 5695, 5005, 1165] avg=4096.0000 max=5695 max_rank=5 min=1165 min_rank=7 imbalance_ratio=1.390381 max_load_reduction=27073 max_load_reduction_pct=0.826202 moved_to_echo=15724 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 287, 1881, 7, 0, 2023, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 946, 0, 2921, 840, 0, 0, 0, 0, 0, 0, 0, 1621, 172, 262, 3, 0, 742, 671, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 80, 0, 2, 101, 0, 715, 223, 0, 0, 0, 0, 0, 0, 0, 2881, 2, 0, 1266, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 328, 0, 23, 0, 0, 5, 2259, 793, 1291, 0, 0, 0, 0, 0, 0, 0, 1681, 0, 2, 0, 0, 3974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 1095, 1980, 1369, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4439, 4707, 3471, 1168, 4154, 4699, 5657, 4473] avg=4096.0000 max=5657 max_rank=6 min=1168 min_rank=3 imbalance_ratio=1.381104 max_load_reduction=27111 max_load_reduction_pct=0.827362 moved_to_echo=18983 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3935, 147, 27, 2, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 86, 2, 0, 0, 73, 0, 2975, 29, 0, 0, 0, 0, 0, 0, 0, 656, 0, 0, 0, 0, 26, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 33, 3850, 3478, 0, 37, 187, 0, 0, 0, 0, 0, 0, 0, 2429, 4, 3, 0, 0, 0, 1795, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 97, 0, 0, 27, 67, 0, 0, 0, 0, 0, 0, 4019, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 54, 0, 1116, 3256, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4219, 3184, 4772, 7586, 4231, 198, 4104, 4474] avg=4096.0000 max=7586 max_rank=3 min=198 min_rank=5 imbalance_ratio=1.852051 max_load_reduction=25182 max_load_reduction_pct=0.768494 moved_to_echo=13007 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 87, 114, 0, 0, 2019, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 221, 2, 0, 188, 0, 0, 0, 2975, 1413, 0, 0, 0, 0, 0, 0, 0, 3766, 0, 0, 0, 0, 1246, 86, 0, 0, 0, 0, 0, 0, 0, 0, 3820, 0, 339, 0, 8, 3, 1009, 0, 713, 37, 0, 0, 0, 0, 0, 0, 3263, 488, 8, 0, 0, 0, 3, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3682, 0, 0, 4, 0, 0, 1244, 1334, 1288, 0, 0, 0, 0, 0, 0, 0, 926, 0, 63, 0, 4, 104, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 1119, 0, 1116, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2223, 4800, 5098, 5929, 3819, 7552, 1097, 2250] avg=4096.0000 max=7552 max_rank=5 min=1097 min_rank=6 imbalance_ratio=1.843750 max_load_reduction=25216 max_load_reduction_pct=0.769531 moved_to_echo=14718 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 10, 3629, 3284, 0, 359, 349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 20, 2979, 181, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 151, 3974, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 3730, 0, 257, 0, 133, 275, 0, 0, 0, 0, 0, 0, 0, 3485, 2, 0, 0, 0, 629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2963, 0, 285, 0, 0, 0, 235, 172, 235, 0, 0, 0, 0, 0, 515, 0, 171, 0, 46, 0, 0, 3443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1110, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7631, 3302, 4127, 4407, 4116, 3890, 4175, 1120] avg=4096.0000 max=7631 max_rank=0 min=1120 min_rank=7 imbalance_ratio=1.863037 max_load_reduction=25137 max_load_reduction_pct=0.767120 moved_to_echo=13301 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 0, 7680, 128, 1792, 128, 128, 128, 10496, 7168, 5760, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3200, 0, 1280, 128, 8704, 14848, 4864, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 256, 256, 384, 256, 0, 1152, 128, 23296, 7680, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3968, 0, 384, 640, 4224, 8832, 9984, 0, 3200, 1920, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 6144, 512, 384, 128, 128, 6784, 19200, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 4736, 5888, 11136, 1536, 9088, 896, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[640, 9728, 7808, 128, 128, 0, 128, 14720, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3328, 23936, 256, 128, 1280, 0, 1024, 3328, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 16, 2960, 135, 0, 0, 0, 4094, 273, 0, 0, 4096, 0, 0, 0, 4072, 0, 0, 0, 0, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 81, 0, 1, 0, 0, 0, 4041, 0, 0, 2567, 9, 0, 0, 0, 0, 0, 1318, 6, 0, 5, 0, 0, 20, 36, 0, 0, 0, 4096, 0, 0, 378] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 4078, 0, 4032, 0, 2, 0, 0, 3964, 1, 0, 1201, 4096, 4, 0, 0, 138, 0, 59, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4069, 0, 0, 10, 0, 0, 0, 3637, 0, 0, 0, 0, 0, 0, 0, 444, 80, 2803, 8, 3, 0, 0, 0, 3927, 0, 12, 0, 0, 165, 0, 33, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[44, 0, 0, 0, 1405, 0, 0, 1, 0, 3603, 11, 78, 376, 4065, 0, 181, 1386, 0, 0, 0, 0, 0, 1, 0, 15, 0, 0, 1, 0, 0, 0, 0, 4095, 15, 4027, 0, 3837, 0, 0, 4075, 1566, 0, 98, 0, 0, 0, 0, 0, 0, 0, 20, 2465, 0, 0, 0, 881, 0, 0, 0, 0, 522, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 1542, 1392, 3711, 333, 0, 0, 0, 3409, 0, 0, 327, 4096, 0, 0, 10, 6, 0, 13, 3830, 0, 2, 0, 0, 0, 0, 0, 0, 226, 0, 0, 4081, 2, 0, 0, 250, 0, 0, 19, 0, 0, 0, 0, 4, 0, 0, 21, 24, 0, 3827, 6, 0, 0, 0, 723, 0, 2321, 0, 0, 2579, 0, 0, 14] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 3900, 3341, 4006, 62, 1, 0, 20, 3417, 0, 0, 94, 3984, 19, 0, 683, 1, 0, 254, 542, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 95, 3932, 0, 0, 0, 112, 0, 0, 1466, 0, 0, 0, 154, 0, 0, 0, 24, 3, 0, 1, 0, 0, 0, 0, 2904, 0, 0, 0, 0, 3751, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[4, 0, 0, 0, 98, 0, 121, 0, 38, 543, 0, 0, 0, 4090, 0, 498, 18, 3318, 0, 3829, 5, 0, 3992, 0, 1, 312, 0, 0, 0, 0, 0, 0, 3621, 1902, 0, 0, 0, 0, 0, 195, 1180, 0, 13, 78, 44, 3, 0, 4, 0, 109, 4039, 600, 0, 2, 0, 5, 0, 46, 0, 0, 4049, 8, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 3346, 3983, 4, 6, 1, 0, 3491, 13, 0, 0, 4096, 0, 926, 110, 3889, 0, 0, 815, 3, 2670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2605, 98, 0, 1, 10, 0, 17, 4067, 1029, 17, 1, 25, 0, 0, 0, 0, 0, 224, 0, 0, 11, 0, 0, 0, 2, 0, 0, 0, 1308, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 2737, 2, 0, 0, 0, 0, 0, 1407, 0, 0, 3, 4096, 0, 0, 1146, 193, 0, 53, 3919, 1944, 7, 0, 0, 0, 1, 0, 0, 7, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 5, 407, 0, 0, 0, 0, 0, 0, 29, 275, 0, 4080, 23, 0, 0, 0, 139, 0, 3545, 0, 672, 3955, 0, 30, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[44, 0, 0, 0, 808, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3603, 11, 78, 376, 596, 0, 181, 0, 0, 0, 0, 0, 0, 0, 0, 1386, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 1, 0, 0, 0, 0, 3469, 570, 0, 0, 0, 0, 0, 0, 657, 15, 4027, 0, 3837, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 1566, 0, 98, 0, 0, 0, 0, 0, 2868, 255, 0, 0, 0, 0, 0, 0, 0, 0, 20, 2465, 0, 0, 0, 881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 0, 0, 0, 342, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[853, 4845, 1387, 4055, 12611, 4787, 3366, 864] avg=4096.0000 max=12611 max_rank=4 min=853 min_rank=0 imbalance_ratio=3.078857 max_load_reduction=20157 max_load_reduction_pct=0.615143 moved_to_echo=21628 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 16, 1716, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 273, 0, 0, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2457, 0, 0, 0, 0, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3496, 570, 0, 0, 0, 0, 0, 0, 657, 81, 0, 1, 0, 0, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2567, 9, 0, 0, 0, 0, 2868, 531, 0, 0, 0, 0, 0, 0, 0, 1318, 6, 0, 5, 0, 0, 20, 1485, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 4096, 0, 0, 378, 713, 130, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1867, 4967, 2926, 4066, 4780, 5975, 2834, 5353] avg=4096.0000 max=5975 max_rank=5 min=1867 min_rank=0 imbalance_ratio=1.458740 max_load_reduction=26793 max_load_reduction_pct=0.817657 moved_to_echo=18942 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 0, 0, 58, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 543, 0, 0, 0, 599, 0, 498, 0, 0, 0, 0, 0, 0, 0, 0, 18, 2002, 0, 3829, 5, 0, 3992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 312, 0, 0, 0, 0, 0, 0, 3491, 504, 0, 0, 0, 0, 0, 0, 581, 1902, 0, 0, 0, 0, 0, 195, 0, 0, 0, 0, 0, 0, 0, 0, 1180, 0, 13, 78, 44, 3, 0, 4, 2536, 17, 0, 0, 0, 0, 0, 0, 0, 109, 4039, 600, 0, 2, 0, 5, 1210, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 4049, 8, 3, 0, 23, 106, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[183, 1678, 9846, 4308, 2678, 3875, 5965, 4235] avg=4096.0000 max=9846 max_rank=2 min=183 min_rank=0 imbalance_ratio=2.403809 max_load_reduction=22922 max_load_reduction_pct=0.699524 moved_to_echo=16753 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4078, 0, 2337, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3964, 1, 0, 1201, 600, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 59, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3496, 567, 0, 0, 0, 0, 0, 0, 653, 0, 0, 10, 0, 0, 0, 3637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 444, 2849, 723, 0, 0, 0, 0, 0, 0, 80, 2803, 8, 3, 0, 0, 0, 3927, 50, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 165, 0, 33, 0, 972, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6417, 5770, 144, 4064, 4300, 4016, 6871, 1186] avg=4096.0000 max=6871 max_rank=6 min=144 min_rank=2 imbalance_ratio=1.677490 max_load_reduction=25897 max_load_reduction_pct=0.790314 moved_to_echo=16373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1542, 1392, 2151, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3409, 0, 0, 327, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4, 0, 13, 3830, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 226, 0, 0, 3496, 568, 0, 0, 0, 0, 0, 0, 655, 2, 0, 0, 250, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 21, 2858, 666, 0, 0, 0, 0, 0, 0, 24, 0, 3827, 6, 0, 0, 0, 723, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2321, 0, 0, 2579, 0, 0, 14, 894, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5418, 4336, 3859, 4290, 926, 3549, 4582, 5808] avg=4096.0000 max=5808 max_rank=7 min=926 min_rank=4 imbalance_ratio=1.417969 max_load_reduction=26960 max_load_reduction_pct=0.822754 moved_to_echo=14865 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3900, 3341, 2321, 62, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 3417, 0, 0, 94, 584, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 683, 1, 0, 254, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 95, 3400, 548, 0, 0, 0, 0, 0, 0, 631, 0, 0, 0, 112, 0, 0, 1466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 0, 0, 0, 24, 2753, 719, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 2904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3751, 0, 0, 0, 966, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9625, 4134, 1480, 4045, 2209, 3650, 2908, 4717] avg=4096.0000 max=9625 max_rank=0 min=1480 min_rank=2 imbalance_ratio=2.349854 max_load_reduction=23143 max_load_reduction_pct=0.706268 moved_to_echo=13484 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3346, 2309, 4, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3491, 13, 0, 0, 600, 0, 926, 0, 0, 0, 0, 0, 0, 0, 0, 110, 2347, 0, 0, 815, 3, 2670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3496, 363, 0, 0, 0, 0, 0, 0, 418, 98, 0, 1, 10, 0, 17, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 17, 1, 25, 0, 0, 0, 0, 1824, 714, 0, 0, 0, 0, 0, 0, 0, 224, 0, 0, 11, 0, 0, 0, 1418, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1308, 0, 0, 0, 960, 124, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5666, 5030, 5945, 3859, 4611, 3610, 1653, 2394] avg=4096.0000 max=5945 max_rank=2 min=1653 min_rank=6 imbalance_ratio=1.451416 max_load_reduction=26823 max_load_reduction_pct=0.818573 moved_to_echo=12268 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2737, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1407, 0, 0, 3, 598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1146, 112, 0, 53, 3919, 1944, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7, 0, 0, 3498, 575, 0, 0, 0, 0, 0, 0, 648, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 407, 0, 0, 0, 0, 0, 0, 29, 2870, 0, 0, 0, 0, 0, 0, 0, 275, 0, 4080, 23, 0, 0, 0, 139, 72, 0, 0, 0, 0, 0, 0, 0, 0, 3545, 0, 672, 3955, 0, 30, 0, 0, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2739, 2008, 7181, 4081, 653, 3306, 4589, 8211] avg=4096.0000 max=8211 max_rank=7 min=653 min_rank=4 imbalance_ratio=2.004639 max_load_reduction=24557 max_load_reduction_pct=0.749420 moved_to_echo=16759 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3456, 7040, 0, 4224, 9216, 2048, 7168, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 12288, 8192, 11776, 640, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4992, 2176, 4096, 128, 4224, 0, 128, 17536, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 6016, 0, 768, 20480, 128, 128, 512, 4992, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 23936, 384, 128, 2048, 4864, 128, 1664, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 128, 2688, 384, 128, 128, 0, 640, 21504, 3712, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[384, 4480, 12032, 3200, 128, 128, 0, 8704, 4352, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 384, 128, 128, 0, 256, 0, 128, 27904, 4352, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 29, 0, 381, 4012, 0, 0, 0, 0, 27, 4076, 0, 408, 3870, 25, 3937, 3, 0, 0, 0, 1607, 3943, 0, 0, 0, 134, 0, 0, 2, 0, 13, 0, 0, 0, 0, 7, 4, 0, 0, 0, 0, 0, 0, 2008, 4068, 128, 793, 0, 0, 5, 0, 214, 0, 0, 0, 0, 2945, 0, 0, 125, 0, 0, 0, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1613, 0, 0, 4, 4095, 0, 0, 40, 0, 2363, 2, 44, 227, 3127, 0, 3862, 251, 0, 0, 0, 929, 4079, 0, 0, 2371, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2306, 8, 0, 0, 0, 0, 271, 4096, 0, 1, 0, 17, 3044, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 7, 346, 42, 3940, 0, 0, 0, 4, 2417, 64, 0, 2572, 4061, 0, 4096, 0, 0, 0, 0, 0, 3898, 0, 0, 2845, 0, 2, 817, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 296, 4, 0, 0, 0, 0, 48, 32, 0, 2551, 0, 0, 0, 635, 23, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 132, 9, 74, 4096, 0, 0, 0, 0, 74, 0, 140, 2149, 3887, 34, 2, 0, 0, 0, 0, 4, 3664, 0, 8, 2480, 11, 0, 292, 50, 4078, 248, 0, 294, 1, 0, 3374, 0, 1799, 0, 18, 0, 1, 2, 0, 3324, 0, 708, 0, 1499, 0, 0, 23, 0, 0, 148, 142, 0, 0, 3, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 1560, 0, 3769, 0, 0, 0, 2751, 3815, 1570, 0, 4009, 4022, 0, 4092, 18, 0, 0, 0, 0, 1132, 1, 0, 0, 0, 0, 0, 0, 13, 0, 0, 12, 0, 0, 4, 0, 0, 27, 0, 0, 0, 0, 14, 183, 76, 2199, 0, 3492, 0, 1, 7, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[13, 0, 372, 2, 4095, 0, 0, 0, 0, 1, 7, 19, 1882, 3863, 0, 3777, 0, 0, 0, 0, 0, 4096, 0, 0, 4063, 0, 14, 0, 0, 604, 0, 0, 4072, 3, 0, 7, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 1561, 0, 38, 9, 144, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 11, 3378, 2308, 0, 0, 0, 16, 2921, 16, 0, 32, 0, 2, 191, 0, 0, 0, 0, 66, 4086, 0, 0, 3916, 93, 0, 0, 0, 3941, 69, 0, 0, 3722, 0, 49, 0, 269, 0, 0, 0, 4, 0, 3680, 0, 1, 0, 0, 7, 3110, 153, 17, 1, 0, 0, 570, 0, 0, 0, 139, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[14, 0, 71, 0, 4096, 0, 133, 807, 0, 4092, 3533, 1, 0, 4095, 0, 3925, 0, 0, 0, 0, 0, 4096, 0, 0, 96, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 1842, 0, 3, 0, 0, 0, 0, 0, 232, 13, 0, 0, 1, 77, 1772, 481, 3384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 132, 9, 74, 3197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 140, 2149, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3664, 0, 8, 4, 0, 0, 0, 0, 0, 0, 0, 2480, 11, 0, 292, 50, 4078, 248, 0, 489, 13, 0, 0, 0, 0, 0, 0, 294, 1, 0, 3374, 0, 1799, 0, 18, 0, 410, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 3324, 0, 708, 0, 12, 0, 0, 0, 0, 0, 0, 0, 1499, 0, 0, 23, 0, 0, 148, 142, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3887, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3412, 2368, 3680, 7661, 5896, 4047, 1814, 3890] avg=4096.0000 max=7661 max_rank=3 min=1814 min_rank=6 imbalance_ratio=1.870361 max_load_reduction=25107 max_load_reduction_pct=0.766205 moved_to_echo=15647 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 29, 0, 381, 3132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4076, 0, 408, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1607, 3943, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 134, 0, 0, 2, 0, 13, 0, 479, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4, 0, 0, 0, 1244, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2008, 4068, 128, 793, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 214, 0, 0, 0, 0, 2264, 0, 0, 0, 0, 0, 0, 0, 2945, 0, 0, 125, 0, 0, 0, 4, 3870, 429, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3542, 4526, 5554, 633, 1656, 7001, 2483, 7373] avg=4096.0000 max=7373 max_rank=7 min=633 min_rank=3 imbalance_ratio=1.800049 max_load_reduction=25395 max_load_reduction_pct=0.774994 moved_to_echo=18513 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1613, 0, 0, 4, 3196, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1383, 2, 44, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 0, 0, 0, 929, 4079, 0, 0, 133, 0, 0, 0, 0, 0, 0, 0, 2371, 0, 0, 16, 0, 0, 0, 0, 489, 445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2306, 8, 0, 1221, 410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 271, 4096, 0, 1, 0, 402, 0, 0, 0, 0, 0, 0, 0, 17, 3044, 0, 0, 0, 0, 0, 0, 2222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 3127, 419, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4853, 1656, 5392, 3321, 3945, 4770, 5283, 3548] avg=4096.0000 max=5392 max_rank=2 min=1656 min_rank=1 imbalance_ratio=1.316406 max_load_reduction=27376 max_load_reduction_pct=0.835449 moved_to_echo=17546 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 346, 42, 3076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1415, 64, 0, 2572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3898, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 2845, 0, 2, 817, 0, 0, 0, 0, 470, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 296, 4, 0, 1295, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 32, 0, 2551, 0, 411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 635, 23, 0, 0, 19, 0, 2356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 445, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3471, 4055, 4034, 4589, 6038, 3042, 3033, 4506] avg=4096.0000 max=6038 max_rank=4 min=3033 min_rank=6 imbalance_ratio=1.474121 max_load_reduction=26730 max_load_reduction_pct=0.815735 moved_to_echo=16619 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[13, 0, 372, 2, 3196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 19, 1882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 14, 0, 0, 604, 0, 0, 489, 0, 0, 0, 0, 0, 0, 0, 4072, 3, 0, 7, 0, 0, 1, 0, 1194, 410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 9, 144, 0, 0, 0, 0, 0, 2173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 3863, 410, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3583, 1909, 4096, 5170, 5687, 5657, 2364, 4302] avg=4096.0000 max=5687 max_rank=4 min=1909 min_rank=1 imbalance_ratio=1.388428 max_load_reduction=27081 max_load_reduction_pct=0.826447 moved_to_echo=18010 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1560, 0, 2942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2751, 2231, 1570, 0, 4009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 1132, 1, 0, 215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 450, 719, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4, 0, 0, 27, 0, 1293, 377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 183, 76, 2199, 0, 650, 0, 0, 0, 0, 0, 0, 0, 3492, 0, 1, 7, 0, 0, 0, 0, 2353, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4022, 446, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4502, 10561, 1366, 1182, 1713, 3122, 5853, 4469] avg=4096.0000 max=10561 max_rank=1 min=1182 min_rank=3 imbalance_ratio=2.578369 max_load_reduction=22207 max_load_reduction_pct=0.677704 moved_to_echo=15157 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 11, 3378, 1802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 1710, 16, 0, 32, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 4086, 0, 0, 164, 0, 0, 0, 0, 0, 0, 0, 3916, 93, 0, 0, 0, 3941, 69, 0, 275, 550, 0, 0, 0, 0, 0, 0, 0, 3722, 0, 49, 0, 269, 0, 0, 62, 231, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3680, 0, 1, 0, 0, 497, 0, 0, 0, 0, 0, 0, 0, 7, 3110, 153, 17, 1, 0, 0, 570, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5191, 1776, 4316, 8844, 4333, 4182, 3967, 159] avg=4096.0000 max=8844 max_rank=3 min=159 min_rank=7 imbalance_ratio=2.159180 max_load_reduction=23924 max_load_reduction_pct=0.730103 moved_to_echo=12641 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[14, 0, 71, 0, 3189, 0, 133, 807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2383, 3533, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 234, 0, 0, 0, 0, 0, 0, 0, 96, 0, 3, 0, 1, 0, 0, 0, 494, 774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1842, 0, 3, 0, 0, 1242, 413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 232, 13, 0, 0, 1, 701, 0, 0, 0, 0, 0, 0, 0, 77, 1772, 481, 3384, 0, 0, 0, 0, 2257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 426, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4214, 5917, 4330, 1368, 3500, 947, 7971, 4521] avg=4096.0000 max=7971 max_rank=6 min=947 min_rank=5 imbalance_ratio=1.946045 max_load_reduction=24797 max_load_reduction_pct=0.756744 moved_to_echo=16939 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[15872, 256, 128, 1152, 128, 8704, 384, 0, 3712, 3072, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4480, 3840, 0, 9344, 128, 4736, 128, 128, 7552, 3072, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2816, 9216, 9344, 256, 11392, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[5248, 8064, 1536, 3712, 128, 0, 256, 768, 13824, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1664, 256, 2432, 3968, 23808, 0, 256, 896, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 6272, 15872, 256, 7936, 128, 2688, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3072, 0, 128, 384, 0, 0, 0, 128, 27008, 2688, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[384, 0, 0, 0, 2688, 29056, 128, 128, 896, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3241, 0, 1924, 0, 0, 0, 1, 161, 0, 0, 0, 0, 0, 3463, 1117, 0, 0, 0, 3, 0, 0, 0, 4087, 0, 0, 39, 0, 0, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 0, 1, 2275, 2, 510, 0, 0, 28, 0, 0, 0, 2546, 1162, 1, 4096, 0, 3930, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 5, 53, 0, 1418, 0, 3692, 0, 0, 4, 51, 0, 0, 22, 0, 232, 2292, 0, 0, 0, 4089, 2692, 0, 0, 4080, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 6, 0, 0, 0, 0, 372, 0, 0, 0, 92, 0, 28, 3907, 0, 0, 0, 0, 1, 0, 1705, 0, 4096, 0, 3929, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4039, 0, 0, 0, 0, 28, 0, 4067, 1, 97, 0, 51, 0, 0, 0, 0, 301, 0, 6, 4049, 0, 0, 0, 2, 255, 3465, 0, 0, 5, 1, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 1721, 0, 305, 225, 466, 0, 3775, 0, 266, 0, 0, 0, 0, 0, 1716, 0, 3854, 0, 4054, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 18, 3816, 0, 4001, 0, 0, 0, 0, 0, 133, 0, 0, 24, 0, 154, 4046, 0, 0, 0, 1, 99, 0, 0, 4089, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 189, 0, 0, 0, 3, 20, 0, 0, 177, 2597, 0, 17, 4068, 0, 0, 0, 0, 0, 3, 867, 0, 4096, 0, 4076, 235] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 3326, 0, 0, 202, 1069, 130, 0, 0, 0, 0, 2, 0, 0, 872, 33, 0, 21, 742, 3630, 0, 0, 1221, 3633, 0, 5, 7, 0, 0, 56, 345, 0, 0, 0, 0, 0, 0, 0, 84, 0, 1, 0, 257, 1, 0, 3985, 3907, 0, 0, 0, 0, 1, 4, 1163, 0, 3982, 1, 4087, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 1, 2368, 0, 3, 4096, 0, 800, 0, 3661, 0, 0, 0, 0, 0, 0, 24, 0, 0, 1, 0, 12, 0, 842, 0, 0, 0, 4095, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 169, 0, 0, 0, 0, 4040, 0, 0, 0, 283, 0, 65, 4083, 0, 0, 12, 0, 0, 4, 2, 0, 3994, 0, 4067, 144] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4, 701, 0, 11, 9, 1, 0, 70, 3135, 0, 0, 0, 43, 0, 0, 477, 0, 4028, 2189, 0, 1503, 4, 35, 126, 0, 0, 3817, 0, 0, 0, 0, 0, 0, 340, 0, 0, 0, 0, 2, 0, 6, 0, 0, 0, 0, 0, 4043, 0, 0, 2029, 0, 11, 0, 6, 0, 1, 3163, 0, 2921, 15, 4078, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 0, 1200, 0, 2974, 0, 4090, 0, 1, 35, 0, 0, 0, 0, 0, 0, 3477, 0, 1715, 0, 3930, 4, 0, 0, 4040, 0, 0, 25, 0, 0, 266, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2437, 0, 0, 438, 5, 0, 0, 0, 36, 3, 4045, 0, 4030, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4039, 0, 0, 0, 0, 767, 0, 0, 0, 0, 0, 0, 0, 28, 0, 4067, 1, 97, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 301, 0, 6, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 255, 1744, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 19, 0, 0, 0, 3919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1721, 0, 305, 225, 1721, 607, 135, 0, 0, 0, 0, 0, 466, 0, 3775, 0, 266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1716, 0, 2480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4806, 4244, 4356, 2006, 3939, 4714, 4507, 4196] avg=4096.0000 max=4806 max_rank=0 min=2006 min_rank=3 imbalance_ratio=1.173340 max_load_reduction=27962 max_load_reduction_pct=0.853333 moved_to_echo=17356 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3241, 0, 815, 0, 0, 0, 0, 0, 0, 0, 1924, 0, 0, 0, 1, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3463, 1117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 2057, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 0, 0, 0, 0, 0, 3800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 0, 1, 2030, 645, 130, 0, 0, 0, 0, 0, 2275, 2, 510, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2546, 1162, 1, 2636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4056, 2086, 4580, 2099, 3948, 6839, 2815, 6345] avg=4096.0000 max=6839 max_rank=5 min=2086 min_rank=1 imbalance_ratio=1.669678 max_load_reduction=25929 max_load_reduction_pct=0.791290 moved_to_echo=19947 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 2368, 0, 3, 4096, 0, 795, 834, 0, 0, 0, 0, 0, 0, 800, 0, 3661, 0, 0, 0, 0, 0, 210, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 1, 0, 12, 0, 445, 0, 0, 0, 0, 0, 0, 0, 842, 0, 0, 0, 2061, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 3932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4040, 0, 0, 0, 2034, 629, 135, 0, 0, 0, 0, 0, 283, 0, 65, 2594, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 2570, 0, 0, 144, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8097, 4671, 482, 2905, 4101, 6838, 2954, 2720] avg=4096.0000 max=8097 max_rank=0 min=482 min_rank=2 imbalance_ratio=1.976807 max_load_reduction=24671 max_load_reduction_pct=0.752899 moved_to_echo=16613 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 701, 0, 11, 9, 1, 581, 414, 0, 0, 0, 0, 0, 0, 0, 70, 3135, 0, 0, 0, 43, 0, 104, 0, 0, 0, 0, 0, 0, 0, 0, 477, 0, 4028, 2189, 0, 1503, 4, 221, 0, 0, 0, 0, 0, 0, 0, 35, 126, 0, 0, 1922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 340, 0, 0, 0, 0, 3942, 0, 0, 0, 0, 0, 0, 0, 2, 0, 6, 0, 0, 0, 0, 0, 1895, 460, 136, 0, 0, 0, 0, 0, 4043, 0, 0, 1290, 0, 11, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3163, 0, 1880, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1721, 3352, 8422, 2083, 4282, 2499, 5350, 5059] avg=4096.0000 max=8422 max_rank=2 min=1721 min_rank=0 imbalance_ratio=2.056152 max_load_reduction=24346 max_load_reduction_pct=0.742981 moved_to_echo=17190 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 3326, 0, 793, 798, 0, 0, 0, 0, 0, 0, 0, 202, 1069, 130, 0, 0, 0, 0, 201, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 872, 33, 0, 21, 742, 426, 0, 0, 0, 0, 0, 0, 0, 3630, 0, 0, 1221, 1829, 0, 5, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 345, 0, 0, 0, 0, 3950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 1, 0, 257, 1804, 627, 137, 0, 0, 0, 0, 0, 1, 0, 3985, 2482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 1163, 0, 2562, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4918, 1602, 2096, 6692, 4351, 2910, 6468, 3731] avg=4096.0000 max=6692 max_rank=3 min=1602 min_rank=1 imbalance_ratio=1.633789 max_load_reduction=26076 max_load_reduction_pct=0.795776 moved_to_echo=17460 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 18, 3816, 0, 821, 833, 0, 0, 0, 0, 0, 0, 4001, 0, 0, 0, 0, 0, 133, 0, 213, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 154, 4046, 0, 0, 0, 444, 0, 0, 0, 0, 0, 0, 0, 1, 99, 0, 0, 2055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 189, 3941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 20, 0, 0, 177, 2034, 647, 135, 0, 0, 0, 0, 0, 2597, 0, 17, 2578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 867, 0, 2628, 0, 0, 235, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5490, 4347, 4668, 2155, 4167, 3016, 5192, 3733] avg=4096.0000 max=5490 max_rank=0 min=2155 min_rank=3 imbalance_ratio=1.340332 max_load_reduction=27278 max_load_reduction_pct=0.832458 moved_to_echo=16108 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 5, 53, 0, 815, 798, 0, 0, 0, 0, 0, 0, 1418, 0, 3692, 0, 0, 4, 51, 0, 201, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 232, 2292, 0, 0, 0, 426, 0, 0, 0, 0, 0, 0, 0, 4089, 2692, 0, 0, 2054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 6, 3798, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 372, 0, 0, 0, 2026, 645, 131, 0, 0, 0, 0, 0, 92, 0, 28, 2482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1705, 0, 2636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1672, 5366, 2972, 8835, 3805, 3174, 2602, 4342] avg=4096.0000 max=8835 max_rank=3 min=1672 min_rank=0 imbalance_ratio=2.156982 max_load_reduction=23933 max_load_reduction_pct=0.730377 moved_to_echo=13923 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 0, 1200, 0, 805, 0, 0, 0, 0, 0, 0, 0, 2974, 0, 4090, 0, 1, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3477, 0, 1715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3930, 4, 0, 0, 2034, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, 14, 0, 0, 0, 0, 3895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2006, 637, 135, 0, 0, 0, 0, 0, 0, 0, 2437, 0, 0, 438, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 3, 2603, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2008, 7100, 5192, 5993, 4175, 2778, 2880, 2642] avg=4096.0000 max=7100 max_rank=1 min=2008 min_rank=0 imbalance_ratio=1.733398 max_load_reduction=25668 max_load_reduction_pct=0.783325 moved_to_echo=12475 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 0, 128, 128, 10240, 128, 384, 768, 15616, 4992, 1152, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 640, 0, 9088, 13184, 128, 7424, 768, 2048, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 128, 7168, 0, 128, 15744, 128, 6272, 3712, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[12544, 2944, 128, 1536, 15872, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 2560, 9856, 128, 20096, 128, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[9856, 128, 10880, 11520, 384, 512, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[11264, 384, 19840, 256, 128, 256, 384, 0, 1024, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 512, 768, 128, 0, 0, 384, 31232, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 5, 142, 0, 0, 0, 4037, 0, 97, 0, 4055, 0, 3773, 0, 13, 0, 1, 1, 0, 36, 4003, 9, 0, 0, 0, 0, 0, 2, 2180, 11, 3872, 2553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3682, 5, 0, 0, 1847, 0, 0, 0, 0, 0, 0, 0, 2, 67, 4, 0, 0, 0, 1616, 0, 755] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 2368, 0, 0, 0, 4090, 18, 2, 0, 9, 0, 4002, 0, 0, 0, 99, 0, 0, 318, 545, 0, 0, 0, 0, 0, 0, 10, 2, 0, 5, 4088, 1274, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 3914, 0, 3794, 4048, 3651, 0, 0, 0, 0, 0, 0, 0, 7, 428, 0, 0, 0, 0, 90, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[31, 0, 0, 11, 0, 147, 2344, 566, 332, 0, 1267, 0, 1504, 0, 48, 0, 0, 0, 0, 0, 3418, 6, 0, 0, 0, 0, 0, 0, 0, 0, 3864, 84, 0, 1, 0, 0, 4, 0, 0, 0, 0, 0, 2439, 206, 0, 9, 3921, 3982, 0, 0, 0, 0, 4, 792, 9, 7, 0, 0, 0, 118, 0, 3558, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 1, 158, 0, 0, 0, 3983, 5, 413, 0, 3144, 0, 4088, 0, 0, 0, 410, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4061, 4048, 3109, 0, 0, 6, 0, 0, 0, 784, 0, 0, 19, 138, 197, 4096, 0, 0, 0, 3, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 75, 3041, 0, 4008, 0, 0, 98, 0, 0, 4096, 0, 0, 0, 52, 2, 0, 0, 2137, 2, 0, 0, 0, 0, 0, 0, 0, 3, 1, 3281, 7, 0, 0, 0, 17, 0, 0, 2127, 16, 4071, 2, 3950, 0, 726, 3372, 43, 0, 0, 0, 0, 0, 1641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 56, 0, 0, 361, 0, 4096, 0, 0, 0, 3451, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3880, 0, 0, 0, 0, 6, 0, 0, 51, 0, 1969, 2360, 311, 2, 0, 478, 0, 0, 18, 0, 0, 0, 0, 4082, 0, 3198, 4, 13, 2, 0, 0, 2545, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1787] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 3, 1, 0, 0, 412, 3, 2, 0, 1, 0, 4093, 0, 0, 0, 2, 0, 0, 4, 29, 0, 1, 0, 0, 0, 2065, 39, 452, 0, 4037, 3011, 0, 4002, 0, 19, 0, 0, 0, 26, 0, 0, 0, 4085, 116, 4096, 199, 3077, 0, 0, 0, 0, 0, 0, 0, 2920, 0, 0, 0, 0, 73, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3168, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 1750, 9, 0, 598, 0, 4067, 4043, 7, 1253, 0, 0, 0, 206, 32, 0, 0, 473, 0, 1542, 0, 2848, 0, 4096, 2, 0, 0, 435, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 20] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2368, 0, 0, 0, 3896, 18, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 9, 0, 2756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 0, 0, 318, 545, 0, 0, 0, 766, 616, 194, 0, 0, 0, 0, 0, 0, 0, 0, 10, 2, 0, 5, 647, 0, 0, 0, 0, 0, 0, 0, 0, 1274, 0, 0, 6, 0, 0, 0, 0, 3158, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 3035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3914, 479, 0, 0, 0, 0, 0, 0, 428, 0, 0, 0, 0, 90, 0, 0, 2962, 636, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6282, 2767, 2538, 664, 4918, 7083, 4400, 4116] avg=4096.0000 max=7083 max_rank=5 min=664 min_rank=3 imbalance_ratio=1.729248 max_load_reduction=25685 max_load_reduction_pct=0.783844 moved_to_echo=20517 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[31, 0, 0, 11, 0, 147, 2233, 566, 0, 0, 0, 0, 0, 0, 0, 0, 332, 0, 1267, 0, 1036, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3418, 6, 0, 0, 288, 675, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3864, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 0, 0, 0, 8, 180, 0, 0, 0, 0, 0, 0, 0, 0, 2439, 0, 0, 0, 3921, 3307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 792, 9, 7, 206, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 0, 3558, 0, 4096, 60, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2988, 2683, 4498, 3879, 193, 9667, 1027, 7833] avg=4096.0000 max=9667 max_rank=5 min=193 min_rank=4 imbalance_ratio=2.360107 max_load_reduction=23101 max_load_reduction_pct=0.704987 moved_to_echo=18720 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 75, 3041, 0, 3817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 2821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 2, 0, 0, 2137, 2, 0, 0, 784, 7, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 520, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 17, 0, 0, 2127, 604, 491, 0, 0, 0, 0, 0, 0, 16, 4071, 2, 0, 0, 0, 3372, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1641, 0, 0, 3950, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2377, 122, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6933, 2919, 3175, 524, 3246, 7497, 5975, 2499] avg=4096.0000 max=7497 max_rank=5 min=524 min_rank=3 imbalance_ratio=1.830322 max_load_reduction=25271 max_load_reduction_pct=0.771210 moved_to_echo=19217 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 56, 0, 0, 361, 0, 3901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3451, 0, 2821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3880, 0, 0, 0, 784, 2, 195, 0, 0, 0, 0, 0, 0, 6, 0, 0, 51, 0, 1969, 374, 0, 0, 0, 0, 0, 0, 0, 0, 311, 2, 0, 478, 0, 0, 18, 0, 2661, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 11, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2545, 0, 0, 0, 1, 4082, 276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1787, 1710, 537, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4318, 6272, 4861, 2400, 3961, 15, 6906, 4035] avg=4096.0000 max=6906 max_rank=6 min=15 min_rank=5 imbalance_ratio=1.686035 max_load_reduction=25862 max_load_reduction_pct=0.789246 moved_to_echo=14917 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 607, 692, 195, 0, 0, 0, 0, 0, 0, 1750, 9, 0, 598, 0, 4067, 641, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1253, 0, 0, 0, 206, 32, 0, 2370, 380, 0, 0, 0, 0, 0, 0, 0, 473, 0, 0, 0, 0, 0, 3404, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 435, 0, 0, 0, 29, 1542, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 2929, 478, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3901, 2181, 5588, 7065, 4248, 3877, 2481, 3427] avg=4096.0000 max=7065 max_rank=3 min=2181 min_rank=1 imbalance_ratio=1.724854 max_load_reduction=25703 max_load_reduction_pct=0.784393 moved_to_echo=14033 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 1, 0, 0, 393, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 2818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4, 29, 0, 1, 0, 784, 519, 19, 0, 0, 0, 0, 0, 0, 0, 2065, 39, 452, 0, 4037, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4002, 0, 19, 0, 0, 0, 26, 3408, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 0, 199, 2558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2920, 4085, 352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 2181, 688, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[400, 2821, 1358, 7071, 7946, 2873, 7357, 2942] avg=4096.0000 max=7946 max_rank=4 min=400 min_rank=0 imbalance_ratio=1.939941 max_load_reduction=24822 max_load_reduction_pct=0.757507 moved_to_echo=21118 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 158, 0, 0, 0, 3790, 5, 0, 0, 0, 0, 0, 0, 0, 0, 413, 0, 3144, 0, 2806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 410, 0, 0, 0, 4083, 0, 0, 0, 787, 0, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4061, 632, 0, 0, 0, 0, 0, 0, 0, 0, 3109, 0, 0, 6, 0, 0, 0, 784, 3409, 495, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 19, 0, 0, 0, 0, 138, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2937, 687, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3954, 6363, 5473, 4696, 7803, 216, 639, 3624] avg=4096.0000 max=7803 max_rank=4 min=216 min_rank=5 imbalance_ratio=1.905029 max_load_reduction=24965 max_load_reduction_pct=0.761871 moved_to_echo=12282 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 142, 0, 0, 0, 3845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 4055, 0, 2597, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 36, 4003, 9, 0, 0, 723, 312, 192, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2180, 11, 3872, 404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 1535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3682, 299, 0, 0, 0, 0, 0, 0, 67, 4, 0, 0, 0, 1616, 0, 755, 1850, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3992, 6762, 5277, 6469, 453, 1540, 3983, 4292] avg=4096.0000 max=6762 max_rank=1 min=453 min_rank=4 imbalance_ratio=1.650879 max_load_reduction=26006 max_load_reduction_pct=0.793640 moved_to_echo=10268 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4736, 5376, 0, 512, 128, 256, 128, 2944, 15744, 3584, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 1792, 2176, 128, 3328, 128, 21888, 3712, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 4608, 2560, 0, 384, 0, 11648, 13952, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[896, 128, 12032, 0, 19840, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 0, 3072, 128, 2560, 128, 3072, 21632, 2816, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[640, 128, 0, 384, 22272, 128, 128, 0, 5632, 2944, 1408, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 2688, 128, 3456, 256, 25856, 640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[512, 128, 0, 128, 128, 5376, 0, 6784, 17024, 3200, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[4095, 27, 0, 636, 295, 0, 3834, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3653, 0, 3930, 125, 0, 4, 0, 30, 0, 0, 1, 0, 4086, 0, 0, 2, 5, 0, 0, 0, 4096, 2, 2, 3413, 3, 0, 0, 92, 101, 0, 0, 0, 0, 4084, 0, 251, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[4095, 4052, 0, 2614, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 2, 0, 14, 0, 0, 1, 0, 0, 0, 3816, 4096, 0, 0, 21, 265, 0, 0, 1487, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[3985, 0, 0, 4081, 1, 0, 3608, 0, 0, 0, 0, 5, 0, 0, 17, 0, 0, 3378, 0, 835, 2, 0, 0, 576, 0, 0, 0, 0, 0, 0, 3430, 0, 0, 0, 42, 4, 11, 0, 3991, 0, 3, 0, 4058, 131, 0, 0, 0, 9, 1, 0, 0, 0, 33, 0, 0, 117, 3526, 290, 34, 172, 0, 177, 251, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 2176, 35, 0, 0, 0, 3848, 1, 368, 0, 283, 0, 2025, 34, 0, 13, 0, 0, 0, 0, 0, 0, 0, 2877, 77, 3956, 5, 180, 0, 903, 4073, 0, 0, 361, 0, 2837, 0, 262, 0, 3, 0, 2, 909, 4073, 0, 0, 0, 0, 0, 0, 0, 243, 0, 17, 0, 0, 0, 0, 0, 0, 3130, 0, 0, 77] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[4050, 80, 22, 401, 0, 3, 4081, 0, 0, 0, 50, 1168, 0, 0, 0, 0, 0, 224, 0, 0, 1, 0, 0, 3975, 0, 3817, 0, 0, 0, 72, 2736, 0, 36, 0, 0, 10, 0, 0, 4055, 1, 0, 0, 23, 34, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3874, 0, 0, 3763, 291, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[4082, 4096, 469, 2806, 0, 0, 4090, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 7, 0, 0, 3, 0, 0, 0, 0, 75, 0, 0, 0, 10, 582, 0, 0, 0, 0, 1, 0, 0, 2002, 0, 0, 0, 4093, 2620, 0, 920, 1, 4000, 0, 0, 45, 0, 5, 0, 0, 0, 1471, 0, 0, 3, 6, 0, 1348, 11] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[3897, 1830, 8, 328, 0, 71, 2776, 0, 0, 0, 1, 47, 0, 0, 0, 0, 0, 0, 0, 0, 189, 0, 0, 3733, 0, 3443, 0, 0, 0, 10, 788, 0, 3, 0, 0, 4083, 0, 0, 4024, 0, 0, 0, 2676, 3664, 0, 0, 0, 481, 0, 0, 3, 0, 18, 146, 1, 0, 0, 0, 8, 2, 1, 328, 201, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[3785, 0, 307, 2951, 5, 0, 4082, 0, 0, 2, 0, 531, 0, 0, 0, 0, 0, 24, 6, 0, 0, 0, 0, 0, 0, 1379, 4, 4, 0, 8, 4000, 0, 0, 0, 0, 305, 0, 0, 0, 0, 72, 1258, 1482, 4058, 0, 0, 189, 4063, 35, 0, 0, 0, 0, 0, 0, 13, 4043, 1, 0, 125, 0, 0, 0, 36] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 2176, 35, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 368, 0, 283, 0, 2025, 34, 0, 13, 0, 1007, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2877, 2376, 0, 0, 0, 0, 0, 0, 0, 77, 3012, 5, 180, 0, 903, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 361, 0, 2837, 0, 262, 0, 3, 723, 509, 0, 0, 0, 0, 0, 0, 0, 2, 909, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 0, 17, 0, 0, 3847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3130, 0, 0, 77, 221, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2212, 3731, 5253, 8250, 4695, 1092, 4107, 3428] avg=4096.0000 max=8250 max_rank=3 min=1092 min_rank=5 imbalance_ratio=2.014160 max_load_reduction=24518 max_load_reduction_pct=0.748230 moved_to_echo=13322 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[801, 4052, 0, 2614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 3294, 1013, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2389, 0, 0, 0, 0, 0, 0, 0, 0, 3094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 14, 0, 0, 1, 0, 743, 512, 0, 0, 0, 0, 0, 0, 0, 0, 3816, 182, 0, 0, 21, 265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1487, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 227, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7467, 4356, 2389, 3094, 1272, 4284, 5582, 4324] avg=4096.0000 max=7467 max_rank=0 min=1272 min_rank=4 imbalance_ratio=1.822998 max_load_reduction=25301 max_load_reduction_pct=0.772125 moved_to_echo=15462 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[780, 0, 0, 4081, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 17, 0, 3205, 32, 0, 0, 0, 0, 0, 0, 0, 3378, 0, 835, 2, 0, 0, 576, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 4, 11, 0, 3991, 0, 0, 16, 0, 0, 0, 0, 0, 0, 3, 0, 4058, 7, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 33, 0, 0, 117, 3608, 0, 0, 0, 0, 0, 0, 0, 3526, 290, 34, 172, 0, 177, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4862, 3259, 4867, 3430, 4064, 4077, 3759, 4450] avg=4096.0000 max=4867 max_rank=2 min=3259 min_rank=1 imbalance_ratio=1.188232 max_load_reduction=27901 max_load_reduction_pct=0.851471 moved_to_echo=16350 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[763, 1830, 8, 328, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 47, 0, 0, 0, 0, 3134, 906, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 0, 0, 3733, 2137, 0, 0, 0, 0, 0, 0, 0, 0, 2621, 0, 0, 0, 10, 788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4083, 0, 0, 4024, 0, 629, 458, 0, 0, 0, 0, 0, 0, 0, 0, 2676, 163, 0, 0, 0, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 18, 146, 1, 0, 2775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 2, 1, 328, 201, 8, 193, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3000, 4089, 6059, 3419, 9197, 3320, 2943, 741] avg=4096.0000 max=9197 max_rank=4 min=741 min_rank=7 imbalance_ratio=2.245361 max_load_reduction=23571 max_load_reduction_pct=0.719330 moved_to_echo=16201 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[801, 27, 0, 636, 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3294, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3653, 2389, 0, 0, 0, 0, 0, 0, 0, 0, 2992, 125, 0, 4, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4086, 0, 0, 2, 5, 718, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 2, 2, 3413, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 101, 0, 0, 0, 0, 3834, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 251, 0, 0, 0, 0, 0, 220, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1759, 4308, 6042, 3151, 5324, 3602, 4027, 4555] avg=4096.0000 max=6042 max_rank=2 min=1759 min_rank=0 imbalance_ratio=1.475098 max_load_reduction=26726 max_load_reduction_pct=0.815613 moved_to_echo=17508 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[792, 80, 22, 401, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 1168, 0, 0, 0, 0, 3258, 8, 1, 0, 0, 0, 0, 0, 0, 224, 0, 0, 1, 0, 0, 3975, 19, 0, 0, 0, 0, 0, 0, 0, 0, 2906, 0, 0, 0, 72, 2736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 10, 0, 0, 4055, 1, 697, 4, 0, 0, 0, 0, 0, 0, 0, 0, 23, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 3874, 0, 0, 3763, 291, 0, 0, 0, 214, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1298, 4485, 4219, 5714, 4803, 26, 4081, 8142] avg=4096.0000 max=8142 max_rank=7 min=26 min_rank=5 imbalance_ratio=1.987793 max_load_reduction=24626 max_load_reduction_pct=0.751526 moved_to_echo=17052 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[795, 4096, 469, 2806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 3287, 650, 1, 0, 0, 0, 0, 0, 0, 7, 0, 0, 3, 0, 0, 0, 1532, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 10, 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2002, 0, 16, 330, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 108, 0, 920, 1, 4000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 5, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 1471, 0, 0, 3, 6, 0, 1348, 11, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8166, 3960, 1542, 644, 2349, 9122, 4139, 2846] avg=4096.0000 max=9122 max_rank=5 min=644 min_rank=3 imbalance_ratio=2.227051 max_load_reduction=23646 max_load_reduction_pct=0.721619 moved_to_echo=18456 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[741, 0, 307, 2951, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 531, 0, 0, 0, 0, 3044, 1003, 0, 0, 0, 0, 0, 0, 0, 24, 6, 0, 0, 0, 0, 0, 2367, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 4, 4, 0, 8, 4000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 305, 0, 0, 0, 0, 252, 507, 0, 0, 0, 0, 0, 0, 72, 1258, 1482, 181, 0, 0, 189, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 13, 4082, 0, 0, 0, 0, 0, 0, 0, 4043, 1, 0, 125, 0, 0, 0, 36, 77, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4004, 4580, 2397, 5066, 1064, 7245, 4130, 4282] avg=4096.0000 max=7245 max_rank=5 min=1064 min_rank=4 imbalance_ratio=1.768799 max_load_reduction=25523 max_load_reduction_pct=0.778900 moved_to_echo=16721 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[5504, 12288, 896, 13824, 384, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 1280, 17152, 1024, 128, 1024, 3712, 8832, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[384, 128, 384, 1792, 2048, 128, 128, 128, 22528, 5632, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 384, 128, 11392, 128, 384, 14080, 128, 3840, 2944, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 15744, 256, 256, 128, 1024, 15744, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 3712, 128, 896, 256, 0, 0, 14848, 13312, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[21120, 384, 384, 4096, 3456, 512, 1920, 256, 1280, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 1664, 384, 128, 256, 128, 256, 30464, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3264, 50, 0, 0, 0, 0, 2, 0, 4018, 0, 0, 3043, 0, 0, 161, 0, 1, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 776, 0, 4039, 47, 0, 12, 0, 0, 3849, 57, 0, 0, 0, 0, 2044, 3215, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 126, 3588, 2246, 648, 0, 0, 0, 1, 0, 4057, 0, 0, 50, 983, 0, 3199, 15, 0, 1831, 3994, 46, 0, 0, 0, 0, 964, 0, 0, 85, 0, 0, 0, 3732, 0, 0, 0, 0, 2578, 54, 0, 121, 0, 0, 0, 0, 428, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 18, 1, 0, 3994] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4084, 0, 1, 0, 0, 0, 833, 0, 4094, 0, 0, 1989, 4018, 0, 3180, 2, 0, 1, 3572, 2624, 0, 0, 0, 0, 25, 4, 0, 0, 0, 0, 0, 3894, 0, 0, 0, 0, 52, 0, 0, 2, 0, 0, 0, 0, 4068, 1, 0, 0, 1, 0, 0, 47, 0, 0, 0, 0, 11, 205, 0, 59] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4056, 3537, 805, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 138, 0, 0, 0, 613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3712, 4, 0, 3851, 0, 0, 0, 0, 2826, 217, 0, 0, 0, 0, 66, 4, 0, 1839, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 2884] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[2, 0, 0, 34, 3822, 4017, 0, 0, 0, 0, 0, 0, 3527, 0, 0, 0, 650, 0, 131, 0, 0, 2710, 4096, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 3974, 0, 12, 0, 0, 3383, 2708, 48, 0, 8, 0, 0, 82, 11, 1, 0, 0, 0, 0, 0, 705, 315, 0, 0, 2, 4, 2471, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4094, 15, 1, 0, 6, 6, 0, 3801, 0, 0, 0, 4071, 0, 22, 2, 0, 0, 3542, 1028, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 2861, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 887, 181, 0, 3049, 90, 0, 0, 463, 0, 0, 29, 242, 0, 0, 167, 0, 4, 5, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 1550, 4096, 191, 3900, 0, 0, 0, 3, 0, 4095, 0, 0, 2517, 3682, 0, 4065, 49, 0, 1, 4067, 1, 97, 0, 0, 0, 62, 0, 0, 0, 0, 17, 0, 15, 0, 0, 0, 0, 3785, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 1, 6, 0, 554] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4064, 477, 6, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 4096, 0, 15, 0, 0, 0, 3831, 0, 0, 0, 0, 0, 0, 0, 0, 0, 727, 1, 0, 4081, 0, 0, 0, 0, 4096, 3982, 0, 0, 0, 0, 0, 1, 3268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1164, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3076, 0, 0, 3043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 0, 1, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 388, 0, 0, 0, 0, 0, 0, 4083, 776, 0, 3569, 47, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3849, 57, 0, 0, 0, 0, 2044, 0, 0, 0, 0, 0, 0, 0, 0, 3215, 0, 0, 0, 0, 0, 0, 0, 942, 470, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 1712, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1214, 6121, 167, 4484, 8487, 5950, 4627, 1718] avg=4096.0000 max=8487 max_rank=4 min=167 min_rank=2 imbalance_ratio=2.072021 max_load_reduction=24281 max_load_reduction_pct=0.740997 moved_to_echo=20782 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 126, 1279, 2246, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3105, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 616, 0, 3199, 15, 0, 1831, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 964, 0, 0, 85, 3994, 427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2286, 54, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 428, 7, 0, 0, 0, 0, 0, 0, 952, 434, 120, 292, 0, 0, 0, 0, 0, 0, 0, 2, 18, 1, 0, 3994, 1882, 247, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4299, 3156, 5707, 5470, 3298, 2461, 2233, 6144] avg=4096.0000 max=6144 max_rank=7 min=2233 min_rank=6 imbalance_ratio=1.500000 max_load_reduction=26624 max_load_reduction_pct=0.812500 moved_to_echo=14136 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1445, 3537, 805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2560, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 613, 483, 0, 0, 0, 0, 0, 0, 3712, 4, 0, 3403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2506, 217, 0, 0, 0, 0, 66, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1839, 0, 0, 0, 0, 0, 0, 961, 448, 503, 320, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 2884, 2128, 1033, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5787, 3135, 2698, 1096, 7119, 2793, 4071, 6069] avg=4096.0000 max=7119 max_rank=4 min=1096 min_rank=3 imbalance_ratio=1.738037 max_load_reduction=25649 max_load_reduction_pct=0.782745 moved_to_echo=20052 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 34, 1362, 4017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 408, 0, 131, 0, 0, 2710, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 455, 0, 0, 0, 0, 0, 0, 0, 8, 0, 3512, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3000, 2708, 48, 0, 8, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1, 0, 0, 0, 0, 0, 705, 827, 462, 79, 383, 0, 0, 0, 0, 315, 0, 0, 2, 4, 2471, 0, 0, 2005, 163, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5415, 2700, 3296, 4551, 3532, 5846, 2468, 4960] avg=4096.0000 max=5846 max_rank=5 min=2468 min_rank=6 imbalance_ratio=1.427246 max_load_reduction=26922 max_load_reduction_pct=0.821594 moved_to_echo=16806 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 1456, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 833, 0, 3134, 0, 0, 1989, 0, 0, 0, 0, 0, 0, 0, 0, 2512, 0, 3180, 2, 0, 1, 0, 2624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 4, 0, 0, 3572, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 1, 0, 0, 1, 0, 0, 47, 960, 453, 493, 5, 0, 0, 0, 0, 0, 0, 0, 0, 11, 205, 0, 59, 2142, 1013, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1458, 5956, 8319, 4087, 3441, 49, 6028, 3430] avg=4096.0000 max=8319 max_rank=2 min=49 min_rank=5 imbalance_ratio=2.031006 max_load_reduction=24449 max_load_reduction_pct=0.746124 moved_to_echo=12948 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1460, 4094, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 6, 0, 2910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2545, 0, 22, 2, 0, 0, 0, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 3542, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3624, 0, 0, 0, 0, 887, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3049, 90, 0, 0, 463, 0, 0, 29, 891, 332, 500, 463, 0, 0, 0, 0, 242, 0, 0, 167, 0, 4, 5, 0, 2149, 1026, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5570, 2922, 3597, 4048, 2529, 4692, 5817, 3593] avg=4096.0000 max=5817 max_rank=6 min=2529 min_rank=4 imbalance_ratio=1.420166 max_load_reduction=26951 max_load_reduction_pct=0.822479 moved_to_echo=16631 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1448, 477, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2560, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3831, 484, 0, 0, 0, 0, 0, 0, 727, 1, 0, 3607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3632, 3982, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3268, 0, 0, 0, 0, 0, 0, 0, 958, 474, 503, 464, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 2132, 1033, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1931, 3128, 2575, 4315, 4335, 7615, 5667, 3202] avg=4096.0000 max=7615 max_rank=5 min=1931 min_rank=0 imbalance_ratio=1.859131 max_load_reduction=25153 max_load_reduction_pct=0.767609 moved_to_echo=20819 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1550, 1453, 191, 3900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3130, 0, 0, 2517, 0, 0, 0, 0, 0, 0, 0, 0, 2293, 0, 4065, 49, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 0, 62, 0, 0, 0, 4067, 491, 0, 0, 0, 0, 0, 0, 0, 17, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3354, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 965, 5, 456, 431, 0, 0, 0, 0, 6, 0, 0, 0, 1, 6, 0, 554, 2152, 933, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7094, 5650, 6409, 4717, 27, 3362, 1857, 3652] avg=4096.0000 max=7094 max_rank=0 min=27 min_rank=4 imbalance_ratio=1.731934 max_load_reduction=25674 max_load_reduction_pct=0.783508 moved_to_echo=8898 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 0, 1792, 11136, 14720, 5376, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[13568, 0, 11008, 128, 128, 4608, 0, 3840, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[14080, 2048, 0, 0, 512, 0, 0, 896, 7552, 3200, 2688, 2432, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[18560, 10880, 128, 256, 128, 896, 256, 2176, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[640, 0, 128, 256, 128, 2688, 128, 7552, 16384, 5504, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8576, 896, 0, 23424, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 896, 0, 24320, 0, 0, 7680, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 1152, 128, 0, 128, 27904, 3712, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[15, 0, 0, 0, 4005, 4056, 0, 19, 4091, 202, 0, 0, 53, 0, 2, 0, 188, 2551, 0, 0, 2, 31, 0, 0, 0, 0, 3042, 0, 0, 11, 0, 65, 3567, 1, 33, 0, 0, 0, 5, 0, 26, 0, 0, 250, 1033, 3864, 163, 9, 0, 584, 0, 37, 3925, 522, 351, 0, 4, 2, 0, 0, 0, 46, 13, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4096, 4095, 0, 0, 87, 4, 0, 0, 0, 4045, 0, 0, 4090, 1059, 0, 3, 0, 0, 0, 0, 0, 3317, 4084, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 841, 460, 469, 0, 0, 16, 0, 0, 172, 31, 1788, 0, 0, 0, 0, 0, 6, 7, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 4091, 2124, 0, 0, 657, 1192, 0, 0, 0, 0, 0, 0, 4094, 1026, 0, 352, 0, 0, 0, 0, 0, 0, 3946, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3419, 0, 3696, 0, 0, 4, 0, 0, 4089, 4019, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[6, 2951, 1094, 0, 4096, 3789, 0, 0, 45, 0, 0, 83, 0, 14, 0, 0, 0, 3644, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 464, 0, 0, 0, 188, 0, 0, 0, 0, 0, 0, 0, 4084, 90, 4023, 0, 0, 0, 0, 0, 6, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4067, 3511, 0, 113, 11, 6, 0, 203, 0, 245, 0, 14, 1094, 3918, 0, 2740, 0, 0, 0, 1, 0, 0, 3995, 0, 0, 0, 0, 0, 4087, 1, 9, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 1072, 50, 2, 0, 26, 136, 8, 3200, 0, 4054, 0, 0, 0, 0, 0, 12, 64, 0, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 4080, 4093, 0, 0, 23, 51, 0, 0, 0, 0, 0, 0, 277, 3777, 0, 21, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4089, 0, 22, 0, 0, 0, 2, 0, 4058, 0, 0, 0, 0, 3312, 0, 0, 0, 4, 0, 0, 3143, 1713, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[7, 0, 27, 0, 2877, 4091, 3, 0, 0, 439, 0, 0, 23, 19, 0, 0, 4095, 847, 0, 0, 0, 1, 397, 0, 0, 14, 3498, 0, 0, 0, 0, 2, 217, 321, 0, 632, 0, 106, 4096, 0, 0, 0, 0, 0, 0, 3952, 75, 0, 0, 0, 0, 0, 3989, 505, 0, 0, 0, 90, 0, 10, 0, 2361, 74, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4096, 2397, 0, 0, 4096, 0, 0, 4074, 0, 818, 0, 0, 263, 0, 1512, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 1498, 0, 0, 0, 0, 0, 2629, 0, 491, 0, 0, 0, 1784, 0, 73, 0, 0, 576, 0, 0, 2359, 0, 2003, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 49, 3511, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 11, 6, 0, 203, 0, 245, 0, 14, 1445, 128, 0, 0, 0, 0, 0, 0, 1094, 3170, 0, 2740, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 1, 9, 0, 121, 0, 0, 0, 394, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1072, 50, 2, 354, 193, 0, 0, 0, 0, 0, 0, 0, 26, 136, 8, 1575, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 64, 0, 8, 3890, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3673, 2052, 7005, 3802, 4792, 1671, 5799, 3974] avg=4096.0000 max=7005 max_rank=2 min=1671 min_rank=5 imbalance_ratio=1.710205 max_load_reduction=25763 max_load_reduction_pct=0.786224 moved_to_echo=16236 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[15, 0, 0, 0, 48, 4056, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 202, 0, 0, 53, 0, 2, 0, 1772, 127, 0, 0, 0, 0, 0, 0, 188, 2065, 0, 0, 2, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2895, 0, 0, 11, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 3567, 1, 33, 0, 0, 0, 5, 0, 256, 221, 0, 0, 0, 0, 0, 0, 26, 0, 0, 250, 1033, 3864, 163, 9, 230, 147, 0, 0, 0, 0, 0, 0, 0, 584, 0, 37, 1932, 522, 351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 0, 0, 46, 13, 0, 3830, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4138, 6247, 2286, 2971, 4083, 5722, 3426, 3895] avg=4096.0000 max=6247 max_rank=1 min=2286 min_rank=2 imbalance_ratio=1.525146 max_load_reduction=26521 max_load_reduction_pct=0.809357 moved_to_echo=17126 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 49, 2124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 657, 1192, 0, 0, 0, 0, 0, 0, 1847, 129, 0, 0, 0, 0, 0, 0, 4094, 831, 0, 352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3755, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 1, 0, 103, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3419, 0, 3696, 0, 92, 191, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 2012, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 3913, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2175, 3825, 5277, 3755, 371, 7398, 6035, 3932] avg=4096.0000 max=7398 max_rank=5 min=371 min_rank=4 imbalance_ratio=1.806152 max_load_reduction=25370 max_load_reduction_pct=0.774231 moved_to_echo=17736 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[6, 2951, 1094, 0, 50, 3789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 83, 0, 14, 0, 0, 1844, 129, 0, 0, 0, 0, 0, 0, 0, 2949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 366, 230, 0, 0, 0, 0, 0, 0, 464, 0, 0, 0, 188, 0, 0, 0, 329, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2010, 90, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 3917, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7890, 2115, 2949, 3896, 4693, 1179, 6123, 3923] avg=4096.0000 max=7890 max_rank=0 min=1179 min_rank=5 imbalance_ratio=1.926270 max_load_reduction=24878 max_load_reduction_pct=0.759216 moved_to_echo=15918 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 50, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 4, 0, 0, 0, 4045, 0, 0, 77, 129, 0, 0, 0, 0, 0, 0, 4090, 858, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3317, 3887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 106, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 841, 460, 469, 0, 95, 197, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 86, 31, 1788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 0, 0, 3917, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4146, 4342, 4951, 7204, 4212, 2062, 1921, 3930] avg=4096.0000 max=7204 max_rank=3 min=1921 min_rank=6 imbalance_ratio=1.758789 max_load_reduction=25564 max_load_reduction_pct=0.780151 moved_to_echo=12125 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 27, 0, 35, 4091, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 439, 0, 0, 23, 19, 0, 0, 1801, 91, 0, 0, 0, 0, 0, 0, 4095, 686, 0, 0, 0, 1, 397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 3329, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 217, 321, 0, 632, 0, 106, 4096, 0, 85, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3952, 75, 0, 76, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1963, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 10, 0, 2361, 74, 0, 2751, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4163, 2373, 5179, 3345, 5682, 4272, 2468, 5286] avg=4096.0000 max=5682 max_rank=4 min=2373 min_rank=1 imbalance_ratio=1.387207 max_load_reduction=27086 max_load_reduction_pct=0.826599 moved_to_echo=17708 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 50, 2397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4074, 0, 818, 0, 0, 1065, 129, 0, 0, 0, 0, 0, 0, 263, 0, 1512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1498, 0, 0, 0, 0, 0, 2629, 0, 0, 133, 0, 0, 0, 0, 0, 0, 491, 0, 0, 0, 1784, 0, 73, 0, 0, 198, 0, 0, 0, 0, 0, 0, 0, 576, 0, 0, 1161, 0, 2003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3917, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2448, 10182, 1775, 3900, 4260, 2546, 3740, 3917] avg=4096.0000 max=10182 max_rank=1 min=1775 min_rank=2 imbalance_ratio=2.485840 max_load_reduction=22586 max_load_reduction_pct=0.689270 moved_to_echo=14463 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 39, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 51, 0, 0, 0, 0, 0, 0, 1424, 134, 0, 0, 0, 0, 0, 0, 277, 3048, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 22, 0, 0, 0, 2, 0, 382, 180, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 0, 3312, 0, 0, 347, 201, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1539, 1713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 3907, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4135, 1632, 3346, 3895, 4675, 7918, 3256, 3911] avg=4096.0000 max=7918 max_rank=5 min=1632 min_rank=1 imbalance_ratio=1.933105 max_load_reduction=24850 max_load_reduction_pct=0.758362 moved_to_echo=19760 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[9088, 1920, 0, 4480, 128, 5248, 128, 128, 11392, 1024, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[5120, 0, 0, 256, 7296, 12672, 4608, 128, 1536, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 1280, 256, 128, 12288, 6912, 12288, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[21760, 384, 128, 640, 128, 128, 6784, 0, 1792, 1408, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[14208, 13696, 1536, 3200, 128, 128, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 3456, 29440, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 3072, 1152, 0, 384, 28160, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 128, 2560, 128, 128, 30080, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 3622, 0, 0, 0, 1, 0, 0, 452, 0, 0, 4074, 0, 0, 9, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 9, 0, 8, 0, 0, 0, 395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4057, 716, 0, 0, 0, 3734, 0, 0, 0, 3367, 0, 0, 4091, 0, 42, 0, 0, 0, 0, 1, 4089, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1, 3125, 0, 0, 0, 2620, 0, 0, 4, 0, 0, 2904, 373, 0, 0, 0, 0, 33, 4096, 0, 0, 0, 3421, 0, 0, 0, 4, 0, 2, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1583, 1346, 0, 0, 0, 143, 0, 0, 0, 3303, 0, 0, 4090, 0, 599, 0, 0, 0, 0, 0, 1044, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[55, 458, 0, 0, 0, 0, 0, 0, 3787, 0, 31, 3822, 0, 0, 6, 0, 0, 0, 3988, 0, 0, 0, 4080, 0, 0, 0, 0, 33, 1789, 0, 3078, 0, 2, 0, 0, 0, 28, 0, 0, 0, 137, 0, 4042, 0, 259, 9, 0, 0, 0, 0, 0, 28, 0, 131, 2326, 0, 916, 2, 35, 0, 0, 0, 3726, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 1326, 21, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 19, 0, 4096, 2, 0, 0, 4096, 0, 0, 0, 3, 0, 3600, 0, 3878, 0, 0, 0, 0, 0, 0, 238, 0, 0, 0, 83, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 2884, 122, 0, 4096, 0, 0, 0, 3, 211, 0, 0, 4061, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 4065, 0, 0, 0, 119, 0, 0, 0, 1, 0, 0, 0, 0, 3675, 0, 32, 0, 3968, 71, 1, 115, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 3, 0, 3, 0, 234, 0, 0, 4096, 17, 0, 63, 0, 452, 0, 0, 0, 2741, 0, 0, 3694, 0, 0, 0, 0, 1315, 128, 40, 3843, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[167, 3583, 0, 0, 0, 3979, 0, 0, 1, 0, 0, 491, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2955, 0, 0, 0, 17, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 3836, 3, 0, 0, 0, 212, 0, 0, 0, 3, 0, 0, 4092, 0, 163, 0, 0, 0, 960, 0, 4094, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 2246, 0, 0, 0, 234, 545, 0, 8, 74, 36, 0, 0, 23, 0, 12, 0, 0, 1406, 0, 0, 26, 3821, 0, 0, 0, 0, 0, 892, 0, 4096, 4, 0, 885, 0, 0, 0, 0, 264, 1081, 3, 150, 3141, 2366, 0, 0, 0, 1025, 0, 0, 0, 3544, 0, 3, 3635, 0, 0, 0, 0, 0, 0, 1, 3247, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 3, 0, 1, 3763, 0, 0, 4093, 0, 0, 0, 0, 3, 0, 4096, 0, 11, 0, 4016, 0, 0, 16, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3233, 0, 0, 1864, 0, 0, 0, 0, 823, 0, 0, 0, 2469, 323, 0, 3994, 0, 0, 0, 0, 359, 0, 0, 3670, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 4065, 0, 0, 497, 410, 0, 0, 0, 0, 0, 0, 0, 119, 0, 0, 0, 1, 0, 0, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 773, 0, 32, 0, 3968, 71, 0, 0, 0, 0, 0, 0, 0, 0, 1, 115, 0, 0, 0, 0, 4091, 0, 391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 234, 2902, 426, 0, 0, 0, 0, 0, 0, 0, 0, 3262, 17, 0, 63, 0, 452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2741, 0, 0, 1705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1315, 128, 40, 3843, 0, 33, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4973, 1186, 4844, 4598, 3568, 3794, 4446, 5359] avg=4096.0000 max=5359 max_rank=7 min=1186 min_rank=1 imbalance_ratio=1.308350 max_load_reduction=27409 max_load_reduction_pct=0.836456 moved_to_echo=17167 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3622, 0, 0, 0, 1, 0, 0, 550, 407, 0, 0, 0, 0, 0, 0, 452, 0, 0, 4074, 0, 0, 9, 0, 1181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 8, 0, 0, 0, 395, 0, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3234, 471, 0, 0, 0, 0, 0, 0, 0, 5, 3230, 716, 0, 0, 0, 3734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3367, 0, 0, 1889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 1, 4089, 0, 33, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4580, 5716, 862, 799, 3705, 7685, 5256, 4165] avg=4096.0000 max=7685 max_rank=5 min=799 min_rank=3 imbalance_ratio=1.876221 max_load_reduction=25083 max_load_reduction_pct=0.765472 moved_to_echo=20811 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[55, 458, 0, 0, 0, 0, 0, 0, 312, 405, 0, 0, 0, 0, 0, 0, 3787, 0, 31, 3822, 0, 0, 6, 0, 671, 0, 0, 0, 0, 0, 0, 0, 0, 0, 839, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 1789, 0, 3078, 0, 386, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 28, 0, 0, 0, 3149, 268, 0, 0, 0, 0, 0, 0, 137, 0, 3218, 0, 259, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 131, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 916, 2, 35, 0, 0, 0, 3726, 0, 33, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1230, 8317, 4919, 5286, 3447, 3623, 1234, 4712] avg=4096.0000 max=8317 max_rank=1 min=1230 min_rank=0 imbalance_ratio=2.030518 max_load_reduction=24451 max_load_reduction_pct=0.746185 moved_to_echo=13016 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 3125, 0, 0, 0, 2620, 0, 0, 550, 158, 0, 0, 0, 0, 0, 0, 4, 0, 0, 2904, 373, 0, 0, 0, 1180, 0, 0, 0, 0, 0, 0, 0, 0, 33, 862, 0, 0, 0, 3421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2, 0, 4074, 0, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3234, 471, 0, 0, 0, 0, 0, 0, 0, 3, 1261, 1346, 0, 0, 0, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3303, 0, 0, 1889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 599, 0, 0, 0, 0, 0, 1044, 0, 13, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6454, 4461, 4316, 4231, 3705, 2753, 5192, 1656] avg=4096.0000 max=6454 max_rank=0 min=1656 min_rank=7 imbalance_ratio=1.575684 max_load_reduction=26314 max_load_reduction_pct=0.803040 moved_to_echo=13306 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1326, 21, 0, 0, 0, 0, 0, 551, 402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 1182, 0, 0, 0, 0, 0, 0, 0, 19, 0, 862, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3600, 0, 3878, 0, 383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 238, 0, 0, 3234, 472, 0, 0, 0, 0, 0, 0, 0, 83, 3194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2884, 122, 0, 1891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 211, 0, 0, 4061, 0, 32, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2300, 1200, 4979, 7864, 3944, 3277, 4897, 4307] avg=4096.0000 max=7864 max_rank=3 min=1200 min_rank=1 imbalance_ratio=1.919922 max_load_reduction=24904 max_load_reduction_pct=0.760010 moved_to_echo=16425 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2246, 0, 0, 0, 234, 545, 0, 489, 315, 0, 0, 0, 0, 0, 0, 8, 74, 36, 0, 0, 23, 0, 12, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 296, 0, 0, 26, 3821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 892, 0, 4096, 4, 300, 0, 0, 0, 0, 0, 0, 0, 0, 885, 0, 0, 0, 0, 264, 1081, 1110, 419, 0, 0, 0, 0, 0, 0, 3, 150, 2501, 2366, 0, 0, 0, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3544, 0, 3, 1678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3247, 0, 25, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3829, 1202, 4143, 5292, 3759, 6045, 5225, 3273] avg=4096.0000 max=6045 max_rank=5 min=1202 min_rank=1 imbalance_ratio=1.475830 max_load_reduction=26723 max_load_reduction_pct=0.815521 moved_to_echo=18302 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 3, 0, 1, 537, 187, 0, 0, 0, 0, 0, 0, 3763, 0, 0, 4093, 0, 0, 0, 0, 1153, 0, 0, 0, 0, 0, 0, 0, 3, 0, 862, 0, 11, 0, 4016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 27, 0, 0, 0, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3233, 3234, 460, 0, 0, 0, 0, 0, 0, 0, 0, 1484, 0, 0, 0, 0, 823, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2469, 323, 0, 1844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 359, 0, 0, 3670, 0, 15, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[732, 9009, 4892, 221, 6927, 2307, 4636, 4044] avg=4096.0000 max=9009 max_rank=1 min=221 min_rank=3 imbalance_ratio=2.199463 max_load_reduction=23759 max_load_reduction_pct=0.725067 moved_to_echo=17914 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[167, 3583, 0, 0, 0, 3979, 0, 0, 553, 388, 0, 0, 0, 0, 0, 0, 1, 0, 0, 491, 0, 0, 0, 0, 1185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 858, 0, 0, 0, 2955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 4092, 0, 0, 0, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3238, 475, 0, 0, 0, 0, 0, 0, 0, 24, 3045, 3, 0, 0, 0, 212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 163, 0, 0, 0, 960, 0, 4094, 0, 35, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8670, 1677, 3813, 4477, 3713, 3284, 1882, 5252] avg=4096.0000 max=8670 max_rank=0 min=1677 min_rank=1 imbalance_ratio=2.116699 max_load_reduction=24098 max_load_reduction_pct=0.735413 moved_to_echo=14131 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8064, 256, 128, 15488, 384, 128, 128, 128, 8704, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[256, 14464, 128, 0, 0, 11008, 640, 128, 4096, 2688, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 0, 18432, 512, 256, 13952, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 256, 128, 128, 10496, 0, 19712, 128, 2560, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 896, 0, 128, 128, 256, 384, 4608, 23424, 3584, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 384, 21248, 4480, 384, 128, 0, 6400, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 6272, 128, 128, 128, 26368, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1792, 128, 128, 1920, 1152, 128, 27776, 0, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True + [2026-09-21 08:25:36] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 264.9 | throughput per GPU (TFLOP/s/GPU): 121.6 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 1.006536E+01 | loss scale: 1.0 | grad norm: 101.299 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1986, 859, 129, 124, 139, 563, 488, 191, 321, 703, 38, 1, 70, 514, 422, 1263, 175, 758, 1479, 1197, 130, 373, 767, 159, 1039, 192, 141, 687, 527, 826, 176, 389, 245, 486, 969, 160, 485, 649, 291, 471, 391, 833, 1018, 8, 1816, 893, 51, 428, 469, 717, 779, 120, 1570, 473, 313, 134, 762, 5, 514, 184, 33, 365, 258, 52] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[2450, 726, 170, 154, 32, 510, 366, 69, 79, 522, 68, 7, 106, 319, 161, 1265, 52, 447, 1348, 1318, 111, 644, 665, 179, 1005, 149, 43, 961, 578, 765, 210, 569, 174, 463, 639, 240, 522, 621, 358, 603, 449, 851, 628, 31, 1880, 1232, 37, 431, 866, 575, 622, 94, 1938, 427, 286, 233, 817, 12, 607, 40, 21, 266, 675, 82] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[2531, 746, 125, 149, 62, 481, 278, 45, 233, 567, 99, 2, 107, 344, 111, 1413, 78, 374, 1118, 1441, 44, 389, 650, 354, 933, 135, 106, 647, 702, 1217, 387, 410, 59, 523, 760, 105, 874, 900, 209, 824, 325, 1185, 795, 39, 1642, 528, 53, 474, 404, 902, 938, 81, 1571, 258, 323, 166, 679, 8, 670, 146, 29, 357, 531, 132] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[2777, 619, 106, 139, 138, 344, 574, 78, 132, 885, 31, 3, 86, 283, 157, 1112, 34, 539, 848, 1355, 68, 477, 517, 381, 1100, 161, 28, 1068, 469, 716, 185, 341, 195, 388, 697, 306, 537, 1049, 264, 760, 349, 824, 708, 25, 2014, 757, 89, 396, 413, 542, 839, 23, 1820, 551, 347, 193, 1221, 14, 501, 60, 10, 489, 602, 34] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1980, 761, 55, 209, 121, 475, 626, 39, 258, 479, 328, 1, 39, 247, 109, 1345, 94, 339, 1243, 1382, 145, 895, 737, 286, 1152, 90, 73, 923, 521, 557, 69, 278, 171, 673, 464, 146, 626, 1115, 216, 482, 515, 783, 578, 22, 1885, 1051, 49, 710, 515, 557, 1007, 157, 1687, 402, 316, 352, 687, 11, 946, 230, 27, 242, 236, 54] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1999, 627, 345, 250, 178, 451, 772, 129, 105, 563, 200, 2, 40, 275, 223, 1562, 308, 332, 626, 1676, 98, 273, 681, 194, 748, 112, 231, 1383, 411, 708, 102, 450, 394, 316, 378, 95, 856, 570, 191, 316, 1055, 955, 831, 25, 1441, 604, 90, 410, 574, 927, 1114, 236, 1754, 386, 260, 272, 827, 4, 702, 279, 18, 354, 405, 75] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[2179, 812, 109, 110, 249, 762, 389, 126, 343, 791, 46, 1, 79, 295, 157, 682, 77, 644, 931, 1413, 175, 684, 626, 423, 815, 142, 44, 1098, 547, 589, 147, 255, 292, 417, 476, 108, 422, 596, 224, 570, 283, 968, 889, 8, 1866, 870, 107, 701, 571, 526, 1072, 118, 1938, 566, 225, 439, 913, 14, 712, 192, 14, 293, 479, 159] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[2202, 799, 70, 280, 75, 560, 531, 22, 305, 958, 164, 0, 34, 356, 146, 1096, 39, 151, 1075, 1573, 95, 717, 981, 287, 798, 124, 183, 767, 678, 875, 208, 466, 177, 714, 555, 245, 528, 764, 243, 619, 480, 691, 626, 3, 1642, 1065, 14, 491, 643, 598, 851, 136, 1503, 407, 432, 393, 1069, 15, 558, 185, 14, 154, 253, 85] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2019, 746, 125, 149, 62, 481, 278, 45, 0, 0, 0, 0, 0, 0, 0, 0, 233, 567, 99, 2, 107, 344, 111, 1413, 621, 557, 45, 0, 0, 0, 0, 0, 78, 374, 1118, 820, 44, 389, 650, 354, 0, 0, 0, 0, 0, 0, 0, 0, 933, 135, 106, 647, 702, 1217, 387, 410, 50, 0, 0, 0, 0, 0, 0, 0, 59, 523, 760, 105, 874, 900, 209, 824, 417, 0, 0, 0, 0, 0, 0, 0, 325, 1185, 795, 39, 471, 528, 53, 474, 0, 0, 0, 0, 0, 0, 0, 0, 404, 902, 938, 81, 756, 258, 323, 166, 0, 0, 0, 0, 0, 0, 0, 0, 679, 8, 670, 146, 29, 357, 531, 132, 1171, 258, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3905, 4099, 3827, 4587, 4671, 3870, 3828, 3981] avg=4096.0000 max=4671 max_rank=4 min=3827 min_rank=2 imbalance_ratio=1.140381 max_load_reduction=28097 max_load_reduction_pct=0.857452 moved_to_echo=16350 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1955, 726, 170, 154, 32, 510, 366, 69, 0, 0, 0, 0, 0, 0, 0, 0, 79, 522, 68, 7, 106, 319, 161, 1265, 568, 687, 44, 0, 0, 0, 0, 0, 52, 447, 1348, 750, 111, 644, 665, 179, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 149, 43, 961, 578, 765, 210, 569, 48, 0, 0, 0, 0, 0, 0, 0, 174, 463, 639, 240, 522, 621, 358, 603, 403, 0, 0, 0, 0, 0, 0, 0, 449, 851, 628, 31, 539, 1232, 37, 431, 0, 0, 0, 0, 0, 0, 0, 0, 866, 575, 622, 94, 933, 427, 286, 233, 0, 0, 0, 0, 0, 0, 0, 0, 817, 12, 607, 40, 21, 266, 675, 82, 1341, 318, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3982, 3826, 4196, 4328, 4023, 4198, 4036, 4179] avg=4096.0000 max=4328 max_rank=3 min=3826 min_rank=1 imbalance_ratio=1.056641 max_load_reduction=28440 max_load_reduction_pct=0.867920 moved_to_echo=16436 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1585, 859, 129, 124, 139, 563, 488, 191, 0, 0, 0, 0, 0, 0, 0, 0, 321, 703, 38, 1, 70, 514, 422, 1263, 516, 556, 35, 0, 0, 0, 0, 0, 175, 758, 1479, 681, 130, 373, 767, 159, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 192, 141, 687, 527, 826, 176, 389, 39, 0, 0, 0, 0, 0, 0, 0, 245, 486, 969, 160, 485, 649, 291, 471, 327, 0, 0, 0, 0, 0, 0, 0, 391, 833, 1018, 8, 521, 893, 51, 428, 0, 0, 0, 0, 0, 0, 0, 0, 469, 717, 779, 120, 756, 473, 313, 134, 0, 0, 0, 0, 0, 0, 0, 0, 762, 5, 514, 184, 33, 365, 258, 52, 1295, 258, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4439, 4522, 4016, 4083, 4143, 3761, 3726] avg=4096.0000 max=4522 max_rank=2 min=3726 min_rank=7 imbalance_ratio=1.104004 max_load_reduction=28246 max_load_reduction_pct=0.862000 moved_to_echo=15713 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1595, 627, 345, 250, 178, 451, 772, 129, 0, 0, 0, 0, 0, 0, 0, 0, 105, 563, 200, 2, 40, 275, 223, 1562, 723, 622, 36, 0, 0, 0, 0, 0, 308, 332, 626, 953, 98, 273, 681, 194, 0, 0, 0, 0, 0, 0, 0, 0, 748, 112, 231, 1383, 411, 708, 102, 450, 39, 0, 0, 0, 0, 0, 0, 0, 394, 316, 378, 95, 856, 570, 191, 316, 329, 0, 0, 0, 0, 0, 0, 0, 1055, 955, 831, 25, 414, 604, 90, 410, 0, 0, 0, 0, 0, 0, 0, 0, 574, 927, 1114, 236, 844, 386, 260, 272, 0, 0, 0, 0, 0, 0, 0, 0, 827, 4, 702, 279, 18, 354, 405, 75, 1027, 288, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4347, 4351, 3465, 4184, 3445, 4384, 4613, 3979] avg=4096.0000 max=4613 max_rank=6 min=3445 min_rank=4 imbalance_ratio=1.126221 max_load_reduction=28155 max_load_reduction_pct=0.859222 moved_to_echo=16421 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2215, 619, 106, 139, 138, 344, 574, 78, 0, 0, 0, 0, 0, 0, 0, 0, 132, 885, 31, 3, 86, 283, 157, 1112, 584, 645, 50, 0, 0, 0, 0, 0, 34, 539, 848, 771, 68, 477, 517, 381, 0, 0, 0, 0, 0, 0, 0, 0, 1100, 161, 28, 1068, 469, 716, 185, 341, 55, 0, 0, 0, 0, 0, 0, 0, 195, 388, 697, 306, 537, 1049, 264, 760, 457, 0, 0, 0, 0, 0, 0, 0, 349, 824, 708, 25, 578, 757, 89, 396, 0, 0, 0, 0, 0, 0, 0, 0, 413, 542, 839, 23, 876, 551, 347, 193, 0, 0, 0, 0, 0, 0, 0, 0, 1221, 14, 501, 60, 10, 489, 602, 34, 1436, 299, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4213, 3968, 3635, 4123, 4653, 3726, 3784, 4666] avg=4096.0000 max=4666 max_rank=7 min=3635 min_rank=2 imbalance_ratio=1.139160 max_load_reduction=28102 max_load_reduction_pct=0.857605 moved_to_echo=16829 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1580, 761, 55, 209, 121, 475, 626, 39, 0, 0, 0, 0, 0, 0, 0, 0, 258, 479, 328, 1, 39, 247, 109, 1345, 596, 598, 35, 0, 0, 0, 0, 0, 94, 339, 1243, 786, 145, 895, 737, 286, 0, 0, 0, 0, 0, 0, 0, 0, 1152, 90, 73, 923, 521, 557, 69, 278, 39, 0, 0, 0, 0, 0, 0, 0, 171, 673, 464, 146, 626, 1115, 216, 482, 326, 0, 0, 0, 0, 0, 0, 0, 515, 783, 578, 22, 541, 1051, 49, 710, 0, 0, 0, 0, 0, 0, 0, 0, 515, 557, 1007, 157, 812, 402, 316, 352, 0, 0, 0, 0, 0, 0, 0, 0, 687, 11, 946, 230, 27, 242, 236, 54, 1344, 277, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3866, 4035, 4525, 3702, 4219, 4249, 4118, 4054] avg=4096.0000 max=4525 max_rank=2 min=3702 min_rank=3 imbalance_ratio=1.104736 max_load_reduction=28243 max_load_reduction_pct=0.861908 moved_to_echo=16640 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1757, 799, 70, 280, 75, 560, 531, 22, 0, 0, 0, 0, 0, 0, 0, 0, 305, 958, 164, 0, 34, 356, 146, 1096, 678, 532, 39, 0, 0, 0, 0, 0, 39, 151, 1075, 895, 95, 717, 981, 287, 0, 0, 0, 0, 0, 0, 0, 0, 798, 124, 183, 767, 678, 875, 208, 466, 44, 0, 0, 0, 0, 0, 0, 0, 177, 714, 555, 245, 528, 764, 243, 619, 362, 0, 0, 0, 0, 0, 0, 0, 480, 691, 626, 3, 471, 1065, 14, 491, 0, 0, 0, 0, 0, 0, 0, 0, 643, 598, 851, 136, 724, 407, 432, 393, 0, 0, 0, 0, 0, 0, 0, 0, 1069, 15, 558, 185, 14, 154, 253, 85, 1171, 247, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4308, 4240, 4143, 4207, 3841, 4184, 3751] avg=4096.0000 max=4308 max_rank=1 min=3751 min_rank=7 imbalance_ratio=1.051758 max_load_reduction=28460 max_load_reduction_pct=0.868530 moved_to_echo=15983 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1726, 812, 109, 110, 249, 762, 389, 126, 0, 0, 0, 0, 0, 0, 0, 0, 343, 791, 46, 1, 79, 295, 157, 682, 615, 690, 43, 0, 0, 0, 0, 0, 77, 644, 931, 798, 175, 684, 626, 423, 0, 0, 0, 0, 0, 0, 0, 0, 815, 142, 44, 1098, 547, 589, 147, 255, 48, 0, 0, 0, 0, 0, 0, 0, 292, 417, 476, 108, 422, 596, 224, 570, 362, 0, 0, 0, 0, 0, 0, 0, 283, 968, 889, 8, 531, 870, 107, 701, 0, 0, 0, 0, 0, 0, 0, 0, 571, 526, 1072, 118, 927, 566, 225, 439, 0, 0, 0, 0, 0, 0, 0, 0, 913, 14, 712, 192, 14, 293, 479, 159, 1335, 321, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4283, 3742, 4358, 3685, 3467, 4357, 4444, 4432] avg=4096.0000 max=4444 max_rank=6 min=3467 min_rank=4 imbalance_ratio=1.084961 max_load_reduction=28324 max_load_reduction_pct=0.864380 moved_to_echo=16700 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1792, 4096, 4992, 1408, 4864, 6272, 2048, 4736, 3072, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[7680, 1152, 896, 7552, 4480, 6272, 1536, 3200, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3968, 7168, 6144, 256, 4096, 7040, 512, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[14464, 6016, 1152, 1536, 1024, 4224, 4096, 768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[896, 3584, 8704, 6528, 896, 4480, 5632, 2304, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[1792, 5504, 1024, 128, 640, 2688, 1536, 9856, 4992, 4992, 384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[7040, 128, 5248, 1408, 256, 2560, 3456, 768, 10240, 2304, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4480, 5376, 7296, 1024, 6656, 3584, 2560, 2304, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 2686, 357, 4, 9, 0, 0, 0, 335, 3, 34, 841, 47, 0, 0, 2, 68, 0, 175, 0, 0, 1576, 0, 4056, 0, 0, 2, 10, 766, 22, 260, 710, 2333, 0, 74, 0, 51, 0, 90, 2046, 4, 800, 1320, 2, 3694, 13, 957, 0, 3226, 834, 2, 786, 0, 0, 0, 2871, 219, 27, 3, 0, 1427, 0, 26, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[4, 1099, 2289, 0, 0, 0, 213, 19, 201, 6, 1745, 536, 213, 0, 19, 58, 249, 0, 0, 0, 5, 2209, 0, 4088, 1, 1, 1, 152, 705, 4, 47, 30, 3755, 0, 119, 138, 8, 0, 16, 179, 109, 568, 360, 0, 1272, 0, 1202, 39, 2648, 2527, 0, 1558, 13, 1, 0, 3909, 10, 7, 25, 0, 411, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 2004, 2731, 29, 0, 1, 141, 0, 30, 0, 118, 32, 126, 16, 2, 13, 2, 24, 13, 0, 1, 3930, 1, 3356, 0, 139, 1, 0, 1082, 0, 4, 149, 135, 0, 28, 0, 8, 0, 42, 155, 176, 3831, 2577, 0, 983, 2, 7, 5, 1754, 3804, 0, 2486, 0, 0, 0, 2501, 46, 191, 3, 9, 3, 0, 77, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[2, 475, 3620, 0, 0, 0, 197, 0, 9, 0, 52, 993, 1620, 7, 36, 0, 1210, 0, 0, 2, 20, 3741, 0, 651, 0, 56, 0, 56, 1051, 2, 4, 9, 425, 0, 178, 0, 0, 3, 0, 384, 52, 3996, 2356, 0, 3262, 0, 50, 59, 3146, 1736, 0, 802, 5, 0, 0, 1073, 435, 701, 16, 0, 271, 0, 5, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[129, 1023, 208, 1, 0, 1, 516, 3, 724, 0, 0, 977, 517, 0, 62, 35, 206, 20, 0, 24, 70, 1829, 1, 3715, 44, 34, 0, 1579, 35, 4, 43, 128, 2125, 1, 4, 1, 0, 1, 13, 1611, 32, 4012, 2272, 0, 3623, 0, 13, 326, 2166, 1424, 0, 196, 1, 0, 24, 2370, 131, 434, 10, 1, 8, 0, 41, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[49, 2833, 2881, 0, 0, 0, 763, 0, 47, 0, 27, 1532, 56, 0, 1, 1, 44, 2, 1, 0, 0, 2573, 39, 3650, 5, 201, 0, 4, 416, 21, 14, 68, 1610, 12, 67, 9, 0, 1, 50, 566, 2, 2457, 1080, 0, 83, 1, 2298, 463, 2314, 826, 0, 3347, 0, 1, 0, 1984, 35, 266, 3, 34, 24, 0, 7, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 2615, 3973, 0, 2, 40, 1356, 0, 69, 0, 248, 590, 72, 140, 0, 0, 132, 0, 139, 0, 64, 3405, 2, 2994, 0, 8, 2, 11, 1297, 1, 78, 50, 1972, 0, 668, 0, 33, 0, 0, 14, 129, 1769, 2957, 0, 47, 15, 87, 24, 1914, 2018, 21, 452, 0, 9, 24, 3161, 25, 2, 2, 2, 0, 0, 135, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 3883, 3796, 0, 14, 0, 2422, 0, 108, 19, 219, 736, 12, 0, 0, 0, 16, 0, 13, 1, 0, 2200, 0, 4048, 0, 43, 0, 120, 327, 34, 100, 6, 531, 15, 407, 0, 0, 0, 77, 477, 0, 1547, 2422, 8, 95, 0, 354, 92, 1480, 1572, 0, 11, 1, 0, 0, 3659, 163, 7, 18, 0, 1164, 0, 551, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 3883, 1952, 0, 14, 0, 2422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 19, 219, 736, 12, 0, 0, 0, 782, 1478, 0, 0, 0, 0, 0, 0, 16, 0, 13, 1, 0, 2200, 0, 1329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 120, 327, 34, 100, 6, 1200, 1241, 0, 0, 0, 0, 0, 0, 531, 15, 407, 0, 0, 0, 77, 477, 1844, 343, 48, 0, 0, 0, 0, 0, 0, 0, 2079, 8, 95, 0, 354, 92, 0, 0, 0, 0, 0, 0, 0, 0, 650, 1572, 0, 11, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 163, 7, 18, 0, 1164, 0, 551, 0, 3659, 347, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8271, 3354, 3559, 3071, 3742, 2628, 2234, 5909] avg=4096.0000 max=8271 max_rank=0 min=2234 min_rank=6 imbalance_ratio=2.019287 max_load_reduction=24497 max_load_reduction_pct=0.747589 moved_to_echo=14513 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4, 1099, 1180, 0, 0, 0, 213, 19, 0, 0, 0, 0, 0, 0, 0, 0, 201, 6, 1745, 536, 213, 0, 19, 58, 1391, 1490, 0, 0, 0, 0, 0, 0, 249, 0, 0, 0, 5, 2209, 0, 1349, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 152, 705, 4, 47, 30, 440, 1249, 0, 0, 0, 0, 0, 0, 3755, 0, 119, 138, 8, 0, 16, 179, 1109, 50, 79, 0, 0, 0, 0, 0, 109, 0, 310, 0, 1272, 0, 1202, 39, 0, 0, 0, 0, 0, 0, 0, 0, 1178, 2527, 0, 1558, 13, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 7, 25, 0, 411, 0, 0, 0, 3909, 128, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2515, 5659, 3812, 2630, 5453, 2932, 5277, 4490] avg=4096.0000 max=5659 max_rank=1 min=2515 min_rank=0 imbalance_ratio=1.381592 max_load_reduction=27109 max_load_reduction_pct=0.827301 moved_to_echo=18152 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[49, 2833, 1486, 0, 0, 0, 763, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 27, 1532, 56, 0, 1, 1, 1216, 1330, 0, 0, 0, 0, 0, 0, 44, 2, 1, 0, 0, 2573, 39, 1205, 0, 0, 0, 0, 0, 0, 0, 0, 5, 201, 0, 4, 416, 21, 14, 68, 1904, 1115, 0, 0, 0, 0, 0, 0, 1610, 12, 67, 9, 0, 1, 50, 566, 1395, 151, 69, 0, 0, 0, 0, 0, 2, 0, 929, 0, 83, 1, 2298, 463, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 826, 0, 3347, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 266, 3, 34, 24, 0, 7, 0, 1984, 553, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5131, 4210, 3864, 3748, 3930, 3776, 5203, 2906] avg=4096.0000 max=5203 max_rank=6 min=2906 min_rank=7 imbalance_ratio=1.270264 max_load_reduction=27565 max_load_reduction_pct=0.841217 moved_to_echo=15815 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[129, 1023, 108, 1, 0, 1, 516, 3, 0, 0, 0, 0, 0, 0, 0, 0, 724, 0, 0, 977, 517, 0, 62, 35, 1138, 1354, 0, 0, 0, 0, 0, 0, 206, 20, 0, 24, 70, 1829, 1, 1226, 0, 0, 0, 0, 0, 0, 0, 0, 44, 34, 0, 1579, 35, 4, 43, 128, 3110, 1135, 0, 0, 0, 0, 0, 0, 2125, 1, 4, 1, 0, 1, 13, 1611, 100, 319, 65, 0, 0, 0, 0, 0, 32, 0, 1953, 0, 3623, 0, 13, 326, 0, 0, 0, 0, 0, 0, 0, 0, 963, 1424, 0, 196, 1, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 434, 10, 1, 8, 0, 41, 0, 2370, 902, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1781, 4807, 3376, 6112, 4240, 5947, 2608, 3897] avg=4096.0000 max=6112 max_rank=3 min=1781 min_rank=0 imbalance_ratio=1.492188 max_load_reduction=26656 max_load_reduction_pct=0.813477 moved_to_echo=16692 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 2004, 1408, 29, 0, 1, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 118, 32, 126, 16, 2, 13, 921, 1223, 0, 0, 0, 0, 0, 0, 2, 24, 13, 0, 1, 3930, 1, 1108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 1, 0, 1082, 0, 4, 149, 2970, 1025, 0, 0, 0, 0, 0, 0, 135, 0, 28, 0, 8, 0, 42, 155, 1323, 362, 52, 0, 0, 0, 0, 0, 176, 0, 2215, 0, 983, 2, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 781, 3804, 0, 2486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 191, 3, 9, 3, 0, 77, 0, 2501, 861, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3583, 2481, 5079, 5370, 2105, 3388, 7071, 3691] avg=4096.0000 max=7071 max_rank=6 min=2105 min_rank=4 imbalance_ratio=1.726318 max_load_reduction=25697 max_load_reduction_pct=0.784210 moved_to_echo=16255 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 475, 1867, 0, 0, 0, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 52, 993, 1620, 7, 36, 0, 1653, 237, 0, 0, 0, 0, 0, 0, 1210, 0, 0, 2, 20, 3741, 0, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 56, 1051, 2, 4, 9, 3098, 198, 0, 0, 0, 0, 0, 0, 425, 0, 178, 0, 0, 3, 0, 384, 1753, 331, 94, 0, 0, 0, 0, 0, 52, 0, 2025, 0, 3262, 0, 50, 59, 0, 0, 0, 0, 0, 0, 0, 0, 1399, 1736, 0, 802, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 435, 701, 16, 0, 271, 0, 5, 0, 1073, 898, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2541, 4607, 5189, 4474, 3168, 5448, 3942, 3399] avg=4096.0000 max=5448 max_rank=5 min=2541 min_rank=0 imbalance_ratio=1.330078 max_load_reduction=27320 max_load_reduction_pct=0.833740 moved_to_echo=15957 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 2615, 2049, 0, 2, 40, 1356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 248, 590, 72, 140, 0, 0, 1005, 1091, 0, 0, 0, 0, 0, 0, 132, 0, 139, 0, 64, 3405, 2, 989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 2, 11, 1297, 1, 78, 50, 1372, 914, 0, 0, 0, 0, 0, 0, 1972, 0, 668, 0, 33, 0, 0, 14, 1924, 415, 57, 0, 0, 0, 0, 0, 129, 0, 2542, 0, 47, 15, 87, 24, 0, 0, 0, 0, 0, 0, 0, 0, 852, 2018, 21, 452, 0, 9, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 2, 2, 2, 0, 0, 135, 0, 3161, 397, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6062, 3215, 4731, 3733, 5083, 2844, 3376, 3724] avg=4096.0000 max=6062 max_rank=0 min=2844 min_rank=5 imbalance_ratio=1.479980 max_load_reduction=26706 max_load_reduction_pct=0.815002 moved_to_echo=15027 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 2686, 185, 4, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 335, 3, 34, 841, 47, 0, 0, 2, 1695, 1478, 0, 0, 0, 0, 0, 0, 68, 0, 175, 0, 0, 1576, 0, 1339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 10, 766, 22, 260, 710, 621, 1239, 0, 0, 0, 0, 0, 0, 2333, 0, 74, 0, 51, 0, 90, 2046, 172, 185, 96, 0, 0, 0, 0, 0, 4, 0, 1135, 2, 3694, 13, 957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1435, 834, 2, 786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 219, 27, 3, 0, 1427, 0, 26, 0, 2871, 179, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2884, 4435, 3158, 3630, 5047, 5805, 3057, 4752] avg=4096.0000 max=5805 max_rank=5 min=2884 min_rank=0 imbalance_ratio=1.417236 max_load_reduction=26963 max_load_reduction_pct=0.822845 moved_to_echo=18661 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1536, 128, 2560, 6272, 2688, 256, 128, 128, 9856, 9728, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 16640, 10240, 128, 128, 128, 5632, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8320, 14848, 128, 9728, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[12928, 128, 1664, 256, 128, 128, 384, 5504, 9728, 2176, 640, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[2048, 128, 384, 128, 256, 21504, 128, 8832, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 512, 128, 2048, 5760, 128, 640, 1152, 14720, 8192, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[512, 0, 13312, 128, 13184, 128, 4992, 1024, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1152, 1664, 128, 128, 3328, 0, 896, 0, 21632, 4352, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1637, 0, 2, 215, 3, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 4096, 1, 417, 0, 0, 0, 0, 0, 9, 0, 4, 3978, 22, 0, 85, 2703, 71, 4020, 0, 2234, 1, 492, 1, 0, 0, 0, 4046, 74, 448, 204, 0, 0, 300, 6, 3, 45, 0, 0, 4011, 2082, 1, 0, 0, 0, 0, 1539, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[2283, 0, 79, 0, 332, 0, 0, 0, 0, 145, 0, 1, 3, 0, 0, 0, 0, 4079, 0, 62, 1, 1, 0, 0, 0, 0, 0, 0, 3822, 0, 0, 0, 1, 0, 2171, 0, 510, 1, 39, 0, 0, 5, 0, 3628, 3611, 982, 0, 0, 1, 3231, 4, 6, 131, 1, 0, 4066, 3528, 22, 0, 0, 0, 0, 22, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[3, 51, 0, 21, 151, 0, 1, 1, 7, 0, 0, 3349, 0, 0, 33, 0, 0, 4094, 0, 528, 55, 0, 0, 0, 0, 1, 0, 0, 471, 2359, 0, 0, 0, 1, 4054, 0, 2538, 0, 1, 0, 17, 0, 0, 3110, 2599, 5, 0, 1, 0, 3778, 0, 0, 1, 133, 0, 3998, 20, 0, 0, 0, 1, 0, 1319, 67] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[3236, 1, 0, 0, 1, 0, 1, 11, 0, 0, 0, 4090, 0, 0, 0, 1, 0, 4087, 12, 2281, 0, 0, 0, 2, 0, 0, 0, 0, 4090, 0, 0, 0, 18, 1, 697, 5, 478, 1, 104, 0, 0, 0, 0, 3989, 1, 2, 0, 0, 1013, 3515, 4, 4, 3, 0, 0, 4061, 340, 0, 677, 0, 0, 0, 42, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[3805, 1, 4, 0, 19, 0, 1, 2, 4, 239, 0, 87, 11, 1, 0, 0, 11, 3895, 282, 0, 0, 21, 3, 0, 0, 0, 1, 51, 2911, 2876, 0, 1, 102, 0, 1384, 0, 2606, 0, 1419, 0, 0, 0, 4, 4068, 1813, 1651, 77, 0, 6, 304, 2, 0, 2, 0, 314, 4016, 84, 0, 0, 2, 0, 0, 688, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1170, 0, 54, 0, 65, 2, 0, 0, 185, 613, 1397, 102, 0, 0, 0, 39, 2, 4050, 3, 4043, 0, 0, 0, 1, 0, 0, 0, 0, 892, 485, 0, 0, 49, 3, 3976, 0, 2117, 0, 2283, 0, 8, 0, 0, 3779, 10, 2675, 0, 0, 1, 317, 250, 0, 0, 0, 79, 1675, 19, 5, 0, 1, 0, 4, 2304, 110] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[3119, 0, 285, 0, 7, 0, 7, 0, 2, 11, 0, 3832, 27, 0, 9, 0, 0, 4086, 0, 2493, 0, 0, 1, 0, 2, 0, 0, 0, 4075, 79, 0, 0, 1, 0, 183, 0, 2116, 0, 23, 0, 2, 0, 0, 1473, 902, 109, 15, 0, 0, 3014, 0, 156, 1, 1, 0, 3216, 464, 0, 0, 0, 0, 0, 3057, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[471, 0, 0, 0, 11, 1, 0, 0, 0, 465, 5, 493, 0, 0, 0, 0, 1, 3745, 10, 3891, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 42, 0, 1, 53, 16, 3987, 0, 1, 1, 2, 0, 0, 0, 0, 3872, 464, 4019, 691, 3, 1, 69, 0, 0, 2, 48, 0, 3487, 257, 0, 0, 0, 0, 0, 2564, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1637, 0, 2, 215, 3, 0, 0, 0, 1462, 596, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 2255, 256, 0, 0, 0, 0, 0, 0, 0, 2378, 1, 417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4, 3978, 22, 0, 85, 474, 0, 0, 0, 0, 0, 0, 0, 2703, 71, 2510, 0, 2234, 1, 492, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1791, 74, 448, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 6, 3, 45, 0, 0, 2254, 0, 0, 0, 0, 0, 0, 0, 0, 2082, 1, 0, 0, 0, 0, 1539, 5, 1161, 1036, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3915, 2524, 2796, 4572, 8012, 2517, 2608, 5824] avg=4096.0000 max=8012 max_rank=4 min=2517 min_rank=5 imbalance_ratio=1.956055 max_load_reduction=24756 max_load_reduction_pct=0.755493 moved_to_echo=18961 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3, 51, 0, 21, 151, 0, 1, 1, 1461, 594, 0, 0, 0, 0, 0, 0, 7, 0, 0, 3349, 0, 0, 33, 0, 1733, 256, 0, 0, 0, 0, 0, 0, 0, 2377, 0, 528, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 471, 2359, 0, 0, 478, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2531, 0, 2538, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 1377, 2599, 5, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3778, 0, 0, 1, 133, 0, 2246, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 1, 0, 1319, 67, 1158, 1045, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2283, 5378, 2960, 3309, 5071, 3999, 6158, 3610] avg=4096.0000 max=6158 max_rank=6 min=2283 min_rank=0 imbalance_ratio=1.503418 max_load_reduction=26610 max_load_reduction_pct=0.812073 moved_to_echo=18838 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1170, 0, 54, 0, 65, 2, 0, 0, 1445, 249, 0, 0, 0, 0, 0, 0, 185, 613, 1397, 102, 0, 0, 0, 39, 2106, 253, 0, 0, 0, 0, 0, 0, 2, 2352, 3, 4043, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 892, 485, 0, 0, 469, 0, 0, 0, 0, 0, 0, 0, 49, 3, 2482, 0, 2117, 0, 2283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 1673, 10, 2675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 317, 250, 0, 0, 0, 79, 941, 0, 0, 0, 0, 0, 0, 0, 0, 19, 5, 0, 1, 0, 4, 2304, 110, 485, 1025, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2985, 4695, 6401, 1846, 6934, 4366, 1588, 3953] avg=4096.0000 max=6934 max_rank=4 min=1588 min_rank=6 imbalance_ratio=1.692871 max_load_reduction=25834 max_load_reduction_pct=0.788391 moved_to_echo=16841 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2283, 0, 79, 0, 332, 0, 0, 0, 1456, 604, 0, 0, 0, 0, 0, 0, 0, 145, 0, 1, 3, 0, 0, 0, 2022, 255, 0, 0, 0, 0, 0, 0, 0, 2368, 0, 62, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3822, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1356, 0, 510, 1, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 1606, 3611, 982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3231, 4, 6, 131, 1, 0, 2285, 0, 0, 0, 0, 0, 0, 0, 0, 3528, 22, 0, 0, 0, 0, 22, 0, 1177, 559, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4754, 2426, 2432, 4078, 1907, 6204, 5659, 5308] avg=4096.0000 max=6204 max_rank=5 min=1907 min_rank=4 imbalance_ratio=1.514648 max_load_reduction=26564 max_load_reduction_pct=0.810669 moved_to_echo=19078 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3119, 0, 285, 0, 7, 0, 7, 0, 1458, 478, 0, 0, 0, 0, 0, 0, 2, 11, 0, 3832, 27, 0, 9, 0, 821, 256, 0, 0, 0, 0, 0, 0, 0, 2372, 0, 2493, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4075, 79, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 1, 0, 115, 0, 2116, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 652, 902, 109, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3014, 0, 156, 1, 1, 0, 1807, 0, 0, 0, 0, 0, 0, 0, 0, 464, 0, 0, 0, 0, 0, 3057, 0, 931, 47, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5354, 4958, 4866, 4177, 2255, 1680, 4979, 4499] avg=4096.0000 max=5354 max_rank=0 min=1680 min_rank=5 imbalance_ratio=1.307129 max_load_reduction=27414 max_load_reduction_pct=0.836609 moved_to_echo=13413 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3805, 1, 4, 0, 19, 0, 1, 2, 1390, 597, 0, 0, 0, 0, 0, 0, 4, 239, 0, 87, 11, 1, 0, 0, 2267, 244, 0, 0, 0, 0, 0, 0, 11, 2261, 282, 0, 0, 21, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 51, 2911, 2876, 0, 1, 163, 0, 0, 0, 0, 0, 0, 0, 102, 0, 865, 0, 2606, 0, 1419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1801, 1813, 1651, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 304, 2, 0, 2, 0, 314, 2256, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 2, 0, 0, 688, 0, 1163, 356, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5819, 2853, 2578, 6003, 4992, 5346, 2884, 2293] avg=4096.0000 max=6003 max_rank=3 min=2293 min_rank=7 imbalance_ratio=1.465576 max_load_reduction=26765 max_load_reduction_pct=0.816803 moved_to_echo=15515 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[471, 0, 0, 0, 11, 1, 0, 0, 1336, 518, 0, 0, 0, 0, 0, 0, 0, 465, 5, 493, 0, 0, 0, 0, 2158, 234, 0, 0, 0, 0, 0, 0, 1, 2175, 10, 3891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 42, 0, 1, 470, 0, 0, 0, 0, 0, 0, 0, 53, 16, 2489, 0, 1, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1714, 464, 4019, 691, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 69, 0, 0, 2, 48, 0, 1959, 0, 0, 0, 0, 0, 0, 0, 0, 257, 0, 0, 0, 0, 0, 2564, 0, 1010, 1028, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2337, 3355, 6077, 4608, 2562, 6891, 2079, 4859] avg=4096.0000 max=6891 max_rank=5 min=2079 min_rank=6 imbalance_ratio=1.682373 max_load_reduction=25877 max_load_reduction_pct=0.789703 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3236, 1, 0, 0, 1, 0, 1, 11, 1463, 608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 1, 2227, 261, 0, 0, 0, 0, 0, 0, 0, 2363, 12, 2281, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 18, 1, 428, 5, 478, 1, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1762, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 3515, 4, 4, 3, 0, 0, 2274, 0, 0, 0, 0, 0, 0, 0, 0, 340, 0, 677, 0, 0, 0, 42, 0, 1179, 184, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5321, 6579, 4658, 4175, 1035, 1765, 6813, 2422] avg=4096.0000 max=6813 max_rank=6 min=1035 min_rank=4 imbalance_ratio=1.663330 max_load_reduction=25955 max_load_reduction_pct=0.792084 moved_to_echo=12035 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[6912, 128, 768, 128, 128, 128, 11648, 256, 8320, 5376, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[2944, 128, 12800, 128, 12672, 128, 4480, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1024, 14592, 384, 256, 256, 256, 512, 16128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[15744, 128, 512, 256, 640, 128, 128, 128, 11520, 4352, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 12416, 9600, 9984, 1024, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[256, 1536, 1536, 12032, 128, 128, 128, 128, 15616, 2048, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 18688, 384, 13824, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 24448, 5888, 0, 128, 2432, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33792 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[25, 1, 0, 0, 3788, 2941, 390, 3998, 0, 107, 0, 0, 0, 0, 0, 216, 1235, 0, 3896, 0, 62, 0, 0, 0, 0, 1, 38, 4042, 0, 3339, 1, 7, 0, 3753, 931, 3454, 0, 0, 0, 0, 14, 0, 0, 59, 0, 11, 28, 0, 1, 20, 0, 0, 0, 122, 0, 0, 0, 0, 1, 61, 53, 0, 0, 173] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[438, 3, 426, 0, 4096, 4061, 1520, 3772, 0, 0, 99, 0, 0, 0, 0, 0, 2650, 0, 2947, 0, 1596, 0, 0, 0, 185, 0, 3, 1354, 0, 3763, 0, 16, 361, 2125, 6, 180, 0, 19, 0, 18, 2, 0, 1, 0, 0, 5, 0, 2, 0, 0, 0, 13, 0, 0, 1, 0, 0, 115, 39, 0, 2952, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 94, 0, 4096, 4052, 573, 3911, 0, 0, 239, 0, 0, 0, 0, 1429, 3408, 0, 69, 0, 0, 9, 0, 0, 0, 0, 0, 3153, 0, 4079, 0, 0, 20, 4096, 6, 3400, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 16, 0, 35, 0, 68] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4073, 3920, 42, 4082, 0, 0, 0, 0, 0, 423, 0, 63, 3282, 0, 3235, 0, 0, 6, 0, 39, 1333, 0, 188, 3788, 0, 4063, 0, 0, 21, 4046, 1, 67, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 7, 1, 6, 0, 16, 23, 0, 0, 0, 0, 0, 6, 0, 0, 31, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 279, 0, 3923, 4038, 0, 4084, 35, 0, 0, 214, 20, 0, 0, 0, 2140, 7, 3408, 0, 43, 6, 0, 0, 0, 6, 0, 2992, 0, 2818, 0, 1, 0, 4066, 0, 3731, 0, 0, 0, 0, 0, 0, 0, 927, 0, 0, 0, 2, 0, 8, 0, 1, 0, 0, 0, 1, 0, 2, 0, 0, 6, 10, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 464, 1, 4049, 4063, 367, 4095, 0, 2, 0, 0, 0, 0, 0, 1648, 876, 0, 3302, 0, 0, 1, 33, 0, 118, 0, 0, 3924, 0, 4093, 0, 2, 3, 4092, 0, 1344, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 6, 0, 0, 97, 1, 0, 0, 174] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 1180, 0, 4074, 3637, 467, 4055, 0, 74, 0, 3, 22, 0, 0, 0, 3024, 20, 3980, 94, 218, 0, 0, 1, 0, 0, 227, 143, 0, 3453, 5, 0, 120, 3958, 0, 3786, 0, 139, 0, 0, 2, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4, 0, 0, 22, 0, 1, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[5, 0, 485, 0, 4096, 4089, 910, 3914, 1, 0, 83, 0, 0, 1364, 1, 506, 1200, 0, 0, 0, 101, 0, 0, 0, 67, 0, 0, 4094, 0, 4096, 1, 1, 133, 4094, 20, 3474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 21] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[25, 1, 0, 0, 0, 2397, 390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 0, 0, 0, 0, 0, 216, 1538, 1518, 544, 0, 0, 0, 0, 0, 1235, 0, 2378, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 38, 4042, 0, 796, 1, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1432, 931, 3454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 59, 0, 11, 28, 0, 3955, 7, 0, 0, 0, 0, 0, 0, 1, 20, 0, 0, 0, 122, 0, 0, 3788, 43, 0, 0, 0, 0, 0, 0, 0, 0, 1, 61, 53, 0, 0, 173, 2536, 783, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2813, 3923, 3675, 4885, 5817, 4074, 3974, 3607] avg=4096.0000 max=5817 max_rank=4 min=2813 min_rank=0 imbalance_ratio=1.420166 max_load_reduction=26951 max_load_reduction_pct=0.822479 moved_to_echo=17472 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[438, 3, 426, 0, 0, 3309, 1520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 0, 0, 0, 0, 0, 871, 1148, 752, 0, 0, 0, 0, 0, 2650, 0, 1799, 0, 1596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 0, 3, 1354, 0, 897, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 361, 811, 6, 180, 0, 19, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 5, 0, 2, 3731, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 1, 0, 4096, 41, 0, 0, 0, 0, 0, 0, 0, 115, 39, 0, 2952, 0, 0, 0, 2858, 443, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5696, 2870, 6045, 2455, 1395, 3749, 4151, 6407] avg=4096.0000 max=6407 max_rank=7 min=1395 min_rank=4 imbalance_ratio=1.564209 max_load_reduction=26361 max_load_reduction_pct=0.804474 moved_to_echo=15702 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 464, 1, 0, 3311, 367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1648, 1677, 1286, 752, 0, 0, 0, 0, 0, 876, 0, 2016, 0, 0, 1, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 0, 0, 3924, 0, 977, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1561, 0, 1344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 4051, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4049, 44, 0, 0, 0, 0, 0, 0, 6, 0, 0, 97, 1, 0, 0, 174, 3108, 854, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4143, 5365, 2926, 5021, 2908, 4063, 4102, 4240] avg=4096.0000 max=5365 max_rank=1 min=2908 min_rank=4 imbalance_ratio=1.309814 max_load_reduction=27403 max_load_reduction_pct=0.836273 moved_to_echo=15313 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 0, 3194, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 423, 0, 63, 1658, 1260, 726, 0, 0, 0, 0, 0, 3282, 0, 1975, 0, 0, 6, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 1333, 0, 188, 3788, 0, 969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 1544, 1, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 7, 1, 4038, 8, 0, 0, 0, 0, 0, 0, 6, 0, 16, 23, 0, 0, 0, 0, 4073, 44, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 31, 1, 0, 0, 3086, 844, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3238, 4130, 5302, 6278, 1633, 4057, 4162, 3968] avg=4096.0000 max=6278 max_rank=3 min=1633 min_rank=4 imbalance_ratio=1.532715 max_load_reduction=26490 max_load_reduction_pct=0.808411 moved_to_echo=13820 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 94, 0, 0, 3302, 573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 239, 0, 0, 0, 0, 1429, 1679, 26, 750, 0, 0, 0, 0, 0, 3408, 0, 43, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3153, 0, 973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 1563, 6, 3400, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3869, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 4096, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 35, 0, 68, 3098, 854, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3969, 4123, 3460, 4126, 4997, 3880, 4142, 4071] avg=4096.0000 max=4997 max_rank=4 min=3460 min_rank=2 imbalance_ratio=1.219971 max_load_reduction=27771 max_load_reduction_pct=0.847504 moved_to_echo=17090 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1180, 0, 0, 2964, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 3, 22, 0, 0, 0, 1622, 1554, 673, 0, 0, 0, 0, 0, 3024, 20, 2426, 94, 218, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 227, 143, 0, 824, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 1510, 0, 3786, 0, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 56, 0, 4012, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4074, 43, 0, 0, 0, 0, 0, 0, 4, 0, 0, 22, 0, 1, 0, 1, 2622, 826, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4611, 3948, 5783, 1199, 5555, 4077, 4119, 3476] avg=4096.0000 max=5783 max_rank=2 min=1199 min_rank=3 imbalance_ratio=1.411865 max_load_reduction=26985 max_load_reduction_pct=0.823517 moved_to_echo=17227 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 279, 0, 0, 3291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 214, 20, 0, 0, 0, 1667, 1328, 747, 0, 0, 0, 0, 0, 2140, 7, 2080, 0, 43, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2992, 0, 672, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1551, 0, 3731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 927, 0, 0, 0, 2, 4040, 6, 0, 0, 0, 0, 0, 0, 0, 8, 0, 1, 0, 0, 0, 1, 3923, 44, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 6, 10, 0, 0, 2140, 848, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3570, 4011, 4276, 3671, 5282, 4975, 3977, 3006] avg=4096.0000 max=5282 max_rank=4 min=3006 min_rank=7 imbalance_ratio=1.289551 max_load_reduction=27486 max_load_reduction_pct=0.838806 moved_to_echo=17240 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 485, 0, 0, 3328, 910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 83, 0, 0, 1364, 1, 506, 1682, 0, 761, 0, 0, 0, 0, 0, 1200, 0, 0, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 4094, 0, 970, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 133, 1554, 20, 3474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 3873, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 4096, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 3114, 858, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4728, 4398, 1301, 5133, 5181, 3893, 4141, 3993] avg=4096.0000 max=5181 max_rank=4 min=1301 min_rank=2 imbalance_ratio=1.264893 max_load_reduction=27587 max_load_reduction_pct=0.841888 moved_to_echo=17208 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 256, 512, 256, 128, 1792, 128, 3968, 12416, 8192, 5760, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[768, 11648, 1024, 19456, 0, 256, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 256, 3072, 128, 0, 512, 22656, 6400, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1792, 128, 512, 23552, 0, 7168, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[17920, 128, 12800, 128, 2048, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[512, 128, 2944, 128, 0, 25216, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 128, 256, 128, 128, 32256, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 0, 128, 1024, 128, 128, 128, 128, 31616, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33792 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1993, 0, 0, 0, 9, 0, 0, 122, 116, 0, 236, 0, 9, 0, 0, 7, 4, 0, 4068, 3970, 4, 0, 0, 392, 10, 34, 3623, 4071, 0, 4, 25, 0, 0, 0, 0, 0, 1612, 8, 1018, 0, 414, 0, 0, 0, 0, 10, 0, 3511, 3899, 2, 145, 0, 0, 0, 0, 0, 37, 1, 3391, 23, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 2, 346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1291, 0, 199, 4064, 4094, 136, 0, 3077, 0, 0, 0, 821, 3873, 0, 0, 1, 3501, 0, 13, 0, 0, 176, 63, 2040, 0, 0, 8, 0, 0, 0, 0, 0, 3861, 0, 4, 51, 1, 9, 6, 0, 2, 0, 0, 3079, 0, 0, 2043, 0, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 195, 0, 1, 0, 0, 0, 26, 2028, 279, 2, 4034, 4022, 0, 1190, 68, 0, 36, 0, 4047, 4068, 1, 0, 1, 0, 0, 0, 3239, 0, 177, 236, 838, 7, 0, 1, 23, 0, 4, 0, 16, 4048, 0, 0, 8, 0, 64, 1, 0, 0, 0, 0, 4094, 0, 13, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 290, 2593, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 4096, 0, 0, 4083, 4077, 0, 0, 0, 0, 2, 0, 0, 4093, 0, 0, 0, 3, 0, 0, 6, 0, 3729, 0, 3456, 32, 329, 0, 26, 0, 0, 0, 3, 1769, 3, 86, 26, 0, 0, 0, 0, 0, 0, 0, 4056, 0, 4, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[8, 2, 1887, 0, 38, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3639, 35, 45, 4091, 4035, 125, 77, 0, 23, 0, 0, 3086, 4056, 0, 0, 2, 65, 0, 0, 6, 1, 2041, 0, 1372, 0, 0, 0, 47, 0, 43, 0, 0, 3986, 0, 0, 0, 0, 0, 7, 0, 0, 2, 2, 4046, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[123, 0, 1045, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 0, 36, 791, 876, 0, 4011, 3886, 0, 2, 9, 0, 0, 0, 4002, 3387, 0, 7, 0, 0, 2, 0, 1540, 0, 170, 2055, 4080, 1, 0, 0, 4, 0, 0, 0, 127, 3353, 0, 101, 0, 22, 0, 0, 0, 0, 11, 0, 3035, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 606, 0, 103, 0, 102, 0, 0, 0, 8, 0, 0, 0, 0, 4026, 0, 1, 4096, 4052, 0, 0, 1895, 0, 0, 0, 172, 4094, 0, 0, 0, 1003, 0, 1, 0, 0, 158, 3125, 4073, 1, 0, 0, 0, 92, 2, 0, 0, 4092, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 907, 0, 0, 156, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[47, 0, 0, 0, 0, 1, 2969, 0, 1, 0, 0, 0, 0, 1, 0, 469, 0, 1, 4092, 3789, 0, 395, 712, 8, 0, 0, 116, 4074, 0, 0, 0, 0, 0, 2, 0, 0, 3487, 1, 4067, 1, 2, 0, 0, 10, 312, 0, 0, 4016, 0, 0, 0, 1, 12, 0, 0, 0, 401, 0, 3679, 0, 0, 102, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 346, 0, 0, 0, 0, 0, 1985, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1291, 980, 396, 304, 0, 0, 0, 0, 0, 0, 199, 0, 2979, 136, 0, 3077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 821, 1492, 0, 0, 1, 3501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 176, 63, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 3861, 340, 0, 0, 0, 0, 0, 0, 0, 0, 4, 51, 1, 9, 6, 0, 2, 3536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3079, 0, 0, 2043, 0, 7, 471, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2861, 2971, 6391, 5815, 1312, 4209, 3609, 5600] avg=4096.0000 max=6391 max_rank=2 min=1312 min_rank=4 imbalance_ratio=1.560303 max_load_reduction=26377 max_load_reduction_pct=0.804962 moved_to_echo=14730 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[8, 2, 1887, 0, 38, 0, 0, 0, 2079, 530, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3639, 659, 415, 300, 0, 0, 0, 0, 0, 35, 45, 0, 2935, 125, 77, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3086, 1562, 0, 0, 2, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 2041, 0, 713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 43, 0, 0, 3986, 336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 3561, 0, 0, 0, 0, 0, 0, 0, 2, 2, 4046, 0, 0, 0, 0, 0, 464, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4544, 5014, 3240, 4715, 2761, 4412, 3568, 4514] avg=4096.0000 max=5014 max_rank=1 min=2761 min_rank=4 imbalance_ratio=1.224121 max_load_reduction=27754 max_load_reduction_pct=0.846985 moved_to_echo=15255 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 290, 2593, 0, 0, 0, 0, 2098, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 4096, 1661, 419, 303, 0, 0, 0, 0, 0, 0, 0, 0, 2966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1576, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 3729, 0, 1795, 32, 0, 0, 0, 0, 0, 0, 0, 0, 329, 0, 26, 0, 0, 0, 3, 1769, 339, 0, 0, 0, 0, 0, 0, 0, 3, 86, 26, 0, 0, 0, 0, 0, 3553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 0, 4, 0, 0, 0, 469, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5511, 6485, 2966, 1581, 5562, 2466, 3668, 4529] avg=4096.0000 max=6485 max_rank=1 min=1581 min_rank=3 imbalance_ratio=1.583252 max_load_reduction=26283 max_load_reduction_pct=0.802094 moved_to_echo=16225 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 2085, 524, 0, 0, 0, 0, 0, 0, 195, 0, 1, 0, 0, 0, 26, 2028, 402, 416, 299, 0, 0, 0, 0, 0, 279, 2, 0, 2926, 0, 1190, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 4047, 1567, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3239, 0, 177, 236, 436, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 23, 0, 4, 0, 16, 4048, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 64, 1, 0, 0, 3510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 13, 0, 1, 0, 463, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2609, 3367, 4465, 5652, 4095, 4426, 3583, 4571] avg=4096.0000 max=5652 max_rank=3 min=2609 min_rank=0 imbalance_ratio=1.379883 max_load_reduction=27116 max_load_reduction_pct=0.827515 moved_to_echo=16675 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1993, 0, 0, 0, 9, 0, 0, 122, 2087, 527, 0, 0, 0, 0, 0, 0, 116, 0, 236, 0, 9, 0, 0, 7, 489, 417, 295, 0, 0, 0, 0, 0, 4, 0, 0, 2888, 4, 0, 0, 392, 0, 0, 0, 0, 0, 0, 0, 0, 10, 34, 3623, 1567, 0, 4, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1612, 8, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 414, 0, 0, 0, 0, 10, 0, 3511, 330, 0, 0, 0, 0, 0, 0, 0, 3899, 2, 145, 0, 0, 0, 0, 0, 3541, 0, 0, 0, 0, 0, 0, 0, 37, 1, 3391, 23, 0, 0, 0, 0, 457, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4738, 1569, 3288, 5263, 2149, 4265, 7587, 3909] avg=4096.0000 max=7587 max_rank=6 min=1569 min_rank=1 imbalance_ratio=1.852295 max_load_reduction=25181 max_load_reduction_pct=0.768463 moved_to_echo=17910 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[123, 0, 1045, 0, 0, 0, 0, 0, 1736, 520, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 0, 36, 791, 1960, 347, 289, 0, 0, 0, 0, 0, 876, 0, 0, 2827, 0, 2, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4002, 1304, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1540, 0, 170, 2055, 2120, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 127, 3353, 323, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 22, 0, 0, 0, 0, 3491, 0, 0, 0, 0, 0, 0, 0, 11, 0, 3035, 0, 0, 0, 0, 0, 447, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3424, 3515, 3714, 5313, 5888, 3807, 3614, 3493] avg=4096.0000 max=5888 max_rank=4 min=3424 min_rank=0 imbalance_ratio=1.437500 max_load_reduction=26880 max_load_reduction_pct=0.820312 moved_to_echo=16802 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 606, 0, 103, 0, 102, 0, 2098, 532, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 4026, 1957, 419, 301, 0, 0, 0, 0, 0, 0, 1, 0, 2948, 0, 0, 1895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 1577, 0, 0, 0, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 158, 3125, 2116, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 2, 0, 0, 4092, 337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 907, 0, 0, 156, 0, 2, 466, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3441, 6711, 4844, 2752, 5401, 4523, 3565, 1531] avg=4096.0000 max=6711 max_rank=1 min=1531 min_rank=7 imbalance_ratio=1.638428 max_load_reduction=26057 max_load_reduction_pct=0.795197 moved_to_echo=15020 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[47, 0, 0, 0, 0, 1, 2969, 0, 2092, 531, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 469, 1958, 421, 286, 0, 0, 0, 0, 0, 0, 1, 0, 2744, 0, 395, 712, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 1561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3487, 1, 2109, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 10, 312, 0, 0, 4016, 320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 12, 0, 0, 0, 3561, 0, 0, 0, 0, 0, 0, 0, 401, 0, 3679, 0, 0, 102, 0, 0, 439, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5640, 3136, 3860, 1677, 5600, 4660, 3574, 4621] avg=4096.0000 max=5640 max_rank=0 min=1677 min_rank=3 imbalance_ratio=1.376953 max_load_reduction=27128 max_load_reduction_pct=0.827881 moved_to_echo=18455 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[2176, 128, 4224, 2688, 256, 128, 3072, 128, 16384, 4224, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 15872, 12288, 128, 128, 128, 4608, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[512, 128, 256, 0, 128, 128, 128, 16384, 10112, 3328, 2432, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 4864, 128, 11648, 5504, 10880, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1280, 256, 0, 23296, 384, 1664, 5888, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[512, 128, 26368, 128, 128, 2304, 128, 128, 3712, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3968, 256, 256, 128, 128, 128, 0, 128, 28416, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[768, 128, 128, 128, 384, 128, 256, 28672, 2688, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 4066, 0, 0, 3868, 0, 0, 0, 1, 0, 0, 0, 2, 0, 4096, 10, 0, 4009, 0, 29, 0, 0, 65, 10, 66, 0, 9, 3, 0, 3168, 0, 119, 60, 0, 387, 0, 0, 48, 0, 0, 0, 0, 0, 0, 1, 3873, 35, 0, 0, 3763, 3485, 0, 0, 0, 0, 429, 0, 1076, 0, 0, 90, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 8, 0, 4085, 0, 109, 3066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2816, 0, 0, 0, 788, 24, 10, 479, 2, 0, 116, 0, 4017, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 4022, 0, 81, 0, 0, 0, 0, 22, 0, 829, 0, 0, 2237, 3759, 0, 4, 0, 0, 3228, 0, 3019, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 4074, 1, 1, 4061, 0, 0, 11, 0, 0, 3, 81, 0, 0, 4080, 4, 0, 3925, 1, 0, 8, 0, 25, 0, 0, 8, 1122, 0, 0, 4078, 7, 0, 0, 0, 0, 0, 273, 17, 5, 0, 0, 0, 41, 2, 0, 239, 0, 0, 0, 4075, 0, 0, 249, 0, 1, 2913, 0, 3373, 27, 0, 0, 63] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 4073, 35, 0, 3930, 0, 3, 0, 0, 227, 0, 0, 0, 0, 3739, 0, 0, 2496, 16, 0, 0, 0, 0, 63, 1402, 1, 3821, 111, 0, 468, 0, 0, 0, 0, 27, 0, 7, 117, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 20, 3984, 34, 0, 0, 20, 0, 3560, 546, 10, 1, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 4040, 0, 0, 432, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3375, 0, 0, 3953, 0, 0, 0, 0, 0, 0, 436, 0, 974, 0, 0, 978, 0, 534, 0, 87, 9, 21, 237, 4094, 0, 0, 0, 603, 0, 0, 0, 0, 1785, 0, 3, 3441, 1083, 0, 0, 0, 0, 1132, 0, 3988, 6, 0, 1549, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 3962, 1, 0, 3161, 1, 0, 0, 37, 0, 0, 0, 0, 4, 4010, 7, 0, 3960, 0, 0, 0, 46, 467, 0, 0, 0, 2470, 0, 15, 582, 8, 15, 0, 1, 0, 2, 603, 1735, 0, 0, 2, 0, 2, 0, 8, 0, 983, 0, 3, 4096, 3611, 0, 0, 0, 0, 0, 0, 2820, 0, 6, 150, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 2, 0, 4091, 0, 0, 3589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 0, 0, 4091, 0, 0, 0, 2, 1, 0, 0, 0, 4080, 4, 0, 272, 0, 0, 0, 1, 111, 0, 0, 394, 0, 0, 0, 0, 0, 6, 0, 1, 834, 0, 0, 4096, 19, 0, 1, 0, 0, 4094, 0, 3025, 0, 0, 0, 18] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 4073, 0, 0, 3538, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4059, 1, 0, 4094, 2, 0, 0, 0, 0, 0, 2132, 0, 4085, 7, 1, 3, 0, 1, 0, 0, 3187, 0, 0, 99, 0, 0, 0, 1, 0, 0, 0, 527, 37, 0, 0, 4093, 74, 0, 0, 0, 0, 2341, 0, 387, 0, 0, 22, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 874, 35, 0, 3930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 227, 0, 0, 0, 0, 3199, 853, 0, 0, 0, 0, 0, 0, 596, 0, 0, 2496, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 1402, 1, 3396, 111, 0, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 7, 117, 1216, 2, 425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2290, 1183, 0, 0, 0, 0, 0, 0, 4057, 0, 20, 1585, 34, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3560, 546, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4839, 4282, 3108, 5441, 1794, 3473, 5716, 4115] avg=4096.0000 max=5716 max_rank=6 min=1794 min_rank=4 imbalance_ratio=1.395508 max_load_reduction=27052 max_load_reduction_pct=0.825562 moved_to_echo=15098 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 873, 0, 0, 3868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 3193, 934, 0, 0, 0, 0, 0, 0, 654, 10, 0, 4009, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 10, 66, 0, 8, 3, 0, 3168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 60, 0, 387, 0, 0, 48, 1148, 310, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3873, 2508, 1118, 0, 0, 0, 0, 0, 0, 35, 0, 0, 1497, 3485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 429, 0, 766, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4741, 4130, 4702, 3320, 2073, 7500, 5017, 1285] avg=4096.0000 max=7500 max_rank=5 min=1285 min_rank=7 imbalance_ratio=1.831055 max_load_reduction=25268 max_load_reduction_pct=0.771118 moved_to_echo=15875 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 875, 1, 1, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 3, 81, 0, 0, 3199, 931, 0, 0, 0, 0, 0, 0, 651, 4, 0, 3925, 1, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 8, 997, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 273, 17, 1243, 974, 125, 0, 0, 0, 0, 0, 5, 0, 0, 0, 41, 2, 0, 239, 2498, 1210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1622, 0, 0, 249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2913, 0, 2399, 27, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4938, 4225, 4589, 5108, 2639, 3995, 1871, 5403] avg=4096.0000 max=5403 max_rank=7 min=1871 min_rank=6 imbalance_ratio=1.319092 max_load_reduction=27365 max_load_reduction_pct=0.835114 moved_to_echo=13908 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 867, 0, 0, 432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3173, 770, 0, 0, 0, 0, 0, 0, 538, 0, 0, 3953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 436, 0, 866, 0, 0, 978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 0, 87, 9, 21, 237, 4094, 1050, 1151, 108, 0, 0, 0, 0, 0, 0, 0, 0, 603, 0, 0, 0, 0, 2067, 1022, 0, 0, 0, 0, 0, 0, 1785, 0, 3, 1369, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1132, 0, 2837, 6, 0, 1549, 4, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1299, 3947, 4491, 2280, 7291, 3692, 4240, 5528] avg=4096.0000 max=7291 max_rank=4 min=1299 min_rank=0 imbalance_ratio=1.780029 max_load_reduction=25477 max_load_reduction_pct=0.777496 moved_to_echo=20751 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 851, 1, 0, 3161, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 4, 3111, 915, 0, 0, 0, 0, 0, 0, 639, 7, 0, 3960, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 467, 0, 0, 0, 2195, 0, 15, 582, 0, 0, 0, 0, 0, 0, 0, 0, 8, 15, 0, 1, 0, 2, 603, 1735, 1250, 814, 275, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 8, 0, 2456, 1217, 0, 0, 0, 0, 0, 0, 983, 0, 3, 1629, 3611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2006, 0, 6, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4014, 4067, 4652, 3259, 4703, 3685, 6226, 2162] avg=4096.0000 max=6226 max_rank=6 min=2162 min_rank=7 imbalance_ratio=1.520020 max_load_reduction=26542 max_load_reduction_pct=0.809998 moved_to_echo=16776 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 878, 0, 0, 3589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3213, 920, 0, 0, 0, 0, 0, 0, 644, 0, 0, 4091, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3626, 4, 0, 272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 111, 0, 0, 394, 1250, 873, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1, 2472, 1217, 0, 0, 0, 0, 0, 0, 834, 0, 0, 1629, 19, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2152, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4469, 4133, 4737, 3903, 3083, 3696, 2483, 6264] avg=4096.0000 max=6264 max_rank=7 min=2483 min_rank=6 imbalance_ratio=1.529297 max_load_reduction=26504 max_load_reduction_pct=0.808838 moved_to_echo=15526 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 870, 0, 0, 3538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3203, 929, 0, 0, 0, 0, 0, 0, 641, 1, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2132, 0, 3626, 7, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3187, 0, 0, 99, 1253, 116, 459, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 527, 2489, 1219, 0, 0, 0, 0, 0, 0, 37, 0, 0, 1621, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2341, 0, 271, 0, 0, 22, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4408, 4134, 4738, 5769, 5115, 4236, 1732, 2636] avg=4096.0000 max=5769 max_rank=3 min=1732 min_rank=6 imbalance_ratio=1.408447 max_load_reduction=26999 max_load_reduction_pct=0.823944 moved_to_echo=13719 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 0, 877, 0, 109, 3066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3208, 642, 0, 0, 0, 0, 0, 0, 450, 0, 0, 0, 788, 24, 10, 479, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 116, 0, 3570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 4022, 682, 872, 447, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 22, 0, 1724, 664, 0, 0, 0, 0, 0, 0, 829, 0, 0, 891, 3759, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3228, 0, 2147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 3850, 1751, 3688, 6070, 2491, 5483, 5375] avg=4096.0000 max=6070 max_rank=4 min=1751 min_rank=2 imbalance_ratio=1.481934 max_load_reduction=26698 max_load_reduction_pct=0.814758 moved_to_echo=19419 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 768, 128, 128, 3840, 128, 1152, 10624, 9216, 5120, 2304, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 640, 128, 128, 128, 4736, 18560, 8960, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8576, 0, 128, 11904, 12160, 0, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 256, 128, 128, 128, 128, 25600, 6912, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[640, 128, 4224, 128, 18304, 128, 128, 9600, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 17792, 640, 12672, 128, 128, 1920, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 0, 7040, 128, 128, 25728, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4864, 128, 0, 26624, 896, 128, 128, 640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 119, 101, 0, 0, 474, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 4095, 0, 0, 0, 0, 0, 0, 4089, 3, 0, 970, 0, 0, 0, 0, 13, 0, 0, 0, 0, 1866, 3488, 0, 0, 343, 977, 18, 9, 1, 0, 4041, 0, 0, 46, 0, 6, 0, 0, 0, 4089, 0, 58, 0, 0, 0, 0, 0, 0, 3858] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[5, 0, 0, 0, 0, 3059, 0, 0, 0, 0, 0, 70, 6, 4021, 0, 4015, 0, 0, 0, 0, 0, 3, 1896, 4, 0, 0, 8, 17, 0, 0, 1, 0, 0, 25, 0, 232, 27, 0, 0, 4004, 0, 1, 1664, 0, 0, 4079, 7, 0, 0, 0, 65, 0, 0, 9, 4096, 0, 1640, 0, 0, 2, 0, 0, 0, 3812] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 454, 4, 0, 3967, 149, 0, 2, 0, 1, 0, 119, 3953, 0, 3803, 0, 12, 6, 0, 99, 1, 3966, 1, 0, 3586, 0, 0, 0, 31, 196, 0, 0, 0, 0, 1773, 4027, 0, 0, 0, 0, 162, 0, 0, 3216, 2793, 0, 1, 2, 0, 0, 0, 22, 0, 3, 0, 343, 1, 0, 0, 0, 59, 0, 16] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[57, 3353, 0, 0, 0, 2364, 0, 0, 0, 23, 0, 0, 0, 4093, 0, 4076, 0, 0, 0, 0, 0, 0, 4035, 0, 0, 1379, 477, 0, 0, 259, 0, 0, 1, 0, 0, 7, 0, 0, 0, 3491, 757, 0, 153, 0, 43, 4085, 1, 0, 0, 0, 0, 0, 2, 11, 4018, 0, 22, 0, 0, 0, 0, 41, 0, 20] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[16, 1, 0, 3, 116, 4054, 4, 0, 3, 0, 42, 36, 0, 4045, 1, 4052, 0, 0, 465, 0, 0, 0, 2443, 0, 0, 4, 0, 0, 0, 111, 0, 0, 0, 0, 1, 56, 2256, 0, 0, 13, 0, 0, 176, 0, 3290, 3801, 2, 0, 0, 0, 3662, 0, 0, 2, 4096, 0, 0, 0, 2, 0, 0, 1, 1, 13] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 8, 1, 0, 6, 132, 208, 0, 0, 2, 0, 0, 2, 3881, 100, 3859, 0, 0, 0, 0, 0, 0, 4093, 0, 2, 2450, 4, 0, 0, 0, 1, 0, 0, 0, 35, 3059, 234, 1, 2, 1130, 0, 0, 334, 2, 0, 3903, 1, 0, 20, 0, 959, 31, 0, 224, 4083, 0, 0, 0, 0, 0, 63, 0, 0, 3938] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[560, 106, 0, 6, 0, 17, 1, 0, 0, 0, 13, 0, 178, 3522, 1, 4025, 1, 0, 0, 21, 0, 0, 1683, 824, 0, 3850, 0, 0, 0, 1, 1389, 0, 0, 0, 74, 2465, 795, 0, 0, 4083, 39, 0, 3, 0, 5, 3857, 27, 0, 27, 4, 1218, 0, 0, 14, 3912, 0, 0, 31, 0, 0, 0, 0, 16, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 0, 25, 0, 0, 2, 4096, 4, 4067, 0, 0, 0, 0, 1, 2, 4058, 11, 0, 439, 0, 0, 0, 0, 2750, 1, 0, 0, 0, 0, 3040, 0, 0, 2055, 348, 0, 1, 0, 11, 3289, 4, 0, 0, 0, 428, 0, 0, 0, 4017, 0, 14, 0, 3, 0, 0, 6, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[57, 3353, 0, 0, 0, 2364, 0, 0, 1140, 707, 258, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 4093, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 0, 635, 326, 0, 0, 0, 0, 0, 0, 0, 1379, 477, 0, 0, 259, 0, 0, 1889, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7, 0, 0, 0, 2149, 0, 0, 0, 0, 0, 0, 0, 0, 757, 0, 153, 0, 43, 2945, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 11, 3692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 41, 0, 20, 1874, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7879, 4171, 4996, 4004, 2157, 3899, 3705, 1957] avg=4096.0000 max=7879 max_rank=0 min=1957 min_rank=7 imbalance_ratio=1.923584 max_load_reduction=24889 max_load_reduction_pct=0.759552 moved_to_echo=11718 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 119, 101, 0, 0, 474, 0, 0, 1128, 69, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3, 62, 331, 0, 0, 0, 0, 0, 0, 0, 970, 0, 0, 0, 0, 13, 0, 1898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1866, 3488, 0, 0, 212, 0, 0, 0, 0, 0, 0, 0, 0, 977, 18, 9, 1, 0, 2913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 6, 0, 0, 0, 3758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 3858, 1883, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2150, 4159, 4485, 2881, 5566, 3918, 3810, 5799] avg=4096.0000 max=5799 max_rank=7 min=2150 min_rank=0 imbalance_ratio=1.415771 max_load_reduction=26969 max_load_reduction_pct=0.823029 moved_to_echo=19093 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 0, 0, 0, 3059, 0, 0, 1138, 810, 254, 0, 0, 0, 0, 0, 0, 0, 0, 70, 6, 4021, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1896, 4, 729, 332, 0, 0, 0, 0, 0, 0, 0, 0, 8, 17, 0, 0, 1, 0, 1861, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 232, 27, 0, 0, 2465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1664, 0, 0, 2941, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 9, 3764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1640, 0, 0, 2, 0, 0, 0, 3812, 1846, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5266, 4151, 2964, 1887, 2749, 4613, 3838, 7300] avg=4096.0000 max=7300 max_rank=7 min=1887 min_rank=3 imbalance_ratio=1.782227 max_load_reduction=25468 max_load_reduction_pct=0.777222 moved_to_echo=18500 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 454, 4, 0, 3967, 149, 0, 779, 0, 240, 0, 0, 0, 0, 0, 2, 0, 1, 0, 119, 3953, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 6, 0, 99, 1, 3966, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3586, 0, 0, 0, 31, 196, 0, 1762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1773, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 0, 0, 3216, 2014, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 22, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 343, 1, 0, 0, 0, 59, 0, 16, 1748, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5593, 4128, 4085, 5575, 5800, 5393, 27, 2167] avg=4096.0000 max=5800 max_rank=4 min=27 min_rank=6 imbalance_ratio=1.416016 max_load_reduction=26968 max_load_reduction_pct=0.822998 moved_to_echo=13387 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 1, 0, 6, 132, 208, 0, 1089, 228, 244, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 3881, 100, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 205, 331, 0, 0, 0, 0, 0, 0, 2, 2450, 4, 0, 0, 0, 1, 0, 1788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 3059, 234, 1, 2, 697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 334, 2, 0, 2814, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 959, 31, 0, 224, 3752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 3938, 1774, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1916, 4038, 4629, 4245, 4028, 3151, 4986, 5775] avg=4096.0000 max=5775 max_rank=7 min=1916 min_rank=0 imbalance_ratio=1.409912 max_load_reduction=26993 max_load_reduction_pct=0.823761 moved_to_echo=17940 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[560, 106, 0, 6, 0, 17, 1, 0, 1076, 826, 255, 0, 0, 0, 0, 0, 0, 0, 13, 0, 178, 3522, 1, 54, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 21, 0, 0, 1683, 824, 743, 317, 0, 0, 0, 0, 0, 0, 0, 3850, 0, 0, 0, 1, 1389, 0, 1865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 2465, 795, 0, 0, 2514, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 3, 0, 5, 2781, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 4, 1218, 0, 0, 14, 3595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 16, 0, 1851, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2847, 3768, 3589, 7105, 5848, 2855, 4858, 1898] avg=4096.0000 max=7105 max_rank=3 min=1898 min_rank=7 imbalance_ratio=1.734619 max_load_reduction=25663 max_load_reduction_pct=0.783173 moved_to_echo=15459 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[16, 1, 0, 3, 116, 4054, 4, 0, 1061, 2, 256, 0, 0, 0, 0, 0, 3, 0, 42, 36, 0, 4045, 1, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 0, 0, 0, 2443, 0, 2, 332, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 111, 0, 0, 1878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 56, 2256, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 3290, 2740, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3662, 0, 0, 2, 3764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 1, 13, 1863, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5513, 4182, 3242, 1993, 2322, 6208, 7428, 1880] avg=4096.0000 max=7428 max_rank=6 min=1880 min_rank=7 imbalance_ratio=1.813477 max_load_reduction=25340 max_load_reduction_pct=0.773315 moved_to_echo=17838 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 922, 420, 261, 0, 0, 0, 0, 0, 0, 25, 0, 0, 2, 4096, 4, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 4058, 11, 377, 329, 0, 0, 0, 0, 0, 0, 0, 439, 0, 0, 0, 0, 2750, 1, 1888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3040, 0, 0, 1258, 0, 0, 0, 0, 0, 0, 0, 0, 348, 0, 1, 0, 11, 2367, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 428, 0, 0, 0, 3688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 3, 0, 0, 6, 0, 4095, 1874, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1604, 4171, 4778, 5078, 4298, 2731, 4116, 5992] avg=4096.0000 max=5992 max_rank=7 min=1604 min_rank=0 imbalance_ratio=1.462891 max_load_reduction=26776 max_load_reduction_pct=0.817139 moved_to_echo=17137 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 12800, 512, 128, 0, 512, 4352, 128, 14848, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[640, 3712, 640, 128, 128, 14080, 384, 0, 8448, 3072, 2048, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[2176, 128, 128, 128, 128, 128, 128, 15872, 14720, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[2176, 256, 2432, 128, 6656, 21632, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 9472, 13952, 128, 128, 9344, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 6400, 128, 128, 384, 26112, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 512, 128, 128, 128, 26368, 896, 2816, 2304, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 384, 31744, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 4004, 184, 0, 1590, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 38, 3383, 0, 0, 0, 0, 0, 0, 2742, 0, 0, 1725, 4076, 0, 0, 0, 111, 0, 0, 783, 10, 859, 21, 10, 1558, 0, 0, 0, 194, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 2434, 126, 687] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 4095, 24, 0, 3132, 0, 1297, 495, 0, 3519, 0, 6, 0, 0, 0, 0, 0, 25, 1, 49, 0, 0, 1645, 0, 1, 0, 2, 0, 2, 365, 0, 0, 0, 3299, 0, 0, 0, 386, 1424, 0, 0, 0, 168, 2, 1970, 0, 0, 0, 0, 0, 118, 0, 0, 41, 0, 976, 0, 1, 0, 1, 4081, 4089, 0, 1554] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 4005, 0, 53, 0, 0, 10, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 446, 0, 0, 0, 4096, 0, 47, 0, 0, 0, 86, 53, 0, 0, 70, 3933, 0, 0, 0, 5, 11, 0, 0, 0, 4086, 0, 1442, 0, 0, 0, 106, 0, 217, 0, 0, 0, 0, 95, 0, 0, 23, 0, 3682, 2438, 41, 3727] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 2480, 0, 0, 210, 0, 467, 0, 0, 4071, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 2207, 0, 41, 4096, 0, 1, 0, 177, 0, 1, 121, 0, 3068, 2, 0, 0, 0, 15, 3, 0, 0, 0, 0, 0, 0, 31, 0, 1, 0, 0, 3964, 3897, 0, 3810] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 1398, 0, 2, 0, 0, 407, 0, 0, 4071, 0, 0, 0, 0, 2, 0, 0, 15, 0, 0, 0, 0, 3628, 0, 0, 0, 0, 7, 2, 0, 809, 0, 14, 3924, 1, 0, 6, 0, 0, 1, 0, 0, 3514, 0, 686, 0, 0, 187, 0, 0, 1117, 0, 0, 0, 0, 1428, 0, 0, 21, 0, 4095, 3433, 0, 4000] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 4090, 0, 0, 1007, 2, 1654, 0, 0, 4050, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 2427, 10, 6, 0, 0, 0, 0, 0, 0, 0, 0, 3412, 47, 2, 0, 897, 9, 0, 0, 0, 4093, 2, 868, 11, 24, 2440, 0, 31, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 3999, 1154, 2, 2430] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 4069, 853, 0, 2155, 0, 978, 0, 2, 3972, 0, 0, 0, 0, 0, 0, 0, 33, 11, 0, 0, 1, 1770, 3, 0, 26, 0, 0, 1, 0, 2101, 6, 89, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 3354, 0, 0, 1, 0, 0, 0, 0, 1, 20, 0, 75, 0, 0, 0, 0, 0, 26, 3604, 1470, 0, 4079] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 4089, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 1, 0, 2, 4094, 0, 26, 0, 17, 5, 0, 1, 0, 3780, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4024, 6, 0, 0, 4, 4067, 4072, 8, 377] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 905, 0, 2, 0, 0, 407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 2, 0, 53, 47, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 3628, 0, 209, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2, 0, 809, 0, 3154, 0, 0, 0, 0, 0, 0, 0, 14, 3518, 1, 0, 6, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3467, 0, 686, 0, 0, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1117, 0, 0, 0, 0, 1428, 1664, 941, 284, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 1769, 0, 4000, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1314, 4173, 4205, 3972, 3540, 4340, 5434, 5790] avg=4096.0000 max=5790 max_rank=7 min=1314 min_rank=0 imbalance_ratio=1.413574 max_load_reduction=26978 max_load_reduction_pct=0.823303 moved_to_echo=19104 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 2589, 184, 0, 1590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 19, 55, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 3383, 0, 601, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2742, 0, 0, 3127, 0, 0, 0, 0, 0, 0, 0, 1725, 3655, 0, 0, 0, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 783, 10, 848, 21, 10, 1558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 194, 81, 0, 0, 0, 0, 0, 1179, 932, 814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1255, 126, 687, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4363, 4159, 4388, 5869, 5491, 3230, 3200, 2068] avg=4096.0000 max=5869 max_rank=3 min=2068 min_rank=7 imbalance_ratio=1.432861 max_load_reduction=26899 max_load_reduction_pct=0.820892 moved_to_echo=13989 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 2648, 24, 0, 3132, 0, 1297, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3519, 0, 6, 0, 0, 0, 0, 44, 2, 0, 0, 0, 0, 0, 0, 0, 25, 1, 49, 0, 0, 1645, 0, 614, 296, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 2, 365, 0, 0, 3143, 0, 0, 0, 0, 0, 0, 0, 0, 2959, 0, 0, 0, 386, 1424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 166, 2, 1970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 0, 0, 41, 0, 976, 1982, 938, 833, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 2107, 0, 1554, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7596, 3571, 2630, 3513, 4769, 2138, 4888, 3663] avg=4096.0000 max=7596 max_rank=0 min=2138 min_rank=5 imbalance_ratio=1.854492 max_load_reduction=25172 max_load_reduction_pct=0.768188 moved_to_echo=15458 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 2590, 0, 53, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 53, 54, 0, 0, 0, 0, 0, 0, 0, 0, 446, 0, 0, 0, 4096, 0, 601, 353, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 86, 53, 0, 0, 2836, 0, 0, 0, 0, 0, 0, 0, 70, 3527, 0, 0, 0, 5, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 0, 1442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 0, 217, 0, 0, 0, 0, 95, 1181, 846, 814, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 1257, 41, 3727, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2653, 4203, 5496, 3022, 3613, 5474, 3259, 5048] avg=4096.0000 max=5496 max_rank=2 min=2653 min_rank=0 imbalance_ratio=1.341797 max_load_reduction=27272 max_load_reduction_pct=0.832275 moved_to_echo=17394 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 2644, 0, 0, 1007, 2, 1654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 28, 46, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2427, 10, 614, 307, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 3081, 0, 0, 0, 0, 0, 0, 0, 0, 3059, 47, 2, 0, 897, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 2, 868, 11, 24, 2440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 73, 0, 0, 0, 0, 559, 918, 832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 595, 2, 2430, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5307, 4178, 3358, 3087, 4014, 7384, 2413, 3027] avg=4096.0000 max=7384 max_rank=5 min=2413 min_rank=6 imbalance_ratio=1.802734 max_load_reduction=25384 max_load_reduction_pct=0.774658 moved_to_echo=16838 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1604, 0, 0, 210, 0, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 33, 0, 55, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 372, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2207, 0, 3053, 0, 0, 0, 0, 0, 0, 0, 41, 3673, 0, 1, 0, 177, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 121, 0, 3027, 2, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 31, 1889, 911, 504, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2008, 0, 3810, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2281, 4200, 4812, 5260, 3893, 3165, 3338, 5819] avg=4096.0000 max=5819 max_rank=7 min=2281 min_rank=0 imbalance_ratio=1.420654 max_load_reduction=26949 max_load_reduction_pct=0.822418 moved_to_echo=16215 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 2632, 853, 0, 2155, 0, 978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3972, 0, 0, 0, 0, 0, 0, 55, 45, 0, 0, 0, 0, 0, 0, 0, 33, 11, 0, 0, 1, 1770, 3, 610, 366, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 1, 0, 2101, 6, 2776, 0, 0, 0, 0, 0, 0, 0, 89, 3647, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3309, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 20, 0, 75, 0, 0, 712, 828, 827, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 758, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6618, 4074, 2794, 4910, 3736, 3310, 2463, 4863] avg=4096.0000 max=6618 max_rank=0 min=2463 min_rank=6 imbalance_ratio=1.615723 max_load_reduction=26150 max_load_reduction_pct=0.798035 moved_to_echo=14372 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 2636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 2, 0, 60, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 617, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 3133, 0, 0, 0, 0, 0, 0, 0, 2, 3662, 0, 26, 0, 17, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4024, 1977, 934, 836, 0, 0, 0, 0, 0, 6, 0, 0, 4, 0, 2095, 8, 377, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2636, 4210, 5085, 3135, 3712, 3727, 7773, 2490] avg=4096.0000 max=7773 max_rank=6 min=2490 min_rank=7 imbalance_ratio=1.897705 max_load_reduction=24995 max_load_reduction_pct=0.762787 moved_to_echo=17702 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 18304, 1152, 128, 8192, 128, 4864, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 0, 11904, 256, 20736, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 256, 1536, 128, 0, 128, 0, 6656, 11264, 7296, 5760, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 128, 3200, 5120, 128, 24320, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1024, 128, 22656, 128, 4992, 1664, 128, 2688, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 512, 128, 0, 128, 25216, 128, 4352, 2816, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[2048, 27776, 128, 128, 128, 1664, 1536, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 32000, 0, 128, 0, 0, 128, 128, 512, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 107, 4090, 210, 40, 0, 1, 4094, 0, 0, 3, 4053, 0, 1, 0, 0, 0, 2, 1, 978, 4027, 0, 16, 0, 8, 55, 0, 0, 3501, 0, 4008, 0, 0, 0, 0, 0, 0, 0, 7, 2, 4035, 0, 0, 0, 0, 0, 0, 408, 16, 0, 0, 0, 64, 0, 1, 324, 0, 219, 0, 0, 0, 2497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[291, 5, 4066, 0, 3742, 0, 0, 115, 0, 3414, 0, 0, 0, 0, 8, 0, 11, 3, 0, 200, 0, 3517, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4070, 0, 88, 0, 0, 2, 0, 0, 0, 0, 3927, 0, 5, 0, 0, 0, 0, 700, 4031, 335, 0, 0, 84, 171, 0, 3956, 0, 15, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 1, 322, 0, 4091, 13, 0, 0, 131, 2314, 54, 0, 19, 0, 0, 4, 2630, 0, 0, 34, 9, 4084, 190, 0, 0, 0, 0, 0, 2, 0, 3996, 0, 4078, 0, 1, 0, 0, 7, 0, 9, 0, 0, 3358, 359, 0, 0, 0, 72, 635, 0, 0, 0, 0, 0, 0, 1760, 0, 148, 0, 4024, 0, 0, 0, 423] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 5, 0, 21, 3248, 0, 319, 0, 0, 4011, 0, 2, 1, 3680, 0, 0, 26, 0, 1, 5, 0, 0, 1832, 0, 0, 2, 0, 0, 1, 5, 1278, 2, 3708, 0, 57, 0, 0, 0, 0, 19, 3099, 0, 4059, 3860, 76, 0, 0, 0, 0, 10, 920, 17, 0, 0, 0, 966, 0, 0, 0, 852, 0, 0, 0, 686] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 1, 4094, 2, 4091, 0, 0, 0, 0, 3919, 0, 0, 0, 3755, 0, 1, 0, 0, 0, 0, 27, 3949, 1283, 0, 0, 0, 0, 0, 182, 0, 0, 0, 4083, 3, 1, 0, 1, 0, 6, 0, 0, 0, 4069, 0, 0, 0, 0, 4, 12, 111, 0, 0, 0, 0, 0, 6, 0, 2481, 0, 17, 0, 0, 4, 666] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 3225, 10, 4088, 13, 0, 0, 0, 4074, 0, 0, 0, 4075, 0, 0, 97, 0, 0, 0, 0, 1955, 2519, 0, 0, 0, 1081, 0, 4, 0, 15, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3419, 9, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1401, 3681, 39, 0, 4056, 0, 49, 9, 0, 4016, 0, 0, 0, 1, 0, 1, 0, 1339, 0, 0, 0, 19, 165, 129, 0, 0, 0, 0, 0, 0, 3937, 4, 3223, 0, 718, 0, 0, 0, 0, 0, 0, 0, 4068, 363, 0, 0, 0, 0, 0, 28, 2394, 26, 0, 0, 62, 0, 225, 1229, 0, 146, 0, 2, 0, 1438] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 122, 0, 4080, 0, 0, 0, 0, 4092, 0, 0, 2, 2872, 0, 0, 751, 2785, 0, 0, 0, 3696, 1279, 0, 0, 0, 28, 0, 0, 0, 0, 1, 4094, 0, 0, 1, 0, 0, 0, 0, 2, 0, 4096, 3659, 0, 0, 0, 0, 0, 0, 29, 0, 24, 0, 0, 0, 4, 0, 0, 212, 1, 0, 0, 938] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4094, 2, 1898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1847, 0, 0, 0, 3755, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 2914, 1283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 0, 0, 0, 1669, 755, 0, 0, 0, 0, 0, 0, 4083, 3, 1, 0, 1, 0, 6, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2729, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 12, 111, 0, 0, 0, 0, 0, 6, 2193, 403, 0, 0, 0, 0, 0, 0, 0, 2481, 0, 17, 0, 0, 4, 666, 585, 926, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5995, 5603, 4224, 2606, 4203, 2733, 2725, 4679] avg=4096.0000 max=5995 max_rank=0 min=2606 min_rank=3 imbalance_ratio=1.463623 max_load_reduction=26773 max_load_reduction_pct=0.817047 moved_to_echo=14340 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[291, 5, 4066, 0, 1736, 0, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1609, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 3, 0, 200, 0, 2595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 1454, 729, 0, 0, 0, 0, 0, 0, 4070, 0, 88, 0, 0, 2, 0, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2634, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 700, 4031, 335, 0, 0, 84, 171, 2006, 351, 0, 0, 0, 0, 0, 0, 0, 3956, 0, 15, 0, 0, 0, 0, 564, 825, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6213, 1617, 2809, 2195, 4257, 2639, 7678, 5360] avg=4096.0000 max=7678 max_rank=6 min=1617 min_rank=1 imbalance_ratio=1.874512 max_load_reduction=25090 max_load_reduction_pct=0.765686 moved_to_echo=19934 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 322, 0, 1898, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 1091, 54, 0, 19, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 2630, 0, 0, 34, 9, 3013, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3996, 0, 985, 623, 0, 0, 0, 0, 0, 0, 4078, 0, 1, 0, 0, 7, 0, 9, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2252, 359, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 635, 0, 0, 0, 0, 0, 0, 1760, 2193, 238, 0, 0, 0, 0, 0, 0, 0, 148, 0, 4024, 0, 0, 0, 423, 483, 958, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2234, 1299, 5876, 5606, 4208, 2683, 4826, 6036] avg=4096.0000 max=6036 max_rank=7 min=1299 min_rank=1 imbalance_ratio=1.473633 max_load_reduction=26732 max_load_reduction_pct=0.815796 moved_to_echo=17753 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 107, 1898, 210, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1930, 0, 0, 3, 4053, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 722, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 8, 55, 0, 0, 3501, 0, 1743, 749, 0, 0, 0, 0, 0, 0, 4008, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 7, 2, 2706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 408, 16, 0, 0, 0, 64, 0, 2192, 421, 0, 0, 0, 0, 0, 0, 1, 324, 0, 219, 0, 0, 0, 2497, 580, 229, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2255, 5988, 4752, 6072, 4035, 2715, 3101, 3850] avg=4096.0000 max=6072 max_rank=3 min=2255 min_rank=0 imbalance_ratio=1.482422 max_load_reduction=26696 max_load_reduction_pct=0.814697 moved_to_echo=13701 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 0, 21, 1507, 0, 319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1891, 0, 2, 1, 3680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 1, 5, 0, 0, 1832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 5, 1278, 2, 1708, 753, 0, 0, 0, 0, 0, 0, 3708, 0, 57, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 3099, 0, 2723, 3860, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 920, 17, 0, 0, 0, 966, 1741, 412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 852, 0, 0, 0, 686, 583, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1852, 5574, 1864, 3749, 3784, 9758, 4066, 2121] avg=4096.0000 max=9758 max_rank=5 min=1852 min_rank=0 imbalance_ratio=2.382324 max_load_reduction=23010 max_load_reduction_pct=0.702209 moved_to_echo=19729 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1401, 3681, 39, 0, 1882, 0, 49, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1893, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1339, 0, 0, 0, 15, 165, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3937, 4, 1710, 755, 0, 0, 0, 0, 0, 0, 3223, 0, 718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2728, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 2394, 26, 0, 0, 62, 0, 2174, 413, 0, 0, 0, 0, 0, 0, 225, 1229, 0, 146, 0, 2, 0, 1438, 585, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7061, 1895, 1648, 6406, 3941, 3091, 5097, 3629] avg=4096.0000 max=7061 max_rank=0 min=1648 min_rank=2 imbalance_ratio=1.723877 max_load_reduction=25707 max_load_reduction_pct=0.784515 moved_to_echo=15758 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 122, 0, 1893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1929, 0, 0, 2, 2872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 751, 2785, 0, 0, 0, 2727, 1279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 1, 1742, 760, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 0, 0, 102, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2747, 3659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 24, 0, 0, 0, 2187, 421, 0, 0, 0, 0, 0, 0, 4, 0, 0, 212, 1, 0, 0, 938, 589, 867, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2015, 4803, 7542, 2531, 4197, 6408, 2661, 2611] avg=4096.0000 max=7542 max_rank=2 min=2015 min_rank=0 imbalance_ratio=1.841309 max_load_reduction=25226 max_load_reduction_pct=0.769836 moved_to_echo=15877 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3225, 10, 1895, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1914, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 0, 0, 1437, 2519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1081, 0, 4, 0, 15, 0, 1739, 764, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2193, 421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3419, 9, 0, 0, 0, 593, 461, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5143, 5989, 4053, 3603, 4143, 2741, 2614, 4482] avg=4096.0000 max=5989 max_rank=1 min=2614 min_rank=6 imbalance_ratio=1.462158 max_load_reduction=26779 max_load_reduction_pct=0.817230 moved_to_echo=13980 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 1152, 128, 256, 128, 12800, 128, 12800, 5888, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[768, 1280, 7424, 384, 128, 0, 256, 2944, 16896, 3200, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1792, 3712, 11904, 256, 14720, 256, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[256, 8192, 0, 8960, 128, 128, 128, 6656, 4608, 4352, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[256, 14208, 128, 128, 128, 18560, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3584, 4224, 128, 256, 128, 13440, 11392, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3200, 128, 21376, 8320, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[31360, 128, 896, 128, 128, 128, 128, 128, 512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4057, 0, 0, 0, 3732, 0, 2955, 49, 350, 361, 783, 4083, 0, 0, 1, 0, 2, 0, 0, 0, 736, 0, 6, 0, 2, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 4062, 0, 3358, 24, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4073, 0, 335, 0, 4085, 1041, 1, 0, 3498, 0, 165, 4083, 0, 0, 0, 1, 38, 0, 10, 494, 3, 0, 0, 0, 0, 2374, 0, 0, 170, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4086, 20, 3998, 10, 0, 5, 178, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 38, 0, 4047, 3622, 0, 0, 18, 53, 551, 0, 0, 4042, 3382, 4050, 4096, 0, 30, 0, 0, 0, 0, 0, 1, 1081, 0, 16, 2, 0, 0, 0, 0, 0, 47, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 3951, 0, 3511, 139, 0, 2, 0, 0, 0, 0, 0, 10, 0, 12, 0, 5, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[258, 0, 0, 4080, 3908, 0, 142, 0, 4094, 4069, 0, 0, 3742, 0, 0, 3953, 0, 0, 0, 0, 0, 311, 0, 0, 25, 0, 0, 150, 0, 0, 12, 0, 3, 0, 0, 3670, 0, 0, 39, 0, 0, 2, 0, 0, 4049, 0, 126, 0, 0, 12, 1, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 6, 0, 18] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 4062, 3760, 0, 115, 816, 4040, 0, 3957, 0, 4096, 0, 0, 4024, 0, 0, 0, 0, 0, 0, 8, 28, 0, 236, 0, 139, 0, 0, 0, 0, 0, 0, 1, 0, 13, 1, 0, 0, 0, 0, 0, 0, 3306, 0, 3833, 18, 307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[7, 134, 151, 4046, 3980, 0, 33, 271, 3376, 750, 4, 467, 4052, 0, 0, 3160, 0, 0, 0, 0, 0, 0, 0, 0, 3231, 87, 19, 119, 1, 202, 1, 2, 0, 0, 0, 2117, 0, 25, 7, 0, 0, 0, 46, 0, 3063, 632, 1381, 0, 1317, 0, 0, 0, 0, 20, 33, 31, 1, 0, 0, 0, 2, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 27, 0, 4093, 4055, 0, 0, 0, 4096, 5, 142, 0, 4010, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 4060, 0, 0, 427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3659, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 4, 0, 4083, 4077, 0, 0, 0, 4096, 1, 972, 3, 4014, 4023, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 17, 0, 0, 0, 0, 0, 124, 1, 0, 82, 0, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 3037, 0, 3850, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 171] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3930, 0, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 747, 1041, 1, 0, 0, 0, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 38, 0, 10, 494, 4083, 21, 8, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 2374, 0, 0, 2087, 143, 0, 0, 0, 0, 0, 0, 170, 0, 2, 0, 0, 0, 1, 0, 2877, 918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1081, 20, 3998, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 178, 0, 0, 0, 0, 0, 3477, 461, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4265, 1954, 4655, 4607, 3968, 5109, 4121, 4089] avg=4096.0000 max=5109 max_rank=5 min=1954 min_rank=1 imbalance_ratio=1.247314 max_load_reduction=27659 max_load_reduction_pct=0.844086 moved_to_echo=17287 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 683, 0, 2955, 49, 0, 361, 783, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 4083, 3, 8, 0, 0, 0, 0, 0, 736, 0, 6, 0, 2, 4095, 0, 1, 2075, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2628, 913, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 1074, 0, 3358, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 347, 421, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3914, 4831, 4097, 7058, 3541, 4467, 768, 4092] avg=4096.0000 max=7058 max_rank=3 min=768 min_rank=6 imbalance_ratio=1.723145 max_load_reduction=25710 max_load_reduction_pct=0.784607 moved_to_echo=12868 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[258, 0, 0, 0, 3771, 0, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 749, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 311, 0, 0, 3953, 23, 9, 0, 0, 0, 0, 0, 25, 0, 0, 150, 0, 0, 12, 0, 2069, 137, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3670, 0, 0, 39, 0, 2883, 910, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1070, 0, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 0, 0, 0, 98, 0, 3719, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 18, 4071, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4171, 4818, 4296, 2393, 7505, 1198, 4292, 4095] avg=4096.0000 max=7505 max_rank=4 min=1198 min_rank=5 imbalance_ratio=1.832275 max_load_reduction=25263 max_load_reduction_pct=0.770966 moved_to_echo=17090 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3628, 0, 115, 816, 0, 0, 0, 0, 0, 0, 0, 0, 739, 0, 3957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 28, 4024, 26, 8, 0, 0, 0, 0, 0, 0, 236, 0, 139, 0, 0, 0, 0, 1689, 132, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 13, 1, 0, 0, 2845, 743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 874, 0, 3833, 18, 0, 0, 0, 0, 0, 0, 0, 0, 307, 0, 0, 0, 0, 0, 0, 0, 4070, 456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 4, 4054, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4559, 4696, 4094, 2196, 3603, 4725, 4833, 4062] avg=4096.0000 max=4833 max_rank=6 min=2196 min_rank=3 imbalance_ratio=1.179932 max_load_reduction=27935 max_load_reduction_pct=0.852509 moved_to_echo=17223 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 38, 0, 0, 3495, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 11, 551, 0, 0, 0, 3382, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 1, 4096, 24, 7, 0, 0, 0, 0, 0, 1081, 0, 16, 2, 0, 0, 0, 0, 2018, 127, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 62, 0, 0, 0, 37, 888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 0, 3511, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 10, 4018, 5, 0, 0, 0, 0, 0, 0, 0, 12, 0, 5, 0, 0, 0, 0, 4040, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3551, 7994, 4158, 3244, 1034, 4695, 4035, 4057] avg=4096.0000 max=7994 max_rank=1 min=1034 min_rank=4 imbalance_ratio=1.951660 max_load_reduction=24774 max_load_reduction_pct=0.756042 moved_to_echo=13821 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[7, 134, 151, 0, 3840, 0, 33, 271, 0, 0, 0, 0, 0, 0, 0, 0, 618, 750, 4, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3160, 24, 7, 0, 0, 0, 0, 0, 3231, 87, 19, 119, 1, 202, 1, 2, 1565, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2117, 0, 25, 7, 0, 2377, 688, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 810, 632, 1381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1317, 0, 0, 0, 0, 20, 33, 31, 4028, 381, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 4039, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4436, 1839, 3191, 5367, 5214, 2869, 5810, 4042] avg=4096.0000 max=5810 max_rank=6 min=1839 min_rank=1 imbalance_ratio=1.418457 max_load_reduction=26958 max_load_reduction_pct=0.822693 moved_to_echo=17935 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 27, 0, 0, 3912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 749, 5, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 24, 8, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4060, 0, 0, 1869, 143, 0, 0, 0, 0, 0, 0, 427, 0, 0, 0, 0, 0, 0, 0, 2885, 822, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 968, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3986, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4085, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3939, 896, 4129, 6074, 4134, 5062, 4448, 4086] avg=4096.0000 max=6074 max_rank=3 min=896 min_rank=1 imbalance_ratio=1.482910 max_load_reduction=26694 max_load_reduction_pct=0.814636 moved_to_echo=17730 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 3929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 741, 1, 972, 3, 0, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 17, 4096, 25, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 1, 0, 1556, 148, 0, 0, 0, 0, 0, 0, 82, 0, 0, 110, 0, 0, 0, 0, 2889, 688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 793, 0, 3850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 3989, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 171, 4074, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3933, 5740, 4148, 1829, 3769, 4643, 4461, 4245] avg=4096.0000 max=5740 max_rank=1 min=1829 min_rank=3 imbalance_ratio=1.401367 max_load_reduction=27028 max_load_reduction_pct=0.824829 moved_to_echo=17118 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[5120, 384, 128, 512, 128, 10880, 128, 128, 14976, 1152, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1664, 128, 256, 0, 0, 128, 256, 128, 27648, 3200, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[768, 128, 128, 6016, 128, 128, 128, 0, 19456, 6656, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[5120, 6528, 8064, 640, 0, 7808, 5120, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 7808, 768, 24192, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[0, 128, 128, 128, 128, 384, 128, 640, 31616, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 256, 32640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[384, 256, 256, 0, 30464, 0, 640, 1152, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33792 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 41, 14, 0, 0, 0, 0, 13, 53, 9, 0, 34, 0, 0, 0, 0, 7, 33, 0, 2559, 2289, 0, 1519, 0, 15, 0, 0, 209, 4055, 0, 0, 0, 9, 0, 0, 0, 3, 0, 5, 1, 119, 0, 0, 0, 0, 0, 0, 0, 2333, 1305, 1600, 4084, 0, 3852, 14, 0, 0, 0, 3865, 0, 4055, 0, 0, 673] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 1, 0, 2, 4040, 0, 1, 0, 0, 3, 0, 50, 2, 8, 4067, 227, 879, 0, 4081, 0, 0, 0, 4087, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4086, 0, 3891, 0, 0, 0, 11, 2922, 1, 4096, 0, 0, 298] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 127, 0, 0, 0, 4051, 0, 0, 0, 0, 0, 190, 32, 0, 3569, 24, 0, 4096, 0, 0, 2, 320, 55, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 21, 32, 0, 3, 119, 0, 0, 0, 0, 21, 0, 0, 0, 516, 6, 4054, 0, 3947, 0, 0, 0, 0, 3423, 0, 4093, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 0, 0, 0, 0, 254, 0, 417, 18, 0, 0, 0, 0, 4, 0, 0, 165, 4042, 30, 39, 0, 1738, 0, 2, 0, 4008, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 3, 81, 7, 0, 24, 0, 1398, 0, 411, 4066, 49, 4053, 485, 0, 0, 0, 3304, 0, 4086, 0, 0, 4056] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 27, 3933, 0, 2971, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1853, 0, 3055, 0, 4096, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 8, 0, 124, 0, 42, 0, 0, 0, 805, 112, 0, 0, 4, 0, 0, 0, 136, 1, 2, 4096, 0, 3847, 0, 2, 80, 1, 0, 4090, 25, 259, 2, 0, 2, 0, 0, 1, 0, 0, 55, 3, 1, 0, 0, 0, 10, 0, 0, 0, 3271, 0, 4096, 0, 3718, 0, 0, 0, 0, 3939, 0, 4036, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 1, 13, 0, 0, 3, 0, 114, 0, 697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 457, 0, 2586, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 4, 4002, 0, 1, 0, 0, 4, 0, 0, 4, 0, 0, 0, 0, 636, 0, 4060, 0, 3679, 81, 0, 0, 0, 4077, 0, 4094, 0, 0, 107] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 0, 0, 4091, 0, 453, 0, 2970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 8, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 296, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 453, 4, 4096, 0, 4096, 0, 0, 0, 0, 4003, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 1, 2057, 1128, 463, 0, 0, 0, 0, 0, 0, 2, 4040, 0, 1, 0, 0, 3, 1526, 483, 197, 0, 0, 0, 0, 0, 0, 50, 2, 8, 2010, 227, 879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 2561, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3623, 36, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 2725, 1, 0, 0, 0, 298, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3650, 6252, 3176, 6652, 3659, 4062, 2282, 3035] avg=4096.0000 max=6652 max_rank=3 min=2282 min_rank=6 imbalance_ratio=1.624023 max_load_reduction=26116 max_load_reduction_pct=0.796997 moved_to_echo=13038 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 41, 14, 0, 0, 0, 0, 13, 1158, 1116, 463, 0, 0, 0, 0, 0, 53, 9, 0, 34, 0, 0, 0, 0, 1514, 478, 261, 0, 0, 0, 0, 0, 7, 33, 0, 2559, 1131, 0, 1519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 209, 2541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 3, 0, 5, 1, 3621, 36, 0, 0, 0, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 4019, 0, 0, 0, 0, 0, 0, 0, 2333, 1305, 1600, 0, 0, 2258, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3604, 0, 0, 0, 0, 673, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2805, 2349, 5249, 2765, 3675, 4138, 7510, 4277] avg=4096.0000 max=7510 max_rank=6 min=2349 min_rank=1 imbalance_ratio=1.833496 max_load_reduction=25258 max_load_reduction_pct=0.770813 moved_to_echo=19600 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 127, 0, 0, 0, 2072, 1144, 460, 0, 0, 0, 0, 0, 4051, 0, 0, 0, 0, 0, 190, 32, 1518, 490, 231, 0, 0, 0, 0, 0, 0, 3569, 24, 0, 2024, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 320, 55, 0, 0, 2547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 32, 0, 3, 3594, 36, 0, 0, 0, 0, 0, 0, 119, 0, 0, 0, 0, 21, 0, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 516, 6, 0, 0, 2313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3805, 6512, 5619, 2922, 3686, 4197, 2835, 3192] avg=4096.0000 max=6512 max_rank=1 min=2835 min_rank=6 imbalance_ratio=1.589844 max_load_reduction=26256 max_load_reduction_pct=0.801270 moved_to_echo=13910 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 13, 0, 0, 3, 0, 2063, 1066, 459, 0, 0, 0, 0, 0, 114, 0, 697, 0, 0, 0, 0, 0, 1520, 457, 276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2015, 0, 457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2586, 0, 0, 0, 2550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4002, 0, 1, 0, 3601, 35, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 636, 0, 0, 0, 2156, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3801, 0, 0, 0, 0, 107, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3605, 3064, 2472, 5136, 7643, 4067, 2873, 3908] avg=4096.0000 max=7643 max_rank=4 min=2472 min_rank=2 imbalance_ratio=1.865967 max_load_reduction=25125 max_load_reduction_pct=0.766754 moved_to_echo=18491 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 0, 0, 0, 0, 2045, 1175, 460, 0, 0, 0, 0, 0, 254, 0, 417, 18, 0, 0, 0, 0, 1497, 503, 223, 0, 0, 0, 0, 0, 4, 0, 0, 165, 1997, 30, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1738, 0, 2, 0, 2511, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 3606, 35, 0, 0, 0, 0, 0, 0, 0, 0, 3, 81, 7, 0, 24, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 1398, 0, 411, 0, 49, 2375, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3081, 0, 0, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3682, 2912, 2235, 4252, 3666, 4166, 4718, 7137] avg=4096.0000 max=7137 max_rank=7 min=2235 min_rank=2 imbalance_ratio=1.742432 max_load_reduction=25631 max_load_reduction_pct=0.782196 moved_to_echo=19687 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 0, 124, 0, 42, 0, 0, 2072, 1078, 463, 0, 0, 0, 0, 0, 0, 805, 112, 0, 0, 4, 0, 0, 1527, 462, 266, 0, 0, 0, 0, 0, 0, 136, 1, 2, 2024, 0, 3847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 80, 1, 0, 2563, 25, 259, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 55, 3633, 34, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 10, 0, 0, 4002, 0, 0, 0, 0, 0, 0, 0, 0, 3271, 0, 0, 0, 2178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3787, 3176, 6010, 2932, 3725, 4016, 5449, 3673] avg=4096.0000 max=6010 max_rank=2 min=2932 min_rank=3 imbalance_ratio=1.467285 max_load_reduction=26758 max_load_reduction_pct=0.816589 moved_to_echo=16863 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 2072, 1187, 347, 0, 0, 0, 0, 0, 84, 0, 359, 0, 0, 0, 0, 0, 1529, 509, 277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2024, 27, 3933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2971, 0, 0, 0, 2567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 2708, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 1853, 0, 0, 0, 2400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3606, 2758, 5984, 5538, 2749, 4062, 4253, 3818] avg=4096.0000 max=5984 max_rank=2 min=2749 min_rank=4 imbalance_ratio=1.460938 max_load_reduction=26784 max_load_reduction_pct=0.817383 moved_to_echo=14882 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 0, 0, 4091, 0, 2076, 1191, 464, 0, 0, 0, 0, 0, 453, 0, 2970, 0, 0, 0, 0, 0, 1533, 514, 275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2020, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 2563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 296, 1, 0, 0, 3632, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 453, 4, 0, 0, 2391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7828, 5745, 2023, 2571, 3965, 4060, 2848, 3728] avg=4096.0000 max=7828 max_rank=0 min=2023 min_rank=2 imbalance_ratio=1.911133 max_load_reduction=24940 max_load_reduction_pct=0.761108 moved_to_echo=14601 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 256, 128, 128, 4096, 128, 15616, 9088, 3584, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[5120, 896, 8704, 128, 128, 128, 256, 128, 12288, 3968, 2048, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 3840, 128, 2816, 15360, 384, 10752, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[11776, 256, 128, 256, 20480, 128, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3840, 8064, 2048, 0, 128, 18432, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 27648, 128, 0, 0, 0, 5248, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 4352, 128, 128, 128, 28032, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[256, 128, 128, 128, 128, 128, 128, 0, 32384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33792 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 72, 3953, 0, 3637, 1, 4032, 0, 2742, 0, 1, 3, 0, 0, 0, 0, 0, 32, 0, 100, 0, 0, 0, 0, 0, 0, 4005, 15, 158, 1, 676, 0, 0, 0, 0, 3842, 0, 0, 7, 0, 0, 0, 117, 240, 5, 4078, 0, 0, 3, 0, 0, 0, 5, 945, 0, 0, 0, 0, 0, 0, 3, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 3746, 0, 0, 4096, 4, 153, 0, 840, 0, 1175, 0, 3560, 0, 0, 0, 0, 0, 0, 1848, 8, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 89, 4080, 1698, 0, 0, 812, 15, 0, 0, 0, 0, 0, 0, 0, 771, 0, 1675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 291, 4058, 0, 4071, 0, 3723, 2, 0, 0, 106, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 3, 2, 0, 0, 4072, 34, 0, 3562, 2663, 45, 0, 0, 103, 74, 9, 0, 0, 29, 0, 1182, 0, 3803, 19, 605, 0, 0, 0, 0, 0, 53, 182, 0, 0, 0, 16, 40, 0, 0, 0, 4017] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 22, 0, 0, 4015, 0, 1879, 0, 3886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3164, 0, 34, 0, 0, 48, 0, 29, 0, 4093, 0, 49, 3052, 618, 617, 0, 284, 0, 0, 0, 0, 0, 0, 5, 111, 0, 0, 126, 3926, 0, 0, 0, 0, 0, 2705, 0, 2, 0, 1, 0, 0, 0, 42, 0, 4060] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 4059, 0, 1613, 0, 4094, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 3919, 4072, 0, 0, 0, 8, 4096, 0, 0, 0, 0, 0, 17, 0, 1765, 0, 1115, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 0, 3694] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 2711, 0, 12, 2130, 0, 0, 0, 3574, 0, 208, 0, 3977, 8, 3, 1, 0, 0, 0, 5, 0, 0, 0, 0, 0, 37, 1271, 0, 4096, 0, 0, 4064, 4096, 3, 0, 213, 0, 0, 0, 0, 0, 16, 209, 0, 0, 4051, 0, 1858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 12, 0, 0, 3, 207] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 4, 0, 6, 3790, 0, 0, 0, 4092, 0, 184, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 37, 0, 11, 1, 0, 37, 6, 4073, 0, 0, 3950, 4095, 1641, 0, 4, 0, 4060, 0, 0, 0, 4, 0, 0, 0, 543, 0, 2257, 0, 0, 0, 0, 0, 1, 1, 0, 1, 6, 0, 0, 0, 172, 0, 3750] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 1484, 0, 0, 2380, 3, 2752, 52, 3983, 0, 2742, 0, 977, 0, 0, 0, 0, 4, 0, 0, 9, 23, 0, 0, 1207, 4, 0, 68, 3948, 0, 0, 376, 22, 0, 4, 0, 0, 805, 27, 9, 0, 823, 0, 0, 0, 0, 4047, 2890, 4, 0, 0, 0, 2, 0, 22, 0, 0, 0, 0, 0, 0, 6, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 6, 1365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2462, 0, 184, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 11, 1630, 1475, 1037, 0, 0, 0, 0, 0, 1, 0, 37, 6, 1352, 0, 0, 3950, 0, 0, 0, 0, 0, 0, 0, 0, 2903, 1641, 0, 4, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 543, 0, 1825, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 2721, 950, 0, 0, 0, 0, 0, 0, 1, 6, 0, 0, 0, 172, 0, 3750, 432, 155, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1375, 2688, 4190, 5346, 8608, 2372, 3673, 4516] avg=4096.0000 max=8608 max_rank=4 min=1375 min_rank=0 imbalance_ratio=2.101562 max_load_reduction=24160 max_load_reduction_pct=0.737305 moved_to_echo=19169 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 72, 1424, 0, 3637, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2426, 0, 2742, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 100, 0, 0, 1606, 1538, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1330, 15, 158, 1, 0, 0, 0, 0, 0, 0, 0, 0, 480, 0, 0, 0, 0, 3842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 117, 240, 5, 3297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 5, 945, 2675, 991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4095, 781, 25, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5134, 5172, 3447, 1504, 4322, 3666, 4619, 4904] avg=4096.0000 max=5172 max_rank=1 min=1504 min_rank=3 imbalance_ratio=1.262695 max_load_reduction=27596 max_load_reduction_pct=0.842163 moved_to_echo=17511 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 22, 0, 0, 1446, 0, 1879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3164, 0, 34, 0, 0, 1548, 1563, 156, 0, 0, 0, 0, 0, 48, 0, 29, 0, 1359, 0, 49, 3052, 0, 0, 0, 0, 0, 0, 0, 0, 439, 617, 0, 284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 111, 0, 0, 126, 3174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2705, 0, 2, 2734, 1006, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 42, 0, 4060, 752, 23, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3347, 2338, 6465, 4537, 1340, 3416, 6447, 4878] avg=4096.0000 max=6465 max_rank=2 min=1340 min_rank=4 imbalance_ratio=1.578369 max_load_reduction=26303 max_load_reduction_pct=0.802704 moved_to_echo=16081 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 291, 1462, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2240, 2, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 1483, 1579, 674, 0, 0, 0, 0, 0, 3, 2, 0, 0, 1352, 34, 0, 3562, 0, 0, 0, 0, 0, 0, 0, 0, 1888, 45, 0, 0, 103, 74, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 1182, 0, 3803, 19, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 182, 0, 2720, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 16, 40, 0, 0, 0, 4017, 115, 101, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5824, 2348, 3740, 4953, 2119, 5523, 3972, 4289] avg=4096.0000 max=5824 max_rank=0 min=2119 min_rank=4 imbalance_ratio=1.421875 max_load_reduction=26944 max_load_reduction_pct=0.822266 moved_to_echo=15903 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 2711, 0, 12, 767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2151, 0, 208, 0, 3977, 8, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 1423, 829, 1038, 0, 0, 0, 0, 0, 0, 37, 1271, 0, 1360, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 2903, 3, 0, 213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 209, 0, 0, 4051, 0, 1503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2736, 534, 0, 0, 0, 0, 0, 0, 0, 2, 1, 12, 0, 0, 3, 207, 355, 155, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3490, 6348, 3295, 6732, 3119, 5779, 3270, 735] avg=4096.0000 max=6732 max_rank=3 min=735 min_rank=7 imbalance_ratio=1.643555 max_load_reduction=26036 max_load_reduction_pct=0.794556 moved_to_echo=12903 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 1462, 0, 1613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2463, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 0, 1631, 1580, 1032, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1360, 0, 0, 3919, 0, 0, 0, 0, 0, 0, 0, 0, 2886, 0, 0, 0, 8, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 1765, 0, 902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 2736, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 0, 3694, 213, 154, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3079, 2521, 4248, 5280, 6990, 2684, 3756, 4210] avg=4096.0000 max=6990 max_rank=4 min=2521 min_rank=1 imbalance_ratio=1.706543 max_load_reduction=25778 max_load_reduction_pct=0.786682 moved_to_echo=17640 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1484, 0, 0, 858, 3, 2752, 52, 0, 0, 0, 0, 0, 0, 0, 0, 2397, 0, 2742, 0, 977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 9, 23, 0, 0, 1586, 926, 5, 0, 0, 0, 0, 0, 1207, 4, 0, 68, 1311, 0, 0, 376, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 4, 0, 0, 805, 27, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 823, 0, 0, 0, 0, 4047, 2337, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 2, 0, 22, 0, 2637, 596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4095, 553, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5149, 6116, 2553, 2966, 862, 7207, 3261, 4654] avg=4096.0000 max=7207 max_rank=5 min=862 min_rank=4 imbalance_ratio=1.759521 max_load_reduction=25561 max_load_reduction_pct=0.780060 moved_to_echo=15984 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 3746, 0, 0, 1467, 4, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 0, 1175, 0, 3560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1848, 8, 0, 0, 0, 338, 1598, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1357, 0, 4, 89, 0, 0, 0, 0, 0, 0, 0, 0, 2883, 1698, 0, 0, 812, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 771, 0, 1350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2739, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 325, 159, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5370, 5237, 4830, 1450, 5408, 2121, 3770, 4582] avg=4096.0000 max=5408 max_rank=4 min=1450 min_rank=3 imbalance_ratio=1.320312 max_load_reduction=27360 max_load_reduction_pct=0.834961 moved_to_echo=15881 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 128, 5120, 128, 256, 0, 128, 11264, 11136, 5248, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 128, 2816, 256, 1024, 21760, 7168, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 8064, 0, 384, 10368, 128, 14208, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[14464, 4096, 128, 512, 1024, 12928, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1280, 128, 1408, 128, 10880, 128, 256, 19072, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 896, 256, 1408, 128, 11264, 4224, 14976, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[17024, 128, 7168, 0, 8704, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 0, 384, 128, 28032, 3584, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[10, 0, 4090, 54, 1, 0, 1013, 0, 0, 1, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 3960, 4, 36, 48, 0, 8, 4069, 6, 162, 3, 0, 0, 0, 0, 3, 0, 0, 0, 511, 0, 4060, 0, 3175, 0, 3374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[2976, 0, 3902, 0, 504, 0, 3773, 280, 4, 0, 5, 1, 5, 0, 0, 0, 0, 4057, 0, 3, 7, 0, 0, 4095, 8, 306, 177, 0, 1, 3090, 2, 71, 0, 1367, 24, 0, 0, 0, 0, 0, 17, 2643, 0, 4080, 0, 12, 0, 653, 0, 3, 0, 0, 0, 0, 0, 0, 162, 289, 0, 0, 79, 1, 171, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[2486, 0, 3855, 0, 0, 162, 3891, 0, 0, 0, 0, 3022, 6, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4053, 13, 0, 262, 1, 0, 4069, 0, 2711, 0, 0, 0, 0, 0, 0, 0, 0, 138, 8, 63, 3940, 0, 0, 350, 50, 0, 14, 0, 0, 1, 0, 203, 22, 4, 3322, 0, 0, 0, 0, 114, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[3522, 0, 3211, 1, 0, 0, 4096, 0, 0, 0, 0, 3808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 509, 0, 0, 24, 3414, 0, 0, 0, 0, 0, 0, 0, 0, 285, 5, 0, 2, 0, 4096, 0, 1662, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 4028, 2, 2, 28, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[33, 0, 52, 0, 0, 0, 4093, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4016, 0, 3966, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 3, 330, 0, 0, 0, 0, 0, 0, 0, 0, 3948, 3968, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[2530, 0, 4096, 0, 193, 405, 4025, 0, 0, 0, 10, 9, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4083, 0, 0, 2, 0, 0, 899, 0, 4096, 0, 0, 0, 61, 0, 0, 20, 0, 3, 71, 0, 4012, 0, 2334, 0, 0, 0, 1, 0, 0, 0, 10, 0, 0, 3340, 832, 0, 204, 0, 34, 1496, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[718, 0, 4075, 0, 0, 301, 3412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 3963, 0, 0, 0, 0, 0, 4045, 0, 0, 4, 1, 0, 3182, 0, 0, 0, 0, 372, 121, 89, 0, 0, 0, 0, 0, 0, 894, 0, 3968, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 3539, 4001, 0, 0, 0, 0, 4, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[4093, 0, 4048, 0, 0, 0, 2698, 0, 0, 0, 0, 3925, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 61, 0, 0, 54, 0, 53, 0, 731, 0, 0, 0, 0, 0, 0, 18, 0, 0, 4096, 38, 659, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3522, 0, 0, 1, 0, 0, 2200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3808, 0, 0, 0, 0, 974, 565, 382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 0, 0, 24, 3032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 285, 5, 1896, 1604, 921, 0, 0, 0, 0, 0, 0, 2, 0, 1776, 0, 1662, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 3211, 716, 0, 0, 0, 0, 0, 0, 0, 2542, 2, 2, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5723, 5729, 3091, 3565, 4711, 3440, 3935, 2574] avg=4096.0000 max=5729 max_rank=1 min=2574 min_rank=7 imbalance_ratio=1.398682 max_load_reduction=27039 max_load_reduction_pct=0.825165 moved_to_echo=14660 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 0, 54, 1, 0, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4078, 0, 0, 0, 0, 949, 575, 456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 3011, 0, 0, 0, 0, 0, 0, 0, 0, 4, 36, 48, 0, 8, 3613, 6, 162, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3, 0, 0, 469, 1590, 936, 0, 0, 0, 0, 0, 0, 511, 0, 1760, 0, 3175, 0, 3374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 710, 0, 0, 0, 0, 0, 0, 0, 2585, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[609, 6059, 3015, 3877, 3001, 8820, 4800, 2587] avg=4096.0000 max=8820 max_rank=5 min=609 min_rank=0 imbalance_ratio=2.153320 max_load_reduction=23948 max_load_reduction_pct=0.730835 moved_to_echo=19208 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2976, 0, 0, 0, 504, 0, 2027, 280, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 5, 1, 5, 0, 0, 0, 981, 40, 346, 0, 0, 0, 0, 0, 0, 4057, 0, 3, 7, 0, 0, 3114, 0, 0, 0, 0, 0, 0, 0, 0, 8, 306, 177, 0, 1, 2744, 2, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1367, 24, 0, 0, 0, 0, 0, 1746, 1598, 66, 0, 0, 0, 0, 0, 17, 2643, 0, 1769, 0, 12, 0, 653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 3902, 713, 0, 0, 0, 0, 0, 0, 162, 183, 0, 0, 79, 1, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5787, 1382, 7181, 3309, 4801, 5094, 4618, 596] avg=4096.0000 max=7181 max_rank=2 min=596 min_rank=7 imbalance_ratio=1.753174 max_load_reduction=25587 max_load_reduction_pct=0.780853 moved_to_echo=15109 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2530, 0, 0, 0, 193, 405, 2162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 9, 0, 0, 0, 0, 979, 116, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 799, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 20, 0, 1863, 1571, 190, 0, 0, 0, 0, 0, 3, 71, 0, 1740, 0, 2334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 10, 0, 0, 4096, 701, 0, 0, 0, 0, 0, 0, 3340, 526, 0, 204, 0, 34, 1496, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5290, 1214, 3106, 4897, 3705, 4148, 4808, 5600] avg=4096.0000 max=5600 max_rank=7 min=1214 min_rank=1 imbalance_ratio=1.367188 max_load_reduction=27168 max_load_reduction_pct=0.829102 moved_to_echo=18261 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[33, 0, 0, 0, 0, 0, 2198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 982, 557, 450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3566, 0, 3966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 1895, 1604, 907, 0, 0, 0, 0, 0, 0, 0, 0, 1776, 0, 2, 3, 330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 716, 0, 0, 0, 0, 0, 0, 3948, 2504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2231, 6071, 3114, 7532, 4489, 2111, 768, 6452] avg=4096.0000 max=7532 max_rank=3 min=768 min_rank=6 imbalance_ratio=1.838867 max_load_reduction=25236 max_load_reduction_pct=0.770142 moved_to_echo=13820 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[718, 0, 0, 0, 0, 301, 1833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 969, 562, 356, 0, 0, 0, 0, 0, 25, 3963, 0, 0, 0, 0, 0, 3076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 2826, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 372, 121, 89, 0, 0, 0, 1579, 350, 915, 0, 0, 0, 0, 0, 0, 0, 0, 388, 0, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 4075, 156, 0, 0, 0, 0, 0, 0, 3539, 2524, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2852, 1887, 7064, 2831, 3426, 4356, 4285, 6067] avg=4096.0000 max=7064 max_rank=2 min=1887 min_rank=1 imbalance_ratio=1.724609 max_load_reduction=25704 max_load_reduction_pct=0.784424 moved_to_echo=18134 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2486, 0, 0, 0, 0, 162, 2090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3022, 6, 0, 0, 0, 971, 466, 456, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 3082, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 262, 1, 0, 3613, 0, 2711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1801, 1543, 759, 0, 0, 0, 0, 0, 138, 8, 63, 1708, 0, 0, 350, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 1, 0, 203, 22, 3855, 689, 0, 0, 0, 0, 0, 0, 4, 2097, 0, 0, 0, 0, 114, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4738, 4921, 3087, 6600, 4103, 2317, 4784, 2218] avg=4096.0000 max=6600 max_rank=3 min=2218 min_rank=7 imbalance_ratio=1.611328 max_load_reduction=26168 max_load_reduction_pct=0.798584 moved_to_echo=13422 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 0, 0, 0, 0, 1445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3925, 0, 0, 4, 0, 986, 579, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 43, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 731, 0, 0, 0, 0, 0, 0, 1253, 1609, 939, 0, 0, 0, 0, 0, 18, 0, 0, 1767, 38, 659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4048, 720, 0, 0, 0, 0, 0, 0, 4096, 2576, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5538, 5505, 3110, 157, 4532, 2482, 4770, 6674] avg=4096.0000 max=6674 max_rank=7 min=157 min_rank=3 imbalance_ratio=1.629395 max_load_reduction=26094 max_load_reduction_pct=0.796326 moved_to_echo=18458 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[128, 2304, 512, 256, 128, 128, 384, 128, 12544, 11520, 5760, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[16384, 0, 0, 128, 768, 896, 14592, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[15104, 15616, 128, 256, 128, 128, 1792, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 18944, 128, 0, 128, 0, 7808, 3584, 2560, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 896, 640, 128, 128, 20352, 128, 11136, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[256, 3328, 128, 12800, 128, 11904, 384, 4480, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 8064, 128, 128, 128, 128, 128, 24704, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 0, 128, 128, 128, 256, 128, 27392, 5248, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33792 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 26, 2524, 0, 0, 0, 0, 0, 0, 0, 122, 0, 1, 0, 83, 0, 4081, 0, 0, 0, 3452, 41, 0, 0, 0, 0, 0, 0, 0, 2, 3867, 180, 0, 4085, 0, 18, 0, 0, 4096, 0, 0, 0, 0, 9, 0, 0, 3967, 4096, 0, 2, 0, 0, 0, 146, 0, 1966, 0, 0, 0, 0, 0, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 4088, 0, 0, 0, 0, 0, 1088, 0, 3674, 0, 0, 0, 36, 0, 3499, 0, 0, 0, 3922, 0, 0, 0, 0, 7, 1, 0, 1, 0, 4096, 0, 1, 4035, 0, 0, 1, 0, 9, 0, 0, 611, 0, 234, 0, 0, 1938, 2417, 0, 0, 0, 0, 0, 2526, 0, 498, 0, 0, 0, 82, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 23, 0, 10, 35, 0, 0, 293, 0, 3825, 0, 0, 0, 2376, 0, 3993, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 17, 0, 1, 3690, 0, 0, 3958, 0, 0, 0, 2, 0, 0, 0, 2712, 43, 0, 0, 134, 4096, 3777, 3, 0, 0, 0, 0, 3752, 13, 3, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4043, 0, 0, 0, 0, 0, 0, 0, 2888, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2032, 0, 4096, 0, 0, 0, 0, 3101, 0, 0, 0, 0, 0, 0, 9, 1998, 1649, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 270, 0, 1, 13, 25, 0, 43, 0, 0, 126, 0, 3228, 0, 2216, 0, 0, 15, 0, 1, 0, 101, 5, 0, 4, 840, 0, 0, 3733, 0, 5, 4083, 0, 0, 0, 0, 45, 0, 0, 1998, 0, 10, 0, 0, 3971, 573, 0, 0, 0, 0, 3, 4090, 0, 0, 0, 0, 0, 0, 344, 2929] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 386, 0, 122, 0, 0, 0, 4095, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 4096, 0, 0, 0, 0, 3271, 158, 0, 0, 0, 0, 0, 44, 3257, 3886, 0, 0, 0, 0, 0, 472, 0, 741, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3968, 0, 2, 0, 0, 0, 1736, 0, 0, 1, 18, 0, 4085, 0, 3915, 0, 0, 0, 870, 0, 6, 0, 0, 0, 0, 83, 34, 0, 4088, 763, 0, 4092, 0, 0, 1, 7, 3468, 0, 0, 0, 0, 0, 0, 8, 107, 1464, 0, 0, 46, 1, 0, 3986, 0, 0, 0, 0, 0, 8, 0, 11] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 3180, 0, 0, 31, 0, 0, 54, 0, 258, 33, 0, 0, 4057, 7, 2388, 0, 2, 0, 0, 0, 0, 78, 1, 0, 0, 0, 0, 23, 3483, 0, 0, 3904, 1, 0, 0, 0, 3918, 0, 0, 751, 4, 282, 0, 24, 4096, 4049, 191, 0, 0, 0, 0, 1656, 65, 0, 0, 130, 101, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 4088, 0, 0, 0, 558, 408, 0, 0, 0, 0, 0, 0, 0, 0, 1088, 0, 3674, 0, 0, 0, 1380, 171, 0, 0, 0, 0, 0, 0, 36, 0, 220, 0, 0, 0, 3922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1, 0, 1, 0, 3936, 0, 0, 0, 0, 0, 0, 0, 3375, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 611, 0, 234, 0, 0, 721, 258, 0, 0, 0, 0, 0, 0, 0, 1751, 0, 0, 0, 0, 0, 2526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 0, 0, 0, 82, 0, 1, 3108, 99, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5057, 6313, 4178, 3945, 3377, 1833, 4277, 3788] avg=4096.0000 max=6313 max_rank=1 min=1833 min_rank=5 imbalance_ratio=1.541260 max_load_reduction=26455 max_load_reduction_pct=0.807343 moved_to_echo=13275 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 26, 2524, 0, 0, 0, 1143, 692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 0, 1, 0, 2824, 200, 0, 0, 0, 0, 0, 0, 83, 0, 256, 0, 0, 0, 3452, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3986, 0, 0, 0, 0, 0, 0, 0, 3186, 180, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 9, 0, 0, 681, 437, 0, 0, 0, 0, 0, 0, 0, 2967, 0, 2, 0, 0, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1966, 0, 0, 0, 0, 0, 4, 3625, 99, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4385, 3147, 3832, 3988, 3384, 5223, 3115, 5694] avg=4096.0000 max=5694 max_rank=7 min=3115 min_rank=6 imbalance_ratio=1.390137 max_load_reduction=27074 max_load_reduction_pct=0.826233 moved_to_echo=17416 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 23, 0, 10, 35, 1182, 638, 0, 0, 0, 0, 0, 0, 0, 0, 293, 0, 3825, 0, 0, 0, 2914, 196, 0, 0, 0, 0, 0, 0, 2376, 0, 250, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 17, 0, 1, 3862, 0, 0, 0, 0, 0, 0, 0, 3041, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2712, 43, 0, 0, 134, 649, 403, 0, 0, 0, 0, 0, 0, 0, 2736, 3, 0, 0, 0, 0, 3752, 0, 0, 0, 0, 0, 0, 0, 0, 13, 3, 0, 0, 0, 0, 0, 0, 3547, 96, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1894, 7228, 2629, 3883, 3043, 3941, 6491, 3659] avg=4096.0000 max=7228 max_rank=1 min=1894 min_rank=0 imbalance_ratio=1.764648 max_load_reduction=25540 max_load_reduction_pct=0.779419 moved_to_echo=17134 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 270, 0, 1, 1145, 96, 0, 0, 0, 0, 0, 0, 13, 25, 0, 43, 0, 0, 126, 0, 2826, 108, 0, 0, 0, 0, 0, 0, 3228, 0, 140, 0, 0, 15, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 5, 0, 4, 840, 0, 0, 3983, 0, 0, 0, 0, 0, 0, 0, 3076, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 1998, 0, 10, 0, 0, 657, 61, 0, 0, 0, 0, 0, 0, 0, 416, 0, 0, 0, 0, 3, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 344, 2929, 1968, 100, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5608, 3141, 3384, 4933, 3081, 2771, 4509, 5341] avg=4096.0000 max=5608 max_rank=0 min=2771 min_rank=5 imbalance_ratio=1.369141 max_load_reduction=27160 max_load_reduction_pct=0.828857 moved_to_echo=15702 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4043, 0, 0, 0, 576, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2888, 0, 0, 0, 1422, 201, 0, 0, 0, 0, 0, 0, 4095, 0, 257, 0, 0, 0, 635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3996, 0, 0, 0, 0, 0, 0, 0, 3375, 2032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3101, 0, 0, 0, 0, 0, 0, 9, 721, 176, 0, 0, 0, 0, 0, 0, 0, 1195, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3638, 100, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4897, 4511, 4987, 3996, 5407, 4007, 1225, 3738] avg=4096.0000 max=5407 max_rank=4 min=1225 min_rank=6 imbalance_ratio=1.320068 max_load_reduction=27361 max_load_reduction_pct=0.834991 moved_to_echo=14377 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3968, 0, 2, 0, 31, 247, 0, 0, 0, 0, 0, 0, 0, 0, 1736, 0, 0, 1, 18, 0, 76, 192, 0, 0, 0, 0, 0, 0, 4085, 0, 246, 0, 0, 0, 870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 83, 34, 0, 3993, 0, 0, 0, 0, 0, 0, 0, 3369, 763, 0, 0, 0, 0, 1, 7, 0, 0, 0, 0, 0, 0, 0, 0, 3468, 0, 0, 0, 0, 0, 0, 8, 719, 156, 0, 0, 0, 0, 0, 0, 0, 1061, 0, 0, 46, 1, 0, 3986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 11, 3477, 99, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4248, 2023, 5201, 4116, 4140, 4351, 5094, 3595] avg=4096.0000 max=5201 max_rank=2 min=2023 min_rank=1 imbalance_ratio=1.269775 max_load_reduction=27567 max_load_reduction_pct=0.841278 moved_to_echo=17180 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 3180, 0, 0, 31, 1181, 684, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 258, 33, 0, 0, 2915, 117, 0, 0, 0, 0, 0, 0, 4057, 7, 150, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 1, 0, 0, 0, 0, 23, 3808, 0, 0, 0, 0, 0, 0, 0, 2870, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3918, 0, 0, 751, 4, 282, 0, 24, 613, 432, 0, 0, 0, 0, 0, 0, 0, 2933, 191, 0, 0, 0, 0, 1656, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 130, 101, 0, 0, 0, 2121, 96, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5077, 3377, 4216, 3910, 2871, 6024, 4780, 2513] avg=4096.0000 max=6024 max_rank=5 min=2513 min_rank=7 imbalance_ratio=1.470703 max_load_reduction=26744 max_load_reduction_pct=0.816162 moved_to_echo=16188 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 940, 661, 0, 0, 0, 0, 0, 0, 0, 0, 386, 0, 122, 0, 0, 0, 2317, 203, 0, 0, 0, 0, 0, 0, 4095, 0, 246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3997, 0, 0, 0, 0, 0, 0, 0, 3371, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3271, 158, 0, 0, 0, 0, 0, 44, 725, 420, 0, 0, 0, 0, 0, 0, 0, 2805, 0, 0, 0, 0, 0, 472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 741, 0, 0, 0, 0, 0, 0, 3600, 99, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1602, 3028, 4341, 3997, 7465, 4618, 3277, 4440] avg=4096.0000 max=7465 max_rank=4 min=1602 min_rank=0 imbalance_ratio=1.822510 max_load_reduction=25303 max_load_reduction_pct=0.772186 moved_to_echo=19800 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[17920, 256, 0, 6144, 128, 640, 0, 256, 5504, 2432, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 3584, 128, 11008, 128, 256, 0, 16768, 1408, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 22016, 384, 128, 128, 6784, 3712, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 15872, 256, 128, 128, 128, 128, 16768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[25728, 7168, 128, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[22144, 128, 1792, 0, 128, 128, 8960, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 3328, 0, 256, 128, 128, 384, 3072, 25088, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 256, 128, 128, 128, 1024, 128, 128, 31616, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4072, 4092, 0, 188, 6, 3, 0, 0, 0, 1336, 0, 0, 2, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 800, 2392, 0, 0, 43, 0, 0, 3788, 0, 0, 1, 4070, 2, 0, 0, 0, 4072, 0, 3743, 0, 0, 0, 3909, 0, 0, 0, 0, 0, 79, 0, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[39, 0, 0, 0, 4034, 4096, 0, 306, 0, 0, 95, 0, 0, 1099, 0, 0, 0, 4096, 0, 0, 0, 0, 393, 1578, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4010, 6, 163, 0, 10, 0, 4064, 0, 0, 0, 3918, 0, 0, 0, 0, 58, 0, 427, 0, 0, 22, 3732, 0, 0, 0, 0, 5, 599, 0, 16] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 3929, 3906, 0, 0, 0, 9, 0, 0, 2, 44, 0, 0, 13, 4073, 0, 0, 0, 0, 9, 33, 0, 26, 0, 34, 128, 0, 10, 0, 0, 171, 406, 0, 45, 231, 3696, 2, 0, 0, 54, 0, 4032, 5, 0, 0, 0, 4083, 40, 4047, 3636, 0, 0, 2, 14, 0, 7, 0, 68, 12, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 1, 0, 33, 4026, 3861, 0, 3, 0, 0, 58, 0, 0, 0, 0, 0, 0, 1201, 0, 0, 5, 29, 0, 3987, 0, 0, 0, 1, 46, 0, 0, 34, 3951, 0, 4, 146, 1, 3, 197, 0, 3974, 0, 0, 165, 3821, 0, 137, 0, 0, 722, 0, 2624, 0, 0, 92, 3593, 0, 0, 28, 0, 0, 0, 23, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 3439, 0, 0, 2, 0, 0, 0, 0, 1938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2232, 0, 0, 2, 0, 0, 0, 0, 0, 175, 5, 1032, 0, 0, 3, 115, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4032, 0, 3630, 0, 0, 0, 3875, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 4092, 4050, 0, 0, 0, 0, 0, 0, 0, 250, 0, 0, 1, 2, 0, 0, 0, 0, 0, 154, 2, 0, 28, 0, 0, 0, 1, 0, 0, 48, 0, 1, 77, 8, 4095, 0, 1814, 0, 0, 2339, 4056, 1, 0, 0, 0, 3923, 0, 3893, 31, 0, 261, 3633, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 3151, 3743, 0, 0, 0, 417, 0, 0, 0, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 20, 0, 0, 9, 0, 0, 0, 0, 2279, 18, 4092, 0, 8, 0, 0, 0, 2091, 0, 0, 0, 3503, 42, 0, 63, 0, 3735, 0, 1688, 0, 0, 589, 83, 0, 0, 10, 0, 0, 2018, 88, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 2119, 0, 0, 0, 1959, 0, 0, 0, 3, 0, 3905, 0, 0, 0, 0, 0, 0, 0, 19, 189, 0, 2047, 0, 0, 0, 4096, 0, 4090, 0, 0, 0, 4089, 0, 0, 0, 0, 1343, 0, 0, 0, 0, 0, 645, 0, 0, 0, 0, 3, 70, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[39, 0, 0, 0, 116, 4096, 0, 306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 1099, 0, 0, 43, 178, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 393, 1578, 65, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3918, 221, 0, 0, 0, 0, 0, 0, 0, 0, 3495, 6, 163, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 0, 0, 22, 3732, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 599, 0, 16, 2947, 15, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4557, 1415, 6647, 4141, 3674, 4814, 3938, 3582] avg=4096.0000 max=6647 max_rank=2 min=1415 min_rank=1 imbalance_ratio=1.622803 max_load_reduction=26121 max_load_reduction_pct=0.797150 moved_to_echo=16008 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 33, 116, 3861, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 526, 1096, 0, 0, 0, 0, 0, 0, 0, 1201, 0, 0, 5, 29, 0, 3987, 404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 46, 0, 0, 34, 3910, 215, 0, 0, 0, 0, 0, 0, 3951, 0, 4, 146, 1, 3, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3974, 0, 0, 165, 732, 0, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1124, 0, 0, 92, 3593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 23, 2, 2874, 196, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4014, 1680, 5626, 4206, 4302, 5008, 4809, 3123] avg=4096.0000 max=5626 max_rank=2 min=1680 min_rank=1 imbalance_ratio=1.373535 max_load_reduction=27142 max_load_reduction_pct=0.828308 moved_to_echo=17242 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 113, 3906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 2, 44, 0, 0, 2974, 1688, 0, 0, 0, 0, 0, 0, 13, 4073, 0, 0, 0, 0, 9, 33, 618, 52, 0, 0, 0, 0, 0, 0, 0, 26, 0, 34, 128, 0, 10, 0, 3816, 227, 0, 0, 0, 0, 0, 0, 0, 171, 354, 0, 45, 231, 3696, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 773, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 1741, 3636, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 7, 0, 68, 12, 0, 0, 3032, 1109, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4020, 4717, 4798, 4241, 4499, 832, 5419, 4242] avg=4096.0000 max=5419 max_rank=6 min=832 min_rank=5 imbalance_ratio=1.322998 max_load_reduction=27349 max_load_reduction_pct=0.834625 moved_to_echo=14992 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 91, 3743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 417, 0, 0, 0, 1023, 0, 0, 2720, 704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 257, 525, 0, 0, 0, 0, 0, 0, 20, 0, 0, 9, 0, 0, 0, 0, 3060, 197, 0, 0, 0, 0, 0, 0, 2279, 18, 3567, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2091, 0, 0, 0, 672, 42, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 727, 0, 0, 589, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 2018, 88, 0, 2634, 1015, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3836, 4864, 4878, 3286, 5872, 2868, 1399, 5765] avg=4096.0000 max=5872 max_rank=4 min=1399 min_rank=6 imbalance_ratio=1.433594 max_load_reduction=26896 max_load_reduction_pct=0.820801 moved_to_echo=15904 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 118, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 250, 0, 0, 2858, 1623, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 154, 594, 0, 0, 0, 0, 0, 0, 0, 2, 0, 28, 0, 0, 0, 1, 0, 3974, 229, 0, 0, 0, 0, 0, 0, 0, 48, 0, 1, 77, 8, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1814, 0, 0, 2339, 777, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1676, 31, 0, 261, 3633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3050, 1065, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4176, 4731, 751, 4234, 4229, 4931, 5601, 4115] avg=4096.0000 max=5601 max_rank=6 min=751 min_rank=2 imbalance_ratio=1.367432 max_load_reduction=27167 max_load_reduction_pct=0.829071 moved_to_echo=18876 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 117, 4092, 0, 188, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 0, 0, 0, 1336, 0, 0, 2966, 1561, 0, 0, 0, 0, 0, 0, 2, 137, 0, 0, 0, 0, 0, 0, 571, 307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 3955, 230, 0, 0, 0, 0, 0, 0, 0, 800, 2085, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3788, 0, 0, 1, 779, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1611, 0, 0, 0, 3909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 3, 3061, 1106, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4398, 5872, 1017, 4214, 2928, 4570, 5520, 4249] avg=4096.0000 max=5872 max_rank=1 min=1017 min_rank=2 imbalance_ratio=1.433594 max_load_reduction=26896 max_load_reduction_pct=0.820801 moved_to_echo=17267 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 118, 3439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1938, 0, 0, 2937, 1514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2232, 554, 132, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 3978, 231, 0, 0, 0, 0, 0, 0, 175, 5, 900, 0, 0, 3, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1562, 0, 0, 0, 3875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3080, 1095, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3557, 6391, 2918, 4211, 1198, 4881, 5437, 4175] avg=4096.0000 max=6391 max_rank=1 min=1198 min_rank=4 imbalance_ratio=1.560303 max_load_reduction=26377 max_load_reduction_pct=0.804962 moved_to_echo=15691 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 114, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2119, 0, 0, 979, 0, 0, 0, 0, 0, 0, 0, 0, 1959, 0, 0, 0, 3, 0, 3905, 0, 266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 3981, 235, 0, 0, 0, 0, 0, 0, 189, 0, 1781, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 70, 0, 0, 3080, 364, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4210, 3098, 6133, 4235, 6066, 4864, 645, 3517] avg=4096.0000 max=6133 max_rank=2 min=645 min_rank=6 imbalance_ratio=1.497314 max_load_reduction=26635 max_load_reduction_pct=0.812836 moved_to_echo=15092 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 512, 256, 0, 128, 7936, 0, 0, 16128, 8448, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 11520, 0, 0, 128, 128, 512, 16000, 3072, 1920, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 128, 8704, 3712, 0, 1024, 19584, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[6656, 1152, 12288, 256, 384, 384, 12288, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[23936, 0, 128, 2560, 6144, 128, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 0, 128, 2816, 128, 128, 23808, 6016, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 1024, 31360, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 256, 0, 128, 128, 30592, 1792, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 3934, 0, 4096, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 7, 1, 0, 4056, 127, 0, 0, 0, 0, 0, 48, 0, 0, 2, 0, 4082, 1, 0, 0, 0, 0, 0, 0, 0, 110, 0, 1, 0, 0, 2, 0, 0, 3164, 0, 0, 1, 0, 4034, 8, 2332, 0, 0, 1296, 0, 45, 0, 357, 995, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[2034, 67, 0, 3997, 0, 4092, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 105, 0, 4025, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 1841, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 98, 0, 0, 0, 0, 0, 10, 0, 3843, 0, 18, 0, 0, 3968, 0, 0, 1, 0, 456, 0, 37, 0, 8, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[193, 21, 0, 4080, 0, 4073, 0, 31, 0, 0, 0, 4061, 0, 0, 24, 28, 0, 1, 3480, 0, 0, 1, 0, 0, 0, 1, 45, 0, 3420, 0, 3801, 0, 0, 0, 0, 5, 0, 6, 870, 0, 0, 0, 2, 0, 0, 24, 21, 4056, 103, 0, 0, 0, 4089, 0, 0, 22, 0, 0, 0, 0, 0, 9, 301, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 33, 0, 4038, 0, 4006, 0, 6, 0, 0, 0, 2933, 0, 0, 0, 0, 0, 0, 4090, 0, 1, 0, 0, 0, 302, 1, 0, 0, 0, 0, 3910, 0, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 1025, 0, 0, 0, 0, 3780, 0, 1551, 80, 0, 4091, 0, 0, 0, 0, 2757, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 4017, 84, 4072, 0, 0, 3, 0, 0, 4060, 0, 0, 17, 0, 0, 0, 4032, 0, 46, 0, 138, 0, 0, 0, 0, 0, 1575, 0, 3853, 0, 1076, 0, 0, 0, 0, 2, 2589, 0, 0, 0, 0, 2, 0, 0, 0, 758, 0, 0, 4, 0, 3472, 31, 14, 39, 0, 991, 0, 0, 0, 3, 1890, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 6, 6, 4090, 0, 4087, 4, 0, 0, 0, 0, 4014, 0, 0, 0, 0, 216, 0, 3445, 839, 0, 0, 0, 0, 0, 0, 0, 0, 238, 0, 4091, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 33, 0, 3983, 0, 0, 0, 0, 3866, 0, 68, 0, 0, 3772, 0, 0, 6, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 77, 4007, 0, 4082, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 51, 4096, 3993, 0, 0, 3, 0, 0, 0, 0, 0, 9, 0, 2879, 0, 62, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1211, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[101, 0, 1690, 3872, 0, 3981, 0, 0, 0, 0, 0, 3320, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 3383, 0, 0, 3, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4030, 0, 0, 0, 14, 4092, 0, 0, 0, 0, 6, 0, 0, 0, 0, 97, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 7, 261, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3002, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 2, 0, 3330, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 3503, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 3164, 444, 501, 112, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3661, 8, 2332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1296, 0, 45, 0, 357, 995, 0, 1054, 593, 251, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3490, 4337, 3130, 3381, 3613, 4224, 6002, 4591] avg=4096.0000 max=6002 max_rank=6 min=3130 min_rank=2 imbalance_ratio=1.465332 max_load_reduction=26766 max_load_reduction_pct=0.816833 moved_to_echo=18430 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2034, 67, 0, 3546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 257, 0, 0, 0, 0, 0, 0, 0, 105, 0, 2979, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 98, 3500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 3843, 451, 0, 110, 0, 0, 0, 0, 0, 0, 18, 0, 0, 3601, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 456, 0, 37, 0, 8, 0, 0, 1046, 592, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5647, 4354, 7150, 1841, 3603, 4414, 3620, 2139] avg=4096.0000 max=7150 max_rank=2 min=1841 min_rank=3 imbalance_ratio=1.745605 max_load_reduction=25618 max_load_reduction_pct=0.781799 moved_to_echo=13776 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 33, 0, 3582, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2933, 0, 0, 0, 0, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3027, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 302, 1, 0, 0, 0, 0, 3190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 0, 0, 0, 0, 0, 0, 3427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 1025, 456, 480, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3430, 0, 1551, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 2757, 0, 1063, 579, 240, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3621, 3178, 3028, 3493, 3572, 2085, 5061, 8730] avg=4096.0000 max=8730 max_rank=7 min=2085 min_rank=5 imbalance_ratio=2.131348 max_load_reduction=24038 max_load_reduction_pct=0.733582 moved_to_echo=19448 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3564, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4060, 0, 0, 17, 0, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2984, 0, 46, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1575, 0, 3143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1076, 0, 0, 0, 0, 2, 2589, 0, 3484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 758, 453, 473, 96, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3151, 31, 14, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 991, 0, 0, 0, 3, 1890, 0, 1048, 588, 237, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3648, 4305, 3168, 4718, 7151, 1782, 3239, 4757] avg=4096.0000 max=7151 max_rank=4 min=1782 min_rank=5 imbalance_ratio=1.745850 max_load_reduction=25617 max_load_reduction_pct=0.781769 moved_to_echo=16929 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 6, 3628, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4014, 0, 0, 0, 0, 251, 0, 0, 0, 0, 0, 0, 0, 216, 0, 2550, 839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 238, 0, 3338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 33, 0, 3983, 462, 502, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3508, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3772, 0, 0, 6, 0, 0, 0, 895, 591, 251, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3644, 4265, 3605, 3576, 3499, 5088, 3576, 5515] avg=4096.0000 max=5515 max_rank=7 min=3499 min_rank=4 imbalance_ratio=1.346436 max_load_reduction=27253 max_load_reduction_pct=0.831696 moved_to_echo=17678 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[193, 21, 0, 3619, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 24, 28, 265, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2576, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 45, 0, 3420, 0, 3101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 6, 870, 0, 3484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 24, 21, 4056, 461, 466, 114, 0, 0, 0, 0, 0, 103, 0, 0, 0, 3710, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 301, 0, 904, 589, 234, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3864, 4378, 2578, 6567, 4365, 5144, 3835, 2037] avg=4096.0000 max=6567 max_rank=3 min=2037 min_rank=7 imbalance_ratio=1.603271 max_load_reduction=26201 max_load_reduction_pct=0.799591 moved_to_echo=15381 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 77, 3555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 266, 0, 0, 0, 0, 0, 0, 0, 1, 51, 3032, 3993, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 2349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 0, 1, 0, 3491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 4096, 452, 353, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1211, 0, 0, 0, 0, 0, 0, 1064, 591, 177, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3632, 4362, 7080, 2358, 3554, 5023, 3716, 3043] avg=4096.0000 max=7080 max_rank=2 min=2358 min_rank=3 imbalance_ratio=1.728516 max_load_reduction=25688 max_load_reduction_pct=0.783936 moved_to_echo=15336 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[101, 0, 1690, 3431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3320, 0, 0, 0, 0, 269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 2753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 2, 0, 3406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4030, 441, 419, 118, 0, 0, 0, 0, 0, 0, 0, 0, 14, 3705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 97, 0, 1067, 575, 211, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5222, 3589, 3029, 6834, 3411, 5008, 3719, 1956] avg=4096.0000 max=6834 max_rank=3 min=1956 min_rank=7 imbalance_ratio=1.668457 max_load_reduction=25934 max_load_reduction_pct=0.791443 moved_to_echo=14094 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 11904, 0, 128, 128, 384, 6144, 0, 8192, 4736, 1664, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[384, 128, 128, 0, 11264, 0, 21248, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[384, 128, 23296, 9088, 128, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 128, 128, 128, 24960, 3712, 3200, 896, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[1408, 128, 0, 128, 128, 128, 3584, 256, 27904, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 28544, 128, 3968, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2432, 128, 1792, 28416, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 0, 30720, 128, 0, 128, 128, 2048, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True + [2026-09-21 08:25:36] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 258.8 | throughput per GPU (TFLOP/s/GPU): 124.5 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.023634E+01 | loss scale: 1.0 | grad norm: 29.817 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1989, 786, 107, 40, 19, 857, 927, 78, 9, 1453, 37, 1, 47, 65, 53, 683, 65, 529, 343, 1599, 37, 439, 402, 116, 705, 55, 46, 886, 838, 1107, 178, 520, 242, 347, 504, 372, 171, 1477, 282, 640, 508, 1068, 881, 5, 1806, 1239, 22, 456, 1035, 472, 730, 112, 1284, 1066, 695, 239, 769, 2, 557, 38, 41, 258, 368, 66] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1946, 626, 195, 32, 45, 1021, 887, 132, 149, 1336, 21, 1, 31, 225, 176, 302, 87, 631, 857, 1863, 56, 381, 294, 134, 634, 116, 45, 457, 592, 842, 313, 402, 339, 260, 798, 453, 185, 1696, 195, 873, 315, 832, 667, 10, 1419, 890, 22, 548, 1067, 384, 1188, 114, 983, 671, 1224, 167, 1174, 39, 603, 12, 82, 225, 417, 87] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1853, 757, 132, 79, 44, 439, 1030, 117, 10, 1657, 102, 0, 45, 203, 98, 775, 53, 346, 519, 1557, 19, 641, 367, 110, 811, 45, 98, 784, 453, 748, 362, 864, 222, 301, 665, 448, 300, 1759, 232, 787, 473, 1401, 577, 20, 2155, 1010, 27, 507, 1097, 581, 408, 28, 1112, 620, 351, 90, 998, 39, 583, 51, 14, 112, 622, 60] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1925, 744, 117, 80, 29, 687, 959, 78, 156, 971, 65, 1, 29, 161, 101, 923, 107, 488, 556, 1404, 41, 433, 288, 35, 859, 66, 126, 879, 468, 1134, 241, 818, 107, 247, 664, 261, 172, 1413, 264, 798, 478, 1640, 546, 22, 2338, 1064, 40, 386, 1025, 533, 828, 42, 938, 816, 466, 114, 1327, 10, 616, 18, 5, 66, 471, 84] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1497, 708, 86, 61, 34, 953, 1019, 208, 51, 880, 64, 12, 35, 108, 173, 528, 40, 431, 560, 1816, 22, 702, 279, 39, 980, 50, 66, 322, 530, 1651, 415, 621, 204, 244, 651, 166, 255, 1728, 240, 434, 482, 1179, 804, 26, 2070, 1264, 37, 699, 905, 486, 955, 118, 1113, 732, 617, 147, 964, 28, 761, 109, 18, 126, 169, 96] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1827, 740, 199, 53, 18, 756, 1429, 79, 12, 1926, 22, 1, 36, 174, 279, 533, 67, 333, 511, 1170, 72, 428, 491, 39, 716, 179, 19, 736, 515, 760, 208, 637, 303, 363, 585, 321, 303, 1189, 221, 820, 515, 808, 997, 18, 2141, 1099, 32, 538, 641, 594, 847, 151, 1124, 835, 732, 47, 1067, 6, 713, 29, 21, 157, 560, 26] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[2479, 781, 237, 28, 28, 811, 929, 59, 76, 1278, 38, 1, 57, 173, 128, 727, 46, 471, 592, 1466, 36, 332, 332, 63, 532, 43, 46, 829, 781, 882, 371, 658, 187, 279, 692, 142, 162, 1288, 290, 832, 627, 1432, 872, 13, 2358, 977, 22, 400, 923, 517, 940, 120, 1158, 775, 387, 51, 709, 14, 584, 100, 7, 328, 234, 38] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1785, 708, 183, 26, 25, 719, 1223, 76, 20, 1376, 96, 3, 17, 144, 136, 620, 132, 270, 759, 1237, 59, 447, 406, 80, 575, 126, 26, 677, 553, 989, 283, 729, 307, 351, 510, 322, 215, 1836, 513, 633, 366, 1505, 629, 19, 1816, 1298, 42, 449, 800, 849, 901, 86, 926, 730, 619, 131, 871, 2, 513, 88, 64, 116, 702, 54] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1242, 786, 107, 40, 19, 857, 927, 78, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1453, 37, 1, 47, 65, 53, 683, 1139, 398, 271, 0, 0, 0, 0, 0, 65, 529, 343, 1599, 37, 439, 402, 116, 349, 154, 31, 3, 0, 0, 0, 0, 705, 55, 46, 886, 838, 1076, 178, 520, 0, 0, 0, 0, 0, 0, 0, 0, 242, 347, 504, 372, 171, 1323, 282, 640, 0, 0, 0, 0, 0, 0, 0, 0, 508, 1065, 881, 5, 0, 1239, 22, 456, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 472, 730, 112, 145, 1066, 695, 239, 0, 0, 0, 0, 0, 0, 0, 0, 769, 2, 557, 38, 41, 258, 368, 66, 1535, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4056, 4156, 4067, 4304, 3881, 4176, 4494, 3634] avg=4096.0000 max=4494 max_rank=6 min=3634 min_rank=7 imbalance_ratio=1.097168 max_load_reduction=28274 max_load_reduction_pct=0.862854 moved_to_echo=16185 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1157, 757, 132, 79, 44, 439, 1030, 117, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1657, 102, 0, 45, 203, 98, 775, 987, 371, 324, 0, 0, 0, 0, 0, 53, 346, 519, 1557, 19, 641, 367, 110, 325, 183, 21, 4, 0, 0, 0, 0, 811, 45, 98, 784, 453, 727, 362, 864, 0, 0, 0, 0, 0, 0, 0, 0, 222, 301, 665, 448, 300, 1576, 232, 787, 0, 0, 0, 0, 0, 0, 0, 0, 473, 1397, 577, 20, 0, 1010, 27, 507, 0, 0, 0, 0, 0, 0, 0, 0, 1097, 581, 408, 28, 125, 620, 351, 90, 0, 0, 0, 0, 0, 0, 0, 0, 998, 39, 583, 51, 14, 112, 622, 60, 1831, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3755, 4572, 4145, 4144, 4531, 4011, 3300, 4310] avg=4096.0000 max=4572 max_rank=1 min=3300 min_rank=6 imbalance_ratio=1.116211 max_load_reduction=28196 max_load_reduction_pct=0.860474 moved_to_echo=16152 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1202, 744, 117, 80, 29, 687, 959, 78, 0, 0, 0, 0, 0, 0, 0, 0, 156, 971, 65, 1, 29, 161, 101, 923, 832, 385, 351, 0, 0, 0, 0, 0, 107, 488, 556, 1404, 41, 433, 288, 35, 338, 147, 32, 5, 0, 0, 0, 0, 859, 66, 126, 879, 468, 1102, 241, 818, 0, 0, 0, 0, 0, 0, 0, 0, 107, 247, 664, 261, 172, 1266, 264, 798, 0, 0, 0, 0, 0, 0, 0, 0, 478, 1635, 546, 22, 0, 1064, 40, 386, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 533, 828, 42, 106, 816, 466, 114, 0, 0, 0, 0, 0, 0, 0, 0, 1327, 10, 616, 18, 5, 66, 471, 84, 1987, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3896, 3975, 3874, 4559, 3779, 4171, 3930, 4584] avg=4096.0000 max=4584 max_rank=7 min=3779 min_rank=4 imbalance_ratio=1.119141 max_load_reduction=28184 max_load_reduction_pct=0.860107 moved_to_echo=16464 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1216, 626, 195, 32, 45, 1021, 887, 132, 0, 0, 0, 0, 0, 0, 0, 0, 149, 1336, 21, 1, 31, 225, 176, 302, 872, 389, 214, 0, 0, 0, 0, 0, 87, 631, 857, 1863, 56, 381, 294, 134, 341, 176, 23, 2, 0, 0, 0, 0, 634, 116, 45, 457, 592, 819, 313, 402, 0, 0, 0, 0, 0, 0, 0, 0, 339, 260, 798, 453, 185, 1520, 195, 873, 0, 0, 0, 0, 0, 0, 0, 0, 315, 830, 667, 10, 0, 890, 22, 548, 0, 0, 0, 0, 0, 0, 0, 0, 1067, 384, 1188, 114, 111, 671, 1224, 167, 0, 0, 0, 0, 0, 0, 0, 0, 1174, 39, 603, 12, 82, 225, 417, 87, 1205, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4154, 3716, 4845, 3378, 4623, 3282, 4926, 3844] avg=4096.0000 max=4926 max_rank=6 min=3282 min_rank=5 imbalance_ratio=1.202637 max_load_reduction=27842 max_load_reduction_pct=0.849670 moved_to_echo=16675 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[936, 708, 86, 61, 34, 953, 1019, 208, 0, 0, 0, 0, 0, 0, 0, 0, 51, 880, 64, 12, 35, 108, 173, 528, 988, 299, 312, 0, 0, 0, 0, 0, 40, 431, 560, 1816, 22, 702, 279, 39, 262, 180, 46, 4, 0, 0, 0, 0, 980, 50, 66, 322, 530, 1605, 415, 621, 0, 0, 0, 0, 0, 0, 0, 0, 204, 244, 651, 166, 255, 1548, 240, 434, 0, 0, 0, 0, 0, 0, 0, 0, 482, 1175, 804, 26, 0, 1264, 37, 699, 0, 0, 0, 0, 0, 0, 0, 0, 905, 486, 955, 118, 125, 732, 617, 147, 0, 0, 0, 0, 0, 0, 0, 0, 964, 28, 761, 109, 18, 126, 169, 96, 1758, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4005, 3450, 4381, 4589, 3742, 4487, 4085, 4029] avg=4096.0000 max=4589 max_rank=3 min=3450 min_rank=1 imbalance_ratio=1.120361 max_load_reduction=28179 max_load_reduction_pct=0.859955 moved_to_echo=16343 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1141, 740, 199, 53, 18, 756, 1429, 79, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1926, 22, 1, 36, 174, 279, 533, 997, 366, 323, 0, 0, 0, 0, 0, 67, 333, 511, 1170, 72, 428, 491, 39, 320, 124, 21, 2, 0, 0, 0, 0, 716, 179, 19, 736, 515, 739, 208, 637, 0, 0, 0, 0, 0, 0, 0, 0, 303, 363, 585, 321, 303, 1065, 221, 820, 0, 0, 0, 0, 0, 0, 0, 0, 515, 806, 997, 18, 0, 1099, 32, 538, 0, 0, 0, 0, 0, 0, 0, 0, 641, 594, 847, 151, 127, 835, 732, 47, 0, 0, 0, 0, 0, 0, 0, 0, 1067, 6, 713, 29, 21, 157, 560, 26, 1818, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4415, 4669, 3578, 3749, 3981, 4005, 3974, 4397] avg=4096.0000 max=4669 max_rank=1 min=3578 min_rank=2 imbalance_ratio=1.139893 max_load_reduction=28099 max_load_reduction_pct=0.857513 moved_to_echo=16357 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1548, 781, 237, 28, 28, 811, 929, 59, 0, 0, 0, 0, 0, 0, 0, 0, 76, 1278, 38, 1, 57, 173, 128, 727, 1028, 496, 354, 0, 0, 0, 0, 0, 46, 471, 592, 1466, 36, 332, 332, 63, 435, 134, 24, 5, 0, 0, 0, 0, 532, 43, 46, 829, 781, 858, 371, 658, 0, 0, 0, 0, 0, 0, 0, 0, 187, 279, 692, 142, 162, 1154, 290, 832, 0, 0, 0, 0, 0, 0, 0, 0, 627, 1427, 872, 13, 0, 977, 22, 400, 0, 0, 0, 0, 0, 0, 0, 0, 923, 517, 940, 120, 130, 775, 387, 51, 0, 0, 0, 0, 0, 0, 0, 0, 709, 14, 584, 100, 7, 328, 234, 38, 2004, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4421, 4356, 3936, 4118, 3738, 4338, 3843, 4018] avg=4096.0000 max=4421 max_rank=0 min=3738 min_rank=4 imbalance_ratio=1.079346 max_load_reduction=28347 max_load_reduction_pct=0.865082 moved_to_echo=15937 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1106, 708, 183, 26, 25, 719, 1223, 76, 0, 0, 0, 0, 0, 0, 0, 0, 20, 1376, 96, 3, 17, 144, 136, 620, 826, 362, 274, 0, 0, 0, 0, 0, 132, 270, 759, 1237, 59, 447, 406, 80, 317, 194, 31, 10, 0, 0, 0, 0, 575, 126, 26, 677, 553, 958, 283, 729, 0, 0, 0, 0, 0, 0, 0, 0, 307, 351, 510, 322, 215, 1642, 513, 633, 0, 0, 0, 0, 0, 0, 0, 0, 366, 1495, 629, 19, 0, 1298, 42, 449, 0, 0, 0, 0, 0, 0, 0, 0, 800, 849, 901, 86, 100, 730, 619, 131, 0, 0, 0, 0, 0, 0, 0, 0, 871, 2, 513, 88, 64, 116, 702, 54, 1542, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 3874, 3942, 3927, 4493, 4298, 4216, 3952] avg=4096.0000 max=4493 max_rank=4 min=3874 min_rank=1 imbalance_ratio=1.096924 max_load_reduction=28275 max_load_reduction_pct=0.862885 moved_to_echo=16959 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[512, 10880, 512, 128, 384, 1280, 1152, 5120, 7680, 3072, 2432, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1920, 2432, 5120, 2560, 1792, 11136, 2304, 5888, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[7552, 4480, 6912, 896, 1024, 6272, 5120, 1024, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[7936, 256, 4992, 512, 256, 1408, 3584, 512, 13696, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[640, 3584, 4736, 12160, 384, 3840, 2944, 640, 2688, 1408, 256, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[5888, 768, 512, 5632, 4736, 7936, 2432, 5376, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[9600, 5888, 1280, 512, 256, 6272, 8448, 896, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3840, 9856, 6016, 256, 0, 8960, 256, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 207, 250, 0, 1, 63, 973, 0, 0, 5, 0, 0, 4066, 0, 1, 0, 1119, 5, 0, 0, 0, 3279, 0, 3287, 3, 163, 0, 1, 1978, 0, 6, 7, 2542, 0, 24, 1, 0, 0, 120, 1809, 173, 2863, 18, 0, 3976, 0, 226, 5, 305, 1176, 32, 4, 0, 0, 0, 3891, 0, 0, 18, 0, 171, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 747, 379, 0, 30, 6, 315, 5, 0, 60, 0, 2, 4016, 2, 3, 0, 3898, 2, 48, 0, 0, 1386, 0, 3465, 0, 0, 0, 3, 1008, 0, 0, 1, 3657, 0, 51, 0, 0, 0, 0, 605, 32, 1098, 1355, 27, 4080, 0, 1, 9, 2689, 277, 2, 0, 0, 37, 0, 3367, 50, 0, 26, 28, 0, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[10, 370, 39, 0, 0, 0, 4, 0, 8, 0, 0, 22, 4054, 0, 0, 0, 3099, 0, 0, 0, 3, 1800, 1, 3948, 0, 0, 0, 0, 1909, 0, 0, 10, 4053, 0, 1, 0, 0, 0, 4, 1058, 33, 1050, 574, 0, 4026, 0, 13, 0, 2505, 44, 0, 25, 0, 0, 0, 4080, 0, 0, 4, 0, 2, 0, 19, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 3160, 137, 0, 0, 0, 22, 0, 0, 0, 54, 0, 3991, 0, 27, 0, 2409, 0, 5, 0, 0, 2863, 8, 805, 0, 0, 0, 0, 3383, 0, 0, 2, 1054, 0, 0, 0, 0, 0, 417, 1379, 15, 3193, 1, 0, 4091, 0, 18, 0, 2116, 1533, 0, 4, 0, 2, 0, 2068, 8, 0, 0, 0, 3, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[6, 1786, 99, 0, 0, 0, 0, 0, 28, 0, 2, 44, 3899, 0, 0, 7, 3788, 6, 5, 0, 0, 503, 9, 1972, 1, 8, 0, 19, 3788, 0, 1, 4, 2693, 7, 13, 0, 3, 0, 322, 897, 13, 3069, 63, 2, 4089, 0, 0, 1, 1129, 636, 1, 19, 0, 0, 0, 3771, 35, 0, 29, 0, 1, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 232, 331, 0, 0, 1, 493, 0, 0, 0, 108, 3, 3604, 0, 96, 0, 3079, 13, 22, 35, 0, 1346, 0, 972, 0, 122, 1, 2, 2978, 0, 5, 82, 4096, 0, 2, 0, 0, 1, 141, 1023, 2, 2488, 200, 2, 3929, 0, 1, 0, 2031, 1202, 41, 2, 0, 0, 2, 4056, 0, 0, 16, 0, 0, 0, 8, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[31, 2027, 2355, 0, 0, 0, 139, 1, 195, 1, 32, 0, 3979, 7, 0, 1, 2645, 0, 128, 7, 7, 820, 12, 4013, 3, 9, 1, 0, 1068, 25, 0, 4, 3613, 0, 1, 0, 0, 0, 165, 1024, 18, 706, 124, 0, 4079, 1, 0, 2, 853, 511, 0, 1, 0, 1, 0, 4086, 1, 0, 56, 3, 1, 0, 12, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[26, 2155, 14, 3, 24, 0, 113, 0, 2, 22, 3, 0, 3736, 2, 3, 0, 2015, 0, 107, 0, 5, 3179, 0, 2352, 0, 362, 31, 35, 2157, 0, 3, 144, 3584, 0, 7, 0, 0, 0, 201, 905, 0, 732, 428, 1, 3972, 0, 0, 0, 1459, 984, 0, 1185, 2, 0, 0, 2671, 0, 0, 94, 0, 49, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 207, 250, 0, 1, 63, 973, 0, 1148, 980, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4066, 0, 1, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 3279, 0, 2711, 0, 0, 0, 0, 0, 0, 0, 0, 3, 163, 0, 1, 1978, 0, 6, 7, 1079, 576, 203, 0, 0, 0, 0, 0, 2271, 0, 24, 1, 0, 0, 120, 1809, 0, 0, 0, 0, 0, 0, 0, 0, 173, 2863, 18, 0, 1753, 0, 226, 5, 0, 0, 0, 0, 0, 0, 0, 0, 305, 1176, 32, 4, 0, 0, 0, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 171, 0, 0, 0, 1119, 1243, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3622, 4140, 5995, 4016, 4225, 5038, 3181, 2551] avg=4096.0000 max=5995 max_rank=2 min=2551 min_rank=7 imbalance_ratio=1.463623 max_load_reduction=26773 max_load_reduction_pct=0.817047 moved_to_echo=14995 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 747, 379, 0, 30, 6, 315, 5, 993, 1006, 0, 0, 0, 0, 0, 0, 0, 60, 0, 2, 4016, 2, 3, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 2, 48, 0, 0, 1386, 0, 2858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1008, 0, 0, 1, 934, 607, 292, 0, 0, 0, 0, 0, 3267, 0, 51, 0, 0, 0, 0, 605, 0, 0, 0, 0, 0, 0, 0, 0, 32, 1098, 1355, 27, 1799, 0, 1, 9, 0, 0, 0, 0, 0, 0, 0, 0, 2689, 277, 2, 0, 0, 37, 0, 1440, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 26, 28, 0, 0, 1, 0, 3898, 1275, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3481, 4181, 4294, 2845, 3923, 4321, 4445, 5278] avg=4096.0000 max=5278 max_rank=7 min=2845 min_rank=3 imbalance_ratio=1.288574 max_load_reduction=27490 max_load_reduction_pct=0.838928 moved_to_echo=17967 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 3160, 137, 0, 0, 0, 22, 0, 610, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 3991, 0, 27, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 2863, 8, 664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3383, 0, 0, 2, 574, 141, 84, 0, 0, 0, 0, 0, 942, 0, 0, 0, 0, 0, 417, 1379, 0, 0, 0, 0, 0, 0, 0, 0, 15, 3193, 1, 0, 1805, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2116, 1533, 0, 4, 0, 2, 0, 884, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 3, 0, 0, 0, 2409, 1278, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4937, 4100, 3540, 4184, 2738, 5032, 4539, 3698] avg=4096.0000 max=5032 max_rank=5 min=2738 min_rank=4 imbalance_ratio=1.228516 max_load_reduction=27736 max_load_reduction_pct=0.846436 moved_to_echo=16007 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[10, 370, 39, 0, 0, 0, 4, 0, 1204, 992, 0, 0, 0, 0, 0, 0, 8, 0, 0, 22, 4054, 0, 0, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1800, 1, 3256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1909, 0, 0, 10, 1132, 692, 324, 0, 0, 0, 0, 0, 3620, 0, 1, 0, 0, 0, 4, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 33, 1050, 574, 0, 1776, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2505, 44, 0, 25, 0, 0, 0, 1744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2, 0, 19, 0, 3099, 1258, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2619, 4193, 5060, 4067, 4683, 3446, 4318, 4382] avg=4096.0000 max=5060 max_rank=2 min=2619 min_rank=0 imbalance_ratio=1.235352 max_load_reduction=27708 max_load_reduction_pct=0.845581 moved_to_echo=16829 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[26, 2155, 14, 3, 24, 0, 113, 0, 788, 979, 0, 0, 0, 0, 0, 0, 2, 22, 3, 0, 3736, 2, 3, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 0, 5, 3179, 0, 1940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 362, 31, 35, 2157, 0, 3, 144, 741, 412, 286, 0, 0, 0, 0, 0, 3202, 0, 7, 0, 0, 0, 201, 905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 732, 428, 1, 1752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1459, 984, 0, 1185, 2, 0, 0, 1142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 49, 0, 1, 0, 2015, 1241, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 3864, 5231, 4171, 4315, 2913, 4772, 3400] avg=4096.0000 max=5231 max_rank=2 min=2913 min_rank=5 imbalance_ratio=1.277100 max_load_reduction=27537 max_load_reduction_pct=0.840363 moved_to_echo=15400 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[6, 1786, 99, 0, 0, 0, 0, 0, 1113, 1008, 0, 0, 0, 0, 0, 0, 28, 0, 2, 44, 3899, 0, 0, 7, 72, 0, 0, 0, 0, 0, 0, 0, 0, 6, 5, 0, 0, 503, 9, 1627, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8, 0, 19, 3788, 0, 1, 4, 1046, 345, 215, 0, 0, 0, 0, 0, 2406, 7, 13, 0, 3, 0, 322, 897, 0, 0, 0, 0, 0, 0, 0, 0, 13, 3069, 63, 2, 1803, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1129, 636, 1, 19, 0, 0, 0, 1612, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 29, 0, 1, 0, 0, 0, 3788, 1278, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4012, 4052, 2150, 5427, 3648, 4951, 3397, 5131] avg=4096.0000 max=5427 max_rank=3 min=2150 min_rank=2 imbalance_ratio=1.324951 max_load_reduction=27341 max_load_reduction_pct=0.834381 moved_to_echo=17127 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 232, 331, 0, 0, 1, 493, 0, 1197, 968, 0, 0, 0, 0, 0, 0, 0, 0, 108, 3, 3604, 0, 96, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 13, 22, 35, 0, 1346, 0, 802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 1, 2, 2978, 0, 5, 82, 1125, 170, 327, 0, 0, 0, 0, 0, 3659, 0, 2, 0, 0, 1, 141, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2488, 200, 2, 1733, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2031, 1202, 41, 2, 0, 0, 2, 1734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 8, 0, 3079, 1228, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3222, 3921, 2218, 4812, 4826, 4426, 5012, 4331] avg=4096.0000 max=5012 max_rank=6 min=2218 min_rank=2 imbalance_ratio=1.223633 max_load_reduction=27756 max_load_reduction_pct=0.847046 moved_to_echo=18595 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[31, 2027, 2355, 0, 0, 0, 139, 1, 1210, 1010, 0, 0, 0, 0, 0, 0, 195, 1, 32, 0, 3979, 7, 0, 1, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 7, 7, 820, 12, 3306, 0, 0, 0, 0, 0, 0, 0, 0, 3, 9, 1, 0, 1068, 25, 0, 4, 1138, 707, 291, 0, 0, 0, 0, 0, 3220, 0, 1, 0, 0, 0, 165, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 18, 706, 124, 0, 1790, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 853, 511, 0, 1, 0, 1, 0, 1738, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 56, 3, 1, 0, 12, 0, 2645, 1279, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6773, 4317, 4280, 3246, 4410, 2641, 3104, 3997] avg=4096.0000 max=6773 max_rank=0 min=2641 min_rank=5 imbalance_ratio=1.653564 max_load_reduction=25995 max_load_reduction_pct=0.793304 moved_to_echo=14152 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 10752, 3712, 128, 128, 128, 2176, 128, 8320, 8064, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[13184, 6400, 128, 1280, 128, 128, 128, 12032, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 384, 128, 128, 15232, 128, 17280, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 768, 128, 128, 18304, 128, 128, 256, 7808, 3712, 2048, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[384, 15232, 2816, 128, 14336, 128, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 256, 128, 256, 0, 128, 0, 22144, 10112, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[22656, 128, 128, 128, 128, 128, 1408, 8704, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[256, 128, 256, 128, 31360, 128, 256, 128, 768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 1, 2, 4, 17, 0, 67, 0, 893, 0, 10, 0, 3, 592, 3795, 71, 3056, 0, 0, 0, 0, 0, 0, 653, 0, 4077, 3185, 0, 0, 0, 0, 28, 113, 3847, 35, 52, 0, 0, 0, 0, 59, 98, 4096, 2, 0, 61, 521, 0, 3975, 1, 1, 0, 1, 0, 0, 1, 0, 0, 0, 3445, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[18, 1, 0, 0, 100, 0, 0, 0, 36, 0, 0, 231, 6, 0, 0, 0, 8, 3862, 6, 4013, 0, 0, 0, 0, 0, 0, 28, 1, 3702, 144, 0, 96, 0, 2, 0, 0, 4084, 2, 0, 0, 0, 0, 0, 161, 19, 3568, 0, 1, 0, 3929, 3, 4027, 44, 0, 29, 17, 30, 494, 0, 6, 0, 16, 4084, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[205, 1, 0, 0, 0, 0, 1, 6, 0, 915, 0, 1392, 5, 0, 0, 0, 2, 3372, 0, 4071, 1031, 0, 0, 0, 0, 0, 119, 0, 2012, 203, 0, 5, 0, 0, 362, 0, 4083, 0, 3, 0, 0, 0, 0, 8, 4, 3953, 0, 0, 2, 2709, 0, 4048, 0, 0, 1, 162, 0, 3, 0, 0, 0, 0, 4090, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[231, 0, 0, 0, 27, 0, 0, 0, 0, 732, 0, 55, 0, 0, 0, 0, 16, 3723, 25, 96, 146, 0, 0, 0, 0, 0, 21, 0, 3914, 2211, 0, 0, 4, 0, 588, 1, 4013, 0, 39, 0, 0, 0, 0, 1630, 1, 3330, 3, 0, 0, 3808, 0, 4055, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 4095, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1947, 0, 64, 0, 3, 0, 0, 0, 50, 2405, 0, 0, 26, 0, 11, 14, 64, 2458, 15, 1794, 1109, 1, 0, 0, 0, 0, 0, 0, 1472, 198, 1, 91, 1, 0, 766, 2, 4075, 1, 112, 4, 0, 0, 0, 0, 8, 3969, 8, 0, 0, 4064, 0, 3971, 0, 24, 5, 14, 0, 0, 0, 0, 0, 0, 4021, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1988, 68, 3, 0, 0, 0, 0, 106, 3006, 0, 3603, 2, 0, 0, 0, 0, 0, 110, 0, 4095, 120, 0, 2, 0, 0, 4, 0, 3699, 0, 0, 0, 0, 0, 0, 227, 0, 3954, 6, 0, 0, 3588, 0, 3900, 0, 0, 92, 1, 0, 428, 0, 0, 0, 0, 3766, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[12, 0, 7, 1, 0, 0, 0, 2, 1, 917, 0, 21, 0, 0, 0, 0, 312, 2646, 4, 1688, 71, 0, 0, 0, 0, 1, 36, 0, 4080, 2206, 0, 0, 0, 0, 807, 5, 3804, 0, 15, 1, 0, 0, 0, 4, 0, 2711, 0, 0, 2, 3221, 0, 4086, 0, 1, 23, 1836, 0, 152, 0, 0, 0, 0, 4095, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[16, 0, 112, 0, 0, 0, 3, 2, 241, 786, 0, 664, 0, 0, 0, 0, 0, 1374, 20, 905, 0, 0, 0, 2, 0, 0, 0, 0, 3515, 1338, 0, 746, 1, 12, 0, 4, 4076, 0, 155, 8, 0, 0, 0, 417, 1, 3929, 2208, 0, 31, 2405, 15, 4076, 0, 0, 0, 1925, 5, 0, 24, 0, 0, 0, 3750, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[18, 1, 0, 0, 100, 0, 0, 0, 3496, 330, 0, 0, 0, 0, 0, 0, 36, 0, 0, 231, 6, 0, 0, 0, 1948, 773, 262, 163, 0, 0, 0, 0, 8, 1584, 6, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 1, 2929, 144, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3822, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 19, 3374, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3929, 3, 531, 44, 0, 29, 17, 0, 0, 0, 0, 0, 0, 0, 0, 30, 494, 0, 6, 0, 16, 4084, 0, 31, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3945, 3419, 5611, 3198, 3826, 3555, 4553, 4661] avg=4096.0000 max=5611 max_rank=2 min=3198 min_rank=3 imbalance_ratio=1.369873 max_load_reduction=27157 max_load_reduction_pct=0.828766 moved_to_echo=16595 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 1, 2, 4, 17, 3451, 324, 0, 0, 0, 0, 0, 0, 0, 67, 0, 893, 0, 10, 0, 3, 1914, 851, 247, 187, 0, 0, 0, 0, 592, 1557, 71, 3056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 653, 0, 3226, 3185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 113, 3600, 35, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 98, 3874, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 521, 0, 524, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3445, 0, 35, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3805, 4172, 5276, 7064, 3828, 4033, 1109, 3481] avg=4096.0000 max=7064 max_rank=3 min=1109 min_rank=6 imbalance_ratio=1.724609 max_load_reduction=25704 max_load_reduction_pct=0.784424 moved_to_echo=12451 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[205, 1, 0, 0, 0, 0, 1, 6, 3515, 288, 0, 0, 0, 0, 0, 0, 0, 915, 0, 1392, 5, 0, 0, 0, 1700, 420, 262, 180, 0, 0, 0, 0, 2, 1384, 0, 4071, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 0, 1592, 203, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 362, 0, 3821, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4, 3739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2709, 0, 533, 0, 0, 1, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4090, 0, 34, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4016, 4874, 6488, 1919, 4186, 3751, 3407, 4127] avg=4096.0000 max=6488 max_rank=2 min=1919 min_rank=3 imbalance_ratio=1.583984 max_load_reduction=26280 max_load_reduction_pct=0.802002 moved_to_echo=15471 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1947, 0, 64, 0, 3, 0, 0, 0, 3448, 210, 0, 0, 0, 0, 0, 0, 50, 2405, 0, 0, 26, 0, 11, 14, 1239, 307, 262, 181, 0, 0, 0, 0, 64, 1009, 15, 1794, 1109, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1165, 198, 1, 91, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 766, 2, 3813, 1, 112, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3754, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 523, 0, 24, 5, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4021, 0, 34, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5672, 4495, 3992, 1455, 4699, 3770, 4630, 4055] avg=4096.0000 max=5672 max_rank=0 min=1455 min_rank=3 imbalance_ratio=1.384766 max_load_reduction=27096 max_load_reduction_pct=0.826904 moved_to_echo=17154 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[231, 0, 0, 0, 27, 0, 0, 0, 3521, 318, 0, 0, 0, 0, 0, 0, 0, 732, 0, 55, 0, 0, 0, 0, 1877, 817, 258, 152, 0, 0, 0, 0, 16, 1528, 25, 96, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 3097, 2211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 588, 1, 3755, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1630, 1, 3149, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3808, 0, 534, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4095, 0, 29, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 3891, 1811, 5329, 4387, 4783, 4343, 4127] avg=4096.0000 max=5329 max_rank=3 min=1811 min_rank=2 imbalance_ratio=1.301025 max_load_reduction=27439 max_load_reduction_pct=0.837372 moved_to_echo=17640 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[16, 0, 112, 0, 0, 0, 3, 2, 3539, 117, 0, 0, 0, 0, 0, 0, 241, 786, 0, 664, 0, 0, 0, 0, 693, 734, 262, 179, 0, 0, 0, 0, 0, 564, 20, 905, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2781, 1338, 0, 746, 0, 0, 0, 0, 0, 0, 0, 0, 1, 12, 0, 4, 3814, 0, 155, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 417, 1, 3716, 2208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 2405, 15, 537, 0, 0, 0, 1925, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 24, 0, 0, 0, 3750, 2, 34, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3789, 3559, 1491, 4865, 3994, 6342, 4913, 3815] avg=4096.0000 max=6342 max_rank=5 min=1491 min_rank=2 imbalance_ratio=1.548340 max_load_reduction=26426 max_load_reduction_pct=0.806458 moved_to_echo=19064 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 3386, 256, 0, 0, 0, 0, 0, 0, 0, 1988, 68, 3, 0, 0, 0, 0, 1516, 855, 238, 180, 0, 0, 0, 0, 106, 1234, 0, 3603, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 3240, 120, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 227, 0, 3740, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3588, 0, 514, 0, 0, 92, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 428, 0, 0, 0, 0, 3766, 0, 34, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3642, 4848, 4945, 3472, 3465, 3973, 4195, 4228] avg=4096.0000 max=4945 max_rank=2 min=3465 min_rank=4 imbalance_ratio=1.207275 max_load_reduction=27823 max_load_reduction_pct=0.849091 moved_to_echo=15861 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 7, 1, 0, 0, 0, 2, 3551, 229, 0, 0, 0, 0, 0, 0, 1, 917, 0, 21, 0, 0, 0, 0, 1338, 857, 249, 127, 0, 0, 0, 0, 312, 1079, 4, 1688, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 36, 0, 3223, 2206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 807, 5, 3555, 0, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3221, 0, 535, 0, 1, 23, 1836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 0, 0, 0, 0, 4095, 0, 27, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3802, 3510, 3154, 5466, 4383, 2561, 5618, 4274] avg=4096.0000 max=5618 max_rank=6 min=2561 min_rank=5 imbalance_ratio=1.371582 max_load_reduction=27150 max_load_reduction_pct=0.828552 moved_to_echo=16836 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[384, 7936, 128, 3328, 128, 128, 128, 128, 12288, 5632, 2048, 1408, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 1024, 128, 21376, 9728, 128, 1024, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1152, 9984, 256, 19328, 2432, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 24320, 128, 4352, 128, 128, 256, 3968, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[2432, 128, 256, 128, 256, 128, 128, 128, 28032, 2176, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 0, 2560, 256, 27904, 2304, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 2560, 128, 29696, 128, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 1152, 128, 128, 0, 128, 31360, 128, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33792 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 1, 0, 1, 4055, 469, 3932, 3844, 0, 37, 1, 0, 2, 0, 0, 0, 4095, 0, 0, 0, 5, 13, 0, 5, 62, 0, 9, 0, 38, 433, 0, 2, 4060, 0, 0, 2808, 0, 0, 0, 3489, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 5, 0, 3, 0, 0, 0, 243, 4095, 56, 1, 1001] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[16, 1, 0, 0, 4095, 0, 4090, 3827, 0, 0, 40, 0, 1, 0, 0, 2, 4096, 0, 0, 0, 50, 0, 0, 69, 0, 2, 0, 0, 0, 8, 1, 0, 3403, 0, 0, 3482, 0, 1, 0, 4055, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1432, 4069, 1, 0, 22] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 437, 0, 4096, 1, 4062, 4071, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 11, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1354, 0, 0, 3311, 8, 0, 779, 0, 0, 0, 1439, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2203, 3987, 2907, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[4, 0, 118, 0, 4090, 53, 4096, 3996, 2, 0, 0, 0, 0, 7, 0, 0, 3920, 0, 0, 0, 135, 0, 0, 0, 2284, 0, 0, 0, 0, 2927, 0, 8, 2177, 11, 0, 554, 0, 0, 0, 2807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4, 0, 1, 0, 0, 0, 0, 2241, 3159, 148, 0, 21] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[2, 0, 4, 0, 4095, 176, 4092, 3907, 0, 0, 1, 0, 1, 0, 0, 0, 4096, 0, 9, 0, 0, 0, 0, 2, 1378, 0, 15, 0, 1, 310, 0, 0, 125, 1, 0, 3389, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 854, 4094, 2116, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 620, 2, 4094, 988, 4086, 4055, 0, 60, 0, 0, 0, 0, 0, 6, 4082, 0, 1, 4, 0, 0, 1, 0, 5, 0, 0, 18, 0, 756, 0, 16, 3749, 0, 0, 1386, 0, 0, 0, 4013, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 696, 4064, 18, 0, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[1, 0, 68, 0, 3928, 2, 4093, 4002, 0, 20, 163, 0, 0, 0, 0, 0, 4063, 0, 911, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 80, 6, 4, 184, 0, 0, 2918, 0, 0, 2, 4032, 58, 1, 0, 8, 0, 0, 2, 0, 0, 40, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 4084, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4081, 27, 4095, 2455, 0, 0, 0, 0, 16, 3, 0, 20, 4086, 0, 0, 0, 39, 0, 5, 4, 32, 0, 0, 0, 0, 0, 0, 0, 3701, 4, 0, 1906, 0, 0, 0, 4067, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 3587, 3874, 386, 0, 372] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 1, 0, 469, 0, 3796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 1, 0, 2, 0, 0, 0, 3456, 606, 0, 0, 0, 0, 0, 0, 3966, 0, 0, 0, 5, 13, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 9, 0, 38, 433, 0, 2, 2138, 1017, 129, 48, 0, 0, 0, 0, 3043, 0, 0, 2808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3932, 18, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 5, 0, 3, 4037, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 1351, 56, 1, 1001, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4267, 4102, 3989, 3876, 5851, 3952, 4079, 2652] avg=4096.0000 max=5851 max_rank=4 min=2652 min_rank=7 imbalance_ratio=1.428467 max_load_reduction=26917 max_load_reduction_pct=0.821442 moved_to_echo=16534 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[16, 1, 0, 0, 0, 0, 0, 3779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 1, 0, 0, 2, 4016, 602, 0, 0, 0, 0, 0, 0, 3967, 0, 0, 0, 50, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8, 1, 0, 2125, 852, 129, 48, 0, 0, 0, 0, 2551, 0, 0, 3482, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4090, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4078, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1432, 1342, 1, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3796, 4661, 4086, 3165, 6034, 4108, 4121, 2797] avg=4096.0000 max=6034 max_rank=4 min=2797 min_rank=7 imbalance_ratio=1.473145 max_load_reduction=26734 max_load_reduction_pct=0.815857 moved_to_echo=17060 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 620, 2, 0, 988, 0, 4004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 6, 3974, 601, 0, 0, 0, 0, 0, 0, 3953, 0, 1, 4, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 18, 0, 756, 0, 16, 2122, 939, 129, 51, 0, 0, 0, 0, 2810, 0, 0, 1386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4086, 17, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 4077, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 696, 1341, 18, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5614, 4641, 3959, 4036, 4196, 4104, 4160, 2058] avg=4096.0000 max=5614 max_rank=0 min=2058 min_rank=7 imbalance_ratio=1.370605 max_load_reduction=27154 max_load_reduction_pct=0.828674 moved_to_echo=14518 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 118, 0, 0, 53, 0, 3946, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 7, 0, 0, 2781, 467, 0, 0, 0, 0, 0, 0, 3796, 0, 0, 0, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2284, 0, 0, 0, 0, 2927, 0, 8, 1649, 545, 124, 50, 0, 0, 0, 0, 1632, 11, 0, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 18, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4, 0, 1, 0, 4072, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2241, 1043, 148, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 3257, 3931, 7587, 2197, 4114, 4108, 3453] avg=4096.0000 max=7587 max_rank=3 min=2197 min_rank=4 imbalance_ratio=1.852295 max_load_reduction=25181 max_load_reduction_pct=0.768463 moved_to_echo=13872 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 437, 0, 0, 1, 0, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1425, 590, 0, 0, 0, 0, 0, 0, 3967, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1354, 0, 0, 2082, 829, 129, 51, 0, 0, 0, 0, 2482, 8, 0, 779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4062, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2203, 1315, 2907, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4458, 2016, 3978, 4448, 3269, 4080, 4093, 6426] avg=4096.0000 max=6426 max_rank=7 min=2016 min_rank=1 imbalance_ratio=1.568848 max_load_reduction=26342 max_load_reduction_pct=0.803894 moved_to_echo=17868 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 4, 0, 0, 176, 0, 3858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 4057, 606, 0, 0, 0, 0, 0, 0, 3967, 0, 9, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1378, 0, 15, 0, 1, 310, 0, 0, 2138, 31, 129, 49, 0, 0, 0, 0, 94, 1, 0, 3389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 4092, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 854, 1350, 2116, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4040, 4665, 3978, 4051, 3484, 4112, 4116, 4322] avg=4096.0000 max=4665 max_rank=1 min=3484 min_rank=4 imbalance_ratio=1.138916 max_load_reduction=28103 max_load_reduction_pct=0.857635 moved_to_echo=16034 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 68, 0, 0, 2, 0, 3952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 163, 0, 0, 0, 0, 0, 3993, 604, 0, 0, 0, 0, 0, 0, 3935, 0, 911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 80, 6, 4, 2132, 46, 128, 50, 0, 0, 0, 0, 138, 0, 0, 2918, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 1, 0, 8, 0, 0, 2, 0, 4093, 18, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 3910, 39, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 1348, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4023, 4780, 4846, 2448, 3058, 4180, 3989, 5444] avg=4096.0000 max=5444 max_rank=7 min=2448 min_rank=3 imbalance_ratio=1.329102 max_load_reduction=27324 max_load_reduction_pct=0.833862 moved_to_echo=16671 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 27, 0, 2421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 3, 0, 20, 4029, 578, 0, 0, 0, 0, 0, 0, 3953, 0, 0, 0, 39, 0, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 2026, 932, 133, 34, 0, 0, 0, 0, 2769, 4, 0, 1906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4095, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4063, 38, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3587, 1270, 386, 0, 372, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2449, 4646, 4001, 3157, 4679, 4118, 4102, 5616] avg=4096.0000 max=5616 max_rank=7 min=2449 min_rank=0 imbalance_ratio=1.371094 max_load_reduction=27152 max_load_reduction_pct=0.828613 moved_to_echo=18515 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[3840, 128, 128, 128, 128, 5888, 128, 128, 16512, 5248, 1152, 384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 0, 15360, 10368, 5760, 128, 1536, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[15616, 128, 0, 17280, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 1280, 128, 0, 1792, 0, 29824, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 256, 0, 128, 128, 0, 128, 27776, 4736, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[31616, 0, 1024, 128, 256, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 128, 0, 128, 128, 32640, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 0, 128, 128, 128, 128, 32512, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33792 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33792 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33792 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 36, 0, 0, 0, 4066, 8, 79, 0, 0, 29, 4042, 0, 0, 790, 0, 0, 3836, 5, 4056, 3, 1, 4, 0, 0, 3605, 4017, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 4, 0, 0, 13, 0, 2, 3203, 0, 0, 2, 5, 0, 0, 2, 0, 0, 579, 0, 0, 0, 0, 52] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 137, 0, 1, 4096, 8, 0, 0, 0, 0, 3189, 0, 0, 104, 0, 9, 4094, 0, 2944, 7, 0, 82, 0, 0, 3999, 3738, 0, 0, 0, 0, 0, 2, 1473, 0, 652, 0, 3982, 0, 1, 0, 0, 0, 71, 3, 0, 92, 4043, 0, 0, 0, 15, 0, 0, 0, 25, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 3436, 0, 0, 70, 0, 5, 3488, 3, 3413, 624, 0, 4, 0, 0, 4096, 4080, 344, 0, 0, 0, 0, 0, 102, 0, 8, 0, 4093, 0, 20, 0, 0, 0, 0, 0, 0, 780, 4095, 1, 1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 690, 40, 37, 4010, 0, 0, 2, 0, 0, 4026, 0, 0, 0, 3, 0, 4045, 604, 3952, 1382, 172, 186, 0, 1, 4086, 863, 19, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4080, 0, 64, 0, 0, 0, 0, 0, 1, 507, 3782, 62, 8, 53, 42, 2, 0, 3, 2, 0, 0, 38, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 5, 0, 1, 0, 4090, 0, 0, 0, 0, 0, 3651, 2, 5, 0, 0, 0, 3861, 3, 4085, 345, 0, 17, 0, 0, 4084, 4059, 0, 0, 0, 0, 0, 0, 401, 0, 20, 0, 4050, 0, 0, 0, 0, 0, 2, 1, 0, 0, 3878, 168, 1, 0, 13, 0, 2, 0, 21, 0, 1, 1, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 4045, 17, 3974, 2669, 1, 2, 0, 0, 4086, 3478, 0, 0, 0, 0, 0, 0, 4, 0, 27, 0, 4080, 1, 0, 0, 0, 0, 0, 0, 0, 3755, 2516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 10, 4084, 0, 0, 59, 13, 0, 2483, 2, 0, 9, 0, 0, 4005, 0, 3106, 983, 0, 42, 2, 0, 3938, 4090, 5, 0, 0, 0, 0, 0, 105, 0, 2, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3223, 1931, 3, 0, 24, 0, 2, 113, 4, 0, 3, 0, 0, 0, 0, 0, 434] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 4073, 0, 6, 0, 0, 0, 2902, 0, 0, 1, 21, 0, 4088, 0, 3927, 3254, 0, 0, 0, 0, 4094, 3787, 2, 0, 0, 22, 0, 0, 2, 0, 51, 0, 4092, 0, 0, 3, 0, 0, 155, 0, 3, 3, 2273, 1, 1, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 1, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 3436, 0, 0, 70, 472, 483, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3, 2587, 624, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 4080, 344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 8, 0, 3761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 780, 3081, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 0, 0, 0, 0, 354, 332, 102, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3488, 16, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3994, 4467, 3223, 4940, 3871, 3881, 4885, 3507] avg=4096.0000 max=4940 max_rank=3 min=3223 min_rank=2 imbalance_ratio=1.206055 max_load_reduction=27828 max_load_reduction_pct=0.849243 moved_to_echo=16144 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 137, 0, 1, 3994, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3189, 0, 0, 104, 407, 472, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 2232, 7, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 3738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1473, 0, 652, 0, 3660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 71, 3, 0, 92, 3008, 0, 0, 0, 0, 0, 0, 0, 4043, 0, 0, 0, 15, 0, 0, 0, 305, 322, 102, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 1, 0, 0, 4094, 16, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4140, 4172, 2330, 4241, 5787, 3175, 4787, 4136] avg=4096.0000 max=5787 max_rank=4 min=2330 min_rank=2 imbalance_ratio=1.412842 max_load_reduction=26981 max_load_reduction_pct=0.823395 moved_to_echo=17885 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 10, 3982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 13, 0, 2483, 2, 0, 9, 429, 465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2355, 983, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 495, 4090, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 2, 0, 3761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3223, 2962, 0, 0, 0, 0, 0, 0, 0, 1931, 3, 0, 24, 0, 2, 113, 4, 322, 332, 102, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 434, 4005, 16, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3992, 3460, 3380, 4592, 3868, 6185, 2833, 4458] avg=4096.0000 max=6185 max_rank=5 min=2833 min_rank=6 imbalance_ratio=1.510010 max_load_reduction=26583 max_load_reduction_pct=0.811249 moved_to_echo=17344 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 36, 0, 0, 0, 3965, 8, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 29, 4042, 0, 0, 790, 561, 425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3074, 3, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 454, 4017, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 13, 0, 2, 2712, 0, 0, 0, 0, 0, 0, 0, 3203, 0, 0, 2, 5, 0, 0, 2, 421, 331, 101, 0, 0, 0, 0, 0, 0, 0, 579, 0, 0, 0, 0, 52, 3836, 14, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4009, 5926, 3087, 4711, 3758, 2731, 4065, 4481] avg=4096.0000 max=5926 max_rank=1 min=2731 min_rank=5 imbalance_ratio=1.446777 max_load_reduction=26842 max_load_reduction_pct=0.819153 moved_to_echo=15035 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 690, 40, 37, 3910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4026, 0, 0, 0, 546, 482, 0, 0, 0, 0, 0, 0, 3, 0, 0, 604, 2996, 1382, 172, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 515, 863, 19, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 1, 507, 3073, 0, 0, 0, 0, 0, 0, 0, 3782, 62, 8, 53, 42, 2, 0, 3, 410, 330, 100, 0, 0, 0, 0, 0, 2, 0, 0, 38, 0, 0, 0, 0, 4045, 16, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4677, 5056, 5343, 1404, 3750, 3645, 4792, 4101] avg=4096.0000 max=5343 max_rank=2 min=1404 min_rank=3 imbalance_ratio=1.304443 max_load_reduction=27425 max_load_reduction_pct=0.836945 moved_to_echo=16288 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 5, 0, 1, 0, 3988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3651, 2, 5, 0, 565, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3096, 345, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 401, 0, 20, 0, 3722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 3072, 0, 0, 0, 0, 0, 0, 0, 3878, 168, 1, 0, 13, 0, 2, 0, 424, 328, 102, 0, 0, 0, 0, 0, 21, 0, 1, 1, 0, 0, 0, 1, 3861, 16, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3994, 4705, 3461, 4573, 4143, 3075, 4916, 3901] avg=4096.0000 max=4916 max_rank=6 min=3075 min_rank=5 imbalance_ratio=1.200195 max_load_reduction=27852 max_load_reduction_pct=0.849976 moved_to_echo=16035 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 549, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 3013, 2669, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 3478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 27, 0, 3750, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3755, 3073, 0, 0, 0, 0, 0, 0, 0, 2516, 0, 0, 0, 0, 0, 0, 0, 412, 330, 102, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 1, 0, 0, 4045, 16, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3994, 1039, 5702, 3993, 3782, 6828, 3360, 4070] avg=4096.0000 max=6828 max_rank=5 min=1039 min_rank=1 imbalance_ratio=1.666992 max_load_reduction=25940 max_load_reduction_pct=0.791626 moved_to_echo=18040 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 3967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 2902, 0, 0, 1, 547, 487, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 2967, 3254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 3787, 2, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 51, 0, 3756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 155, 0, 3, 3, 3084, 0, 0, 0, 0, 0, 0, 0, 2273, 1, 1, 0, 0, 0, 0, 0, 413, 336, 106, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 1, 1, 4088, 20, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3968, 3943, 6242, 4314, 3809, 3248, 3130, 4114] avg=4096.0000 max=6242 max_rank=2 min=3130 min_rank=6 imbalance_ratio=1.523926 max_load_reduction=26526 max_load_reduction_pct=0.809509 moved_to_echo=14301 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 23808, 128, 128, 1024, 4096, 3840, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 256, 128, 128, 8448, 24192, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 640, 22400, 9344, 256, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[25728, 256, 128, 128, 128, 128, 128, 128, 3072, 2688, 896, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 4096, 28160, 640, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 128, 896, 128, 128, 31872, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 640, 128, 0, 128, 128, 512, 31488, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 2176, 0, 768, 0, 29952, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4002, 0, 0, 4, 4093, 0, 0, 0, 0, 67, 34, 3544, 0, 0, 3975, 448, 0, 184, 0, 0, 375, 698, 0, 2, 1053, 0, 0, 0, 0, 1302, 0, 66, 0, 0, 173, 5, 0, 2, 0, 0, 0, 0, 0, 3, 0, 4, 0, 0, 0, 3903, 4096, 0, 0, 0, 2, 4096, 0, 634, 0, 0, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 3763, 1662, 2, 6, 4094, 7, 0, 0, 0, 0, 0, 3890, 0, 0, 4073, 0, 0, 3, 0, 0, 442, 0, 13, 0, 2332, 0, 0, 0, 0, 0, 20, 0, 0, 0, 1199, 1267, 63, 0, 0, 9, 0, 22, 0, 135, 12, 0, 0, 1, 0, 3088, 2242, 0, 0, 0, 507, 3913, 0, 0, 0, 0, 0, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4095, 1665, 0, 8, 4005, 0, 6, 0, 0, 1, 0, 2977, 0, 0, 4094, 0, 1, 4, 0, 0, 645, 96, 0, 0, 2310, 0, 0, 0, 0, 645, 0, 0, 0, 0, 28, 20, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 151, 0, 0, 3956, 4094, 0, 0, 0, 23, 3937, 4, 0, 0, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 3890, 889, 0, 61, 3943, 0, 0, 0, 0, 0, 0, 2201, 0, 0, 4094, 5, 1, 0, 0, 0, 1531, 239, 201, 259, 3165, 0, 0, 0, 0, 0, 0, 0, 34, 0, 1285, 0, 0, 395, 0, 11, 0, 0, 0, 0, 0, 42, 0, 0, 0, 2869, 3957, 0, 2, 0, 0, 3694, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4057, 0, 0, 418, 4075, 0, 52, 1, 4, 0, 0, 4079, 0, 1, 4047, 0, 103, 90, 0, 0, 0, 5, 1, 0, 1481, 0, 0, 0, 0, 161, 0, 0, 0, 1, 226, 0, 0, 5, 0, 0, 0, 0, 2, 1, 2, 29, 3, 0, 0, 3712, 4085, 0, 1753, 0, 1, 4092, 220, 37, 8, 0, 16, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 2, 3873, 603, 0, 4023, 4092, 0, 7, 0, 0, 0, 0, 1980, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 685, 0, 0, 1, 0, 124, 0, 0, 0, 0, 614, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4013, 0, 430, 0, 0, 3980, 0, 105, 0, 0, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 3902, 1499, 11, 4039, 4038, 0, 1, 5, 1, 0, 0, 1049, 0, 0, 4092, 0, 16, 0, 0, 0, 206, 0, 0, 6, 1123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 676, 0, 140, 0, 0, 0, 0, 19, 7, 0, 0, 3, 4, 0, 0, 3645, 4087, 0, 14, 0, 0, 4083, 0, 2, 0, 0, 100, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 8, 4035, 28, 0, 3912, 4095, 0, 1, 0, 0, 0, 1, 1955, 0, 1, 4066, 0, 8, 0, 2, 0, 54, 0, 0, 18, 0, 0, 0, 7, 0, 780, 0, 0, 0, 0, 3360, 0, 14, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 3844, 2754, 0, 5, 0, 2, 3773, 29, 11, 0, 0, 1, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1714, 889, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2201, 0, 0, 673, 619, 60, 0, 0, 0, 0, 0, 3475, 5, 1, 0, 0, 0, 1531, 239, 0, 0, 0, 0, 0, 0, 0, 0, 201, 259, 3165, 0, 0, 0, 0, 0, 1503, 657, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 1285, 0, 0, 395, 3121, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 42, 3943, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2869, 175, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2664, 3553, 5251, 5785, 4835, 4000, 3046, 3634] avg=4096.0000 max=5785 max_rank=3 min=2664 min_rank=0 imbalance_ratio=1.412354 max_load_reduction=26983 max_load_reduction_pct=0.823456 moved_to_echo=15515 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1658, 1662, 2, 6, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3890, 0, 0, 651, 616, 63, 0, 0, 0, 0, 0, 3457, 0, 0, 3, 0, 0, 442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 2332, 0, 0, 0, 0, 0, 1454, 372, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 1199, 1267, 63, 0, 1768, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 22, 0, 135, 12, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3088, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 3850, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3335, 5220, 3902, 4171, 4317, 4274, 3189, 4360] avg=4096.0000 max=5220 max_rank=1 min=3189 min_rank=6 imbalance_ratio=1.274414 max_load_reduction=27548 max_load_reduction_pct=0.840698 moved_to_echo=16140 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1763, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 34, 3544, 0, 0, 692, 601, 66, 0, 0, 0, 0, 0, 3374, 448, 0, 184, 0, 0, 375, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1053, 0, 0, 0, 0, 1302, 1547, 680, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 173, 5, 0, 2, 3230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4, 4093, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3903, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4030, 0, 634, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1767, 5004, 5079, 4584, 3476, 4104, 4085, 4669] avg=4096.0000 max=5079 max_rank=2 min=1767 min_rank=0 imbalance_ratio=1.239990 max_load_reduction=27689 max_load_reduction_pct=0.845001 moved_to_echo=16334 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1804, 1665, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 1, 0, 2977, 0, 0, 708, 619, 63, 0, 0, 0, 0, 0, 3475, 0, 1, 4, 0, 0, 645, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2310, 0, 0, 0, 0, 645, 1583, 680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 20, 1, 1, 3229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4005, 4, 0, 0, 0, 0, 0, 0, 151, 0, 0, 3956, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 3874, 4, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3477, 4374, 4221, 5218, 3279, 4009, 4288, 3902] avg=4096.0000 max=5218 max_rank=3 min=3279 min_rank=4 imbalance_ratio=1.273926 max_load_reduction=27550 max_load_reduction_pct=0.840759 moved_to_echo=15478 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1787, 0, 0, 418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 1, 4, 0, 0, 4079, 0, 1, 702, 612, 66, 0, 0, 0, 0, 0, 3435, 0, 103, 90, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1481, 0, 0, 0, 0, 161, 1568, 679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 226, 0, 0, 5, 3221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 2, 29, 4075, 4, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3712, 181, 0, 1753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4026, 220, 37, 8, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2205, 5517, 3633, 3890, 3453, 4113, 5649, 4308] avg=4096.0000 max=5649 max_rank=6 min=2205 min_rank=0 imbalance_ratio=1.379150 max_load_reduction=27119 max_load_reduction_pct=0.827606 moved_to_echo=17523 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1719, 1499, 11, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 1, 0, 0, 1049, 0, 0, 675, 619, 66, 0, 0, 0, 0, 0, 3473, 0, 16, 0, 0, 0, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1123, 0, 0, 0, 0, 0, 1508, 679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 676, 0, 140, 0, 3223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 7, 0, 0, 3, 4038, 4, 0, 0, 0, 0, 0, 0, 4, 0, 0, 3645, 181, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4017, 0, 2, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7268, 2416, 3695, 3316, 4039, 4071, 3844, 4119] avg=4096.0000 max=7268 max_rank=0 min=2416 min_rank=1 imbalance_ratio=1.774414 max_load_reduction=25500 max_load_reduction_pct=0.778198 moved_to_echo=16073 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 1706, 603, 0, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 1980, 0, 0, 670, 619, 64, 0, 0, 0, 0, 0, 3477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 685, 0, 0, 1, 0, 124, 1497, 667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 614, 41, 0, 0, 3165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 177, 0, 430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3916, 0, 105, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6334, 3340, 3477, 2974, 3820, 4096, 4704, 4023] avg=4096.0000 max=6334 max_rank=0 min=2974 min_rank=3 imbalance_ratio=1.546387 max_load_reduction=26434 max_load_reduction_pct=0.806702 moved_to_echo=16643 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 1770, 28, 0, 3912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1955, 0, 1, 701, 620, 65, 0, 0, 0, 0, 0, 3446, 0, 8, 0, 2, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 7, 0, 780, 1564, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3360, 0, 14, 0, 2175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 4095, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3844, 114, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3708, 29, 11, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5718, 3344, 3510, 2830, 5549, 4101, 3963, 3753] avg=4096.0000 max=5718 max_rank=0 min=2830 min_rank=3 imbalance_ratio=1.395996 max_load_reduction=27050 max_load_reduction_pct=0.825500 moved_to_echo=17366 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 13952, 6400, 128, 12544, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 384, 12160, 0, 0, 128, 0, 3072, 12288, 4992, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 21760, 0, 128, 5504, 4992, 640, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 7680, 1408, 256, 512, 23168, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 128, 0, 29184, 1408, 0, 2304, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[27648, 512, 256, 384, 128, 0, 3328, 1152, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[640, 31104, 256, 896, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 0, 128, 128, 256, 128, 128, 32512, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1, 12, 0, 0, 10, 1221, 0, 0, 0, 2, 0, 0, 0, 9, 11, 0, 0, 0, 0, 0, 0, 0, 4074, 1, 5, 3970, 1, 3, 0, 0, 291, 15, 2, 0, 0, 4095, 0, 0, 0, 0, 4054, 0, 0, 1, 3278, 3985, 2291, 0, 222, 0, 0, 1, 0, 0, 3160, 0, 25, 0, 0, 0, 1, 0, 46, 1981] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 11, 0, 874, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 133, 0, 0, 2, 0, 549, 1442, 0, 754, 0, 3314, 0, 0, 0, 0, 3350, 0, 0, 0, 3804, 4096, 1581, 0, 632, 0, 35, 0, 0, 0, 3083, 0, 0, 0, 0, 0, 0, 0, 904, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 466, 0, 0, 0, 4, 0, 0, 11, 44, 13, 3, 0, 0, 0, 0, 47, 0, 3766, 53, 0, 4007, 0, 0, 0, 0, 329, 0, 0, 0, 17, 3907, 0, 0, 1, 0, 4053, 0, 0, 0, 4073, 4067, 2818, 0, 152, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 1, 905, 3933] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 10, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 0, 31, 0, 0, 0, 0, 590, 129, 0, 2223, 12, 4096, 0, 0, 0, 0, 3396, 0, 47, 0, 4094, 3813, 1738, 0, 538, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 3908, 4086] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 7, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 3, 0, 1, 692, 383, 0, 4090, 2, 4092, 0, 0, 0, 49, 570, 0, 39, 0, 4037, 4095, 2789, 0, 2, 0, 74, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3728, 3989] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 179, 1, 0, 0, 92, 0, 0, 0, 0, 0, 195, 1, 5, 0, 0, 0, 2, 0, 0, 0, 3942, 0, 0, 68, 0, 2, 7, 4, 245, 1216, 0, 480, 185, 1280, 0, 0, 0, 0, 3782, 0, 0, 0, 3388, 3939, 3650, 0, 24, 0, 0, 0, 0, 0, 1889, 1, 0, 0, 0, 0, 2, 14, 4079, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1, 20, 101, 0, 8, 9, 1, 0, 0, 0, 8, 0, 0, 3, 0, 0, 13, 0, 0, 0, 28, 0, 3968, 1, 0, 2, 0, 7, 42, 0, 22, 472, 1, 4005, 0, 3450, 52, 0, 0, 931, 3140, 0, 52, 0, 4056, 4096, 38, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 0, 5, 0, 0, 4068, 4063] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 6, 4, 0, 0, 360, 0, 0, 0, 0, 0, 0, 0, 7, 0, 1, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 25, 0, 0, 3, 0, 900, 3788, 0, 3350, 10, 3547, 0, 0, 0, 1, 4089, 0, 32, 0, 3185, 4095, 62, 0, 6, 0, 0, 0, 0, 0, 245, 0, 0, 2, 0, 0, 0, 0, 909, 4047] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 12, 0, 0, 10, 1221, 0, 0, 3215, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 9, 11, 0, 3985, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 1, 87, 0, 0, 0, 0, 0, 0, 0, 5, 3970, 1, 3, 0, 0, 291, 15, 1262, 760, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2705, 0, 0, 1, 0, 0, 2291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 222, 0, 0, 1, 0, 0, 3160, 0, 1060, 887, 33, 0, 0, 0, 0, 0, 25, 0, 0, 0, 1, 0, 46, 921, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4459, 4037, 4162, 6307, 2450, 4997, 5363, 993] avg=4096.0000 max=6307 max_rank=3 min=993 min_rank=7 imbalance_ratio=1.539795 max_load_reduction=26461 max_load_reduction_pct=0.807526 moved_to_echo=13803 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 179, 1, 0, 0, 92, 0, 3323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 195, 1, 5, 0, 3939, 30, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3942, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 2, 7, 4, 245, 1216, 1177, 237, 0, 0, 0, 0, 0, 0, 0, 480, 185, 766, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2524, 0, 0, 0, 0, 0, 3650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 1889, 1, 2192, 277, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 14, 4079, 1904, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3595, 4170, 4025, 2956, 1431, 6174, 4418, 5999] avg=4096.0000 max=6174 max_rank=5 min=1431 min_rank=4 imbalance_ratio=1.507324 max_load_reduction=26594 max_load_reduction_pct=0.811584 moved_to_echo=18022 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 10, 0, 0, 4017, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3813, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 590, 129, 1057, 760, 0, 0, 0, 0, 0, 0, 0, 2223, 12, 2449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2266, 0, 47, 0, 0, 0, 1738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 0, 0, 0, 0, 0, 12, 0, 2187, 887, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3908, 1899, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4027, 3850, 4117, 2567, 4684, 4051, 3665, 5807] avg=4096.0000 max=5807 max_rank=7 min=2567 min_rank=3 imbalance_ratio=1.417725 max_load_reduction=26961 max_load_reduction_pct=0.822784 moved_to_echo=18207 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 466, 0, 0, 3995, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 11, 44, 13, 3, 4067, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 3766, 53, 87, 0, 0, 0, 0, 0, 0, 0, 0, 4007, 0, 0, 0, 0, 329, 0, 1262, 725, 0, 0, 0, 0, 0, 0, 0, 0, 17, 2336, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2704, 0, 0, 0, 0, 0, 2818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 0, 0, 0, 0, 0, 0, 0, 2105, 846, 41, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 1, 905, 1828, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4463, 4179, 3953, 6323, 2354, 5522, 3144, 2830] avg=4096.0000 max=6323 max_rank=3 min=2354 min_rank=4 imbalance_ratio=1.543701 max_load_reduction=26445 max_load_reduction_pct=0.807037 moved_to_echo=13850 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 7, 12, 0, 0, 3960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 692, 383, 177, 759, 0, 0, 0, 0, 0, 0, 0, 4090, 2, 2446, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 381, 0, 39, 0, 0, 0, 2789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 74, 44, 0, 0, 0, 0, 2135, 887, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3728, 1854, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3982, 4130, 4079, 2015, 6587, 3209, 3184, 5582] avg=4096.0000 max=6587 max_rank=4 min=2015 min_rank=3 imbalance_ratio=1.608154 max_load_reduction=26181 max_load_reduction_pct=0.798981 moved_to_echo=18562 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 20, 101, 0, 8, 9, 1, 0, 3980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 3, 0, 0, 4096, 35, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 28, 0, 3968, 1, 67, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7, 42, 0, 22, 472, 977, 640, 0, 0, 0, 0, 0, 0, 1, 4005, 0, 2063, 52, 0, 0, 931, 0, 0, 0, 0, 0, 0, 0, 0, 2096, 0, 52, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 2175, 747, 41, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4068, 1888, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4142, 4077, 2162, 7052, 2186, 3068, 5961] avg=4096.0000 max=7052 max_rank=4 min=2162 min_rank=3 imbalance_ratio=1.721680 max_load_reduction=25716 max_load_reduction_pct=0.784790 moved_to_echo=18267 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 11, 0, 874, 11, 0, 0, 3731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 72, 0, 0, 0, 0, 0, 0, 0, 0, 133, 0, 0, 2, 0, 549, 1442, 1043, 615, 0, 0, 0, 0, 0, 0, 0, 754, 0, 1981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2235, 0, 0, 0, 0, 0, 1581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 632, 0, 35, 0, 0, 0, 3083, 0, 2192, 718, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 904, 1904, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4627, 4129, 4169, 3784, 2735, 3816, 6700, 2808] avg=4096.0000 max=6700 max_rank=6 min=2735 min_rank=4 imbalance_ratio=1.635742 max_load_reduction=26068 max_load_reduction_pct=0.795532 moved_to_echo=16059 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 4, 0, 0, 360, 0, 0, 3125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 1, 4095, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 3, 0, 900, 3788, 1278, 660, 0, 0, 0, 0, 0, 0, 0, 3350, 10, 2114, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2719, 0, 32, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 245, 0, 2170, 773, 32, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 909, 1877, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3495, 4131, 4186, 6654, 5475, 2813, 3226, 2788] avg=4096.0000 max=6654 max_rank=3 min=2788 min_rank=7 imbalance_ratio=1.624512 max_load_reduction=26114 max_load_reduction_pct=0.796936 moved_to_echo=14302 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[128, 8320, 128, 128, 128, 128, 3712, 7552, 8320, 5248, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[17664, 0, 256, 128, 0, 0, 14976, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1664, 0, 128, 128, 0, 0, 8576, 128, 16256, 6144, 384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 128, 128, 18560, 14080, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 14976, 256, 16640, 128, 0, 128, 1024, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 384, 128, 1024, 2176, 128, 0, 29440, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 256, 128, 128, 128, 32256, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 0, 128, 0, 32128, 128, 640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33792 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 4095, 4078, 29, 1490, 0, 0, 4, 0, 4096, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 604, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 402, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 449, 1041, 0, 0, 4091, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1, 3481, 4089, 0, 0, 8, 0, 0, 0, 4029, 8, 3198, 0, 0, 0, 0, 0, 1, 12, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 41, 0, 0, 4054, 3513, 14, 217, 0, 0, 0, 0, 3761, 2, 0, 0, 0, 0, 0, 774, 30, 0, 1483, 5, 0, 0, 0, 0, 0, 0, 84, 0, 0, 3904, 51, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[32, 4086, 4094, 20, 555, 0, 0, 5, 0, 4094, 0, 175, 2, 0, 0, 0, 2, 0, 0, 1, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4033, 3321, 0, 12, 0, 46, 0, 0, 4087, 0, 0, 0, 15, 0, 0, 2173, 5, 0, 1857, 0, 0, 0, 0, 0, 0, 0, 0, 2, 49, 4087, 0, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 4089, 3146, 0, 11, 0, 0, 0, 0, 4096, 0, 859, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 639, 0, 1, 0, 0, 3, 0, 3895, 0, 0, 0, 0, 0, 1032, 3495, 0, 0, 3283, 0, 0, 0, 0, 1, 5, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[3, 4062, 1820, 0, 388, 0, 0, 0, 0, 4096, 0, 21, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4087, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1886, 2635, 0, 0, 1270, 0, 0, 0, 0, 209, 0, 1, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 3995, 4082, 0, 3999, 0, 0, 54, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 2, 101, 3, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4080, 0, 0, 19, 0, 0, 0, 4096, 0, 0, 0, 9, 0, 3, 12, 0, 0, 69, 0, 2, 0, 1, 1, 0, 8, 0, 0, 0, 4041, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 4068, 3317, 383, 0, 1, 0, 0, 1, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 45, 0, 10, 1, 0, 0, 0, 0, 0, 0, 4042, 4068, 0, 0, 0, 139, 39, 0, 3883, 0, 0, 0, 1269, 0, 8, 0, 0, 0, 4032, 0, 25, 0, 0, 0, 3, 0, 0, 2, 0, 3045, 0, 291] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 4004, 4096, 137, 1556, 1, 0, 0, 0, 4095, 0, 390, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4, 0, 19, 0, 4094, 872, 0, 0, 0, 0, 0, 0, 4053, 0, 1, 1, 0, 1, 2, 3172, 0, 0, 2411, 0, 0, 0, 0, 0, 0, 0, 1, 0, 98, 3754, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3425, 0, 3999, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 101, 3, 0, 0, 0, 2576, 1465, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3995, 100, 0, 0, 0, 0, 0, 0, 1420, 4080, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1896, 0, 0, 0, 9, 0, 3, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 2, 0, 1, 1, 735, 584, 508, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 4041, 0, 0, 149, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7478, 3510, 4147, 4096, 5519, 1920, 1900, 4198] avg=4096.0000 max=7478 max_rank=0 min=1900 min_rank=6 imbalance_ratio=1.825684 max_load_reduction=25290 max_load_reduction_pct=0.771790 moved_to_echo=13537 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3421, 29, 1490, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3512, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 604, 0, 2576, 1465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4095, 100, 0, 0, 0, 0, 0, 0, 1420, 402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1896, 0, 0, 0, 2, 0, 449, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 1, 735, 584, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 149, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4944, 3608, 4645, 4198, 1822, 3388, 5919, 4244] avg=4096.0000 max=5919 max_rank=6 min=1822 min_rank=4 imbalance_ratio=1.445068 max_load_reduction=26849 max_load_reduction_pct=0.819366 moved_to_echo=15373 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 3431, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3454, 8, 3198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 12, 0, 0, 0, 8, 0, 2549, 1345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 3481, 99, 0, 0, 0, 0, 0, 0, 1406, 3513, 14, 217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1741, 2, 0, 0, 0, 0, 0, 774, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 1483, 5, 0, 0, 0, 0, 675, 575, 509, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 3904, 51, 0, 149, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3440, 6660, 3915, 3621, 5150, 2517, 3277, 4188] avg=4096.0000 max=6660 max_rank=1 min=2517 min_rank=5 imbalance_ratio=1.625977 max_load_reduction=26108 max_load_reduction_pct=0.796753 moved_to_echo=15132 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[32, 0, 3435, 20, 555, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3510, 0, 175, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 4, 0, 2536, 1462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4086, 99, 0, 0, 0, 0, 0, 0, 1398, 3321, 0, 12, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1891, 0, 0, 0, 15, 0, 0, 2173, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 1857, 0, 0, 0, 0, 0, 734, 584, 510, 0, 0, 0, 0, 0, 0, 0, 0, 2, 49, 4087, 0, 5, 149, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4047, 3687, 4005, 4191, 4777, 4079, 3690, 4292] avg=4096.0000 max=4777 max_rank=4 min=3687 min_rank=1 imbalance_ratio=1.166260 max_load_reduction=27991 max_load_reduction_pct=0.854218 moved_to_echo=16838 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2639, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3512, 0, 859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 9, 0, 2576, 1393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 100, 0, 0, 0, 0, 0, 0, 1420, 639, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1803, 0, 0, 0, 0, 0, 1032, 3495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3283, 0, 0, 0, 0, 1, 699, 584, 392, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4096, 0, 0, 115, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2650, 4371, 3990, 4189, 2063, 6330, 4959, 4216] avg=4096.0000 max=6330 max_rank=5 min=2063 min_rank=4 imbalance_ratio=1.545410 max_load_reduction=26438 max_load_reduction_pct=0.806824 moved_to_echo=17568 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2783, 383, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 45, 0, 2542, 1389, 0, 0, 0, 0, 0, 0, 10, 1, 0, 0, 0, 0, 0, 0, 4068, 99, 0, 0, 0, 0, 0, 0, 1401, 4068, 0, 0, 0, 139, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1797, 0, 0, 0, 1269, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 0, 25, 0, 0, 0, 697, 582, 413, 0, 0, 0, 0, 0, 3, 0, 0, 2, 0, 3045, 0, 291, 121, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3167, 3500, 3991, 4178, 5647, 3074, 5749, 3462] avg=4096.0000 max=5749 max_rank=6 min=3074 min_rank=5 imbalance_ratio=1.403564 max_load_reduction=27019 max_load_reduction_pct=0.824554 moved_to_echo=17932 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 1528, 0, 388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3512, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 2573, 1465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 100, 0, 0, 0, 0, 0, 0, 1419, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1896, 0, 0, 0, 0, 0, 1886, 2635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1270, 0, 0, 0, 0, 209, 735, 584, 226, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1919, 3533, 4044, 4162, 5506, 6417, 3024, 4163] avg=4096.0000 max=6417 max_rank=5 min=1919 min_rank=0 imbalance_ratio=1.566650 max_load_reduction=26351 max_load_reduction_pct=0.804169 moved_to_echo=19110 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3429, 137, 1556, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3505, 0, 390, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2578, 1453, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4, 0, 19, 0, 4004, 104, 0, 0, 0, 0, 0, 0, 1412, 872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1866, 0, 1, 1, 0, 1, 2, 3172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2411, 0, 0, 0, 0, 0, 734, 590, 515, 0, 0, 0, 0, 0, 0, 0, 1, 0, 98, 3754, 0, 0, 152, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5123, 3899, 4031, 4133, 2284, 5043, 4250, 4005] avg=4096.0000 max=5123 max_rank=0 min=2284 min_rank=4 imbalance_ratio=1.250732 max_load_reduction=27645 max_load_reduction_pct=0.843658 moved_to_echo=15582 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 256, 128, 0, 768, 0, 20608, 11520, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[11392, 20992, 128, 256, 128, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 18560, 128, 128, 0, 128, 256, 5760, 4736, 3584, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 24192, 640, 8064, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[14848, 128, 128, 128, 1408, 128, 3456, 13312, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 28032, 128, 4864, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 0, 0, 128, 128, 128, 0, 32000, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 256, 31232, 128, 384, 1152, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[4087, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 2, 25, 0, 0, 0, 14, 0, 0, 0, 0, 579, 2844, 0, 0, 0, 0, 0, 0, 0, 4096, 26, 85, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4056, 0, 0, 4096, 0, 0, 0, 0, 2586, 4, 0, 5, 0, 0, 0, 0, 0, 0, 2074, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[2203, 0, 0, 0, 3676, 0, 0, 0, 0, 0, 0, 0, 0, 261, 0, 0, 0, 122, 0, 0, 0, 291, 4092, 0, 0, 30, 0, 0, 20, 9, 4026, 0, 3662, 2, 0, 0, 2, 0, 15, 0, 1, 0, 3964, 444, 46, 2602, 0, 0, 0, 0, 408, 0, 0, 2, 427, 1, 62, 0, 0, 8, 2606, 0, 3, 3783] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[3143, 0, 43, 0, 4089, 0, 0, 9, 0, 46, 0, 3, 173, 300, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 3680, 0, 0, 0, 403, 0, 0, 0, 0, 0, 623, 0, 3921, 0, 1, 3800, 0, 0, 0, 0, 4069, 0, 3, 0, 0, 2, 26, 188, 0, 0, 91, 0, 0, 4059] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[3353, 0, 0, 0, 4015, 92, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 0, 29, 3127, 0, 0, 0, 131, 0, 0, 0, 4089, 0, 1030, 0, 1159, 0, 0, 0, 0, 0, 16, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1055, 0, 0, 0, 0, 0, 0, 11, 0, 0, 2326, 0, 0, 4040] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4031, 0, 0, 0, 3054, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 323, 4094, 0, 0, 0, 158, 0, 0, 0, 4096, 4, 10, 0, 0, 0, 0, 0, 0, 0, 3604, 0, 4086, 0, 0, 557, 0, 0, 0, 0, 4093, 36, 0, 1, 0, 0, 0, 0, 0, 0, 518, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[4095, 6, 6, 0, 3991, 0, 0, 0, 1, 0, 0, 4, 3, 0, 0, 0, 0, 0, 32, 0, 1, 0, 4091, 0, 0, 5, 0, 0, 0, 1, 4073, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 4070, 0, 0, 0, 0, 4093, 0, 0, 0, 2, 0, 0, 108, 0, 2, 0, 0, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[3299, 4, 0, 0, 3645, 0, 0, 5, 0, 0, 4, 0, 0, 0, 10, 0, 613, 0, 0, 0, 0, 0, 3569, 0, 0, 0, 0, 0, 1, 0, 4090, 0, 255, 0, 0, 0, 0, 0, 0, 0, 22, 0, 4086, 3, 0, 4034, 0, 0, 0, 0, 2419, 0, 1, 28, 35, 0, 0, 516, 3, 0, 2027, 5, 0, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[4068, 0, 0, 0, 3049, 0, 0, 36, 0, 4, 0, 1, 10, 52, 0, 0, 671, 1, 0, 0, 0, 1019, 4096, 0, 0, 0, 0, 0, 1, 0, 3912, 1, 266, 0, 0, 40, 0, 0, 0, 0, 12, 0, 3951, 0, 8, 2836, 0, 0, 0, 0, 2838, 0, 0, 0, 638, 0, 10, 969, 3, 0, 189, 0, 0, 4087] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4087, 0, 0, 0, 593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 25, 0, 0, 0, 3817, 206, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 579, 2747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 26, 30, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 3290, 254, 0, 0, 0, 0, 0, 0, 4, 0, 239, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2586, 4, 0, 5, 0, 0, 1179, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2074, 0, 0, 2663, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4680, 4050, 3340, 4152, 3629, 4339, 3841, 4737] avg=4096.0000 max=4737 max_rank=7 min=3340 min_rank=2 imbalance_ratio=1.156494 max_load_reduction=28031 max_load_reduction_pct=0.855438 moved_to_echo=16546 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2203, 0, 0, 0, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 261, 0, 0, 3731, 185, 0, 0, 0, 0, 0, 0, 0, 122, 0, 0, 0, 291, 3953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 20, 9, 4026, 0, 43, 0, 0, 0, 0, 0, 0, 0, 3662, 2, 0, 0, 2, 0, 15, 0, 2957, 235, 0, 0, 0, 0, 0, 0, 1, 0, 233, 444, 46, 2602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 408, 0, 0, 2, 427, 1, 1089, 96, 0, 0, 0, 0, 0, 0, 62, 0, 0, 8, 2606, 0, 3, 2459, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2737, 4177, 4366, 4128, 6873, 3326, 2023, 5138] avg=4096.0000 max=6873 max_rank=4 min=2023 min_rank=6 imbalance_ratio=1.677979 max_load_reduction=25895 max_load_reduction_pct=0.790253 moved_to_echo=17360 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3143, 0, 43, 0, 593, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 3, 173, 300, 0, 0, 3690, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3680, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 403, 0, 0, 0, 0, 0, 3290, 252, 0, 0, 0, 0, 0, 0, 623, 0, 231, 0, 1, 3800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 3, 0, 0, 2, 1168, 96, 0, 0, 0, 0, 0, 0, 26, 188, 0, 0, 91, 0, 0, 2639, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3788, 4418, 3957, 3723, 3945, 4655, 5338, 2944] avg=4096.0000 max=5338 max_rank=6 min=2944 min_rank=7 imbalance_ratio=1.303223 max_load_reduction=27430 max_load_reduction_pct=0.837097 moved_to_echo=16882 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4068, 0, 0, 0, 442, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 10, 52, 0, 0, 3719, 154, 0, 0, 0, 0, 0, 0, 671, 1, 0, 0, 0, 1019, 3957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3912, 1, 43, 0, 0, 0, 0, 0, 0, 0, 266, 0, 0, 40, 0, 0, 0, 0, 2453, 254, 0, 0, 0, 0, 0, 0, 12, 0, 232, 0, 8, 2836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2838, 0, 0, 0, 638, 0, 1176, 96, 0, 0, 0, 0, 0, 0, 10, 969, 3, 0, 189, 0, 0, 2657, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4546, 3940, 5648, 3957, 3013, 3088, 4748, 3828] avg=4096.0000 max=5648 max_rank=2 min=3013 min_rank=4 imbalance_ratio=1.378906 max_load_reduction=27120 max_load_reduction_pct=0.827637 moved_to_echo=14677 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3353, 0, 0, 0, 582, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 3855, 203, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 0, 29, 3021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 0, 0, 0, 4089, 0, 33, 0, 0, 0, 0, 0, 0, 0, 1030, 0, 1159, 0, 0, 0, 0, 0, 3230, 251, 0, 0, 0, 0, 0, 0, 16, 0, 241, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 0, 0, 0, 0, 0, 1163, 73, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 2326, 0, 0, 2626, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4027, 4070, 3141, 4253, 5670, 4353, 2291, 4963] avg=4096.0000 max=5670 max_rank=4 min=2291 min_rank=6 imbalance_ratio=1.384277 max_load_reduction=27098 max_load_reduction_pct=0.826965 moved_to_echo=17277 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4031, 0, 0, 0, 443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 3846, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 323, 3955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 0, 0, 0, 4096, 4, 43, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 2457, 254, 0, 0, 0, 0, 0, 0, 3604, 0, 240, 0, 0, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 36, 0, 1, 0, 0, 1179, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 0, 0, 2663, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4474, 4007, 4278, 4301, 2721, 4401, 5405, 3181] avg=4096.0000 max=5405 max_rank=6 min=2721 min_rank=4 imbalance_ratio=1.319580 max_load_reduction=27363 max_load_reduction_pct=0.835052 moved_to_echo=15708 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3299, 4, 0, 0, 529, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 10, 0, 3846, 184, 0, 0, 0, 0, 0, 0, 613, 0, 0, 0, 0, 0, 3448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4090, 0, 37, 0, 0, 0, 0, 0, 0, 0, 255, 0, 0, 0, 0, 0, 0, 0, 2932, 254, 0, 0, 0, 0, 0, 0, 22, 0, 240, 3, 0, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2419, 0, 1, 28, 35, 0, 1178, 84, 0, 0, 0, 0, 0, 0, 0, 516, 3, 0, 2027, 5, 0, 2662, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3837, 4044, 4061, 4128, 3441, 4299, 3745, 5213] avg=4096.0000 max=5213 max_rank=7 min=3441 min_rank=4 imbalance_ratio=1.272705 max_load_reduction=27555 max_load_reduction_pct=0.840912 moved_to_echo=16698 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4095, 6, 6, 0, 572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 3, 0, 0, 0, 3849, 205, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 1, 0, 3944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 1, 4073, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 3214, 259, 0, 0, 0, 0, 0, 0, 0, 0, 237, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 2, 0, 1182, 100, 0, 0, 0, 0, 0, 0, 0, 108, 0, 2, 0, 0, 0, 2654, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4679, 4062, 3977, 4126, 3476, 4307, 5377, 2764] avg=4096.0000 max=5377 max_rank=6 min=2764 min_rank=7 imbalance_ratio=1.312744 max_load_reduction=27391 max_load_reduction_pct=0.835907 moved_to_echo=15924 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[5376, 128, 1664, 128, 128, 0, 128, 0, 23936, 2048, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 0, 21632, 128, 128, 128, 1152, 128, 9344, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 1792, 128, 128, 9856, 128, 128, 21120, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[28288, 128, 128, 0, 4352, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 256, 640, 128, 0, 30464, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4352, 0, 1920, 512, 128, 26112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1408, 128, 128, 0, 128, 2304, 29056, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 384, 0, 128, 128, 32128, 128, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[3629, 3840, 0, 4003, 0, 0, 0, 0, 4095, 0, 49, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 35, 0, 0, 23, 0, 4081, 0, 0, 37, 0, 0, 0, 0, 6, 63, 0, 0, 0, 4067, 277, 4030, 1, 0, 0, 60, 367, 0, 0, 0, 0, 2, 4, 0, 0, 0, 0, 2, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[3979, 3782, 0, 3695, 13, 0, 0, 0, 4080, 0, 334, 2, 3, 23, 251, 1, 0, 520, 0, 0, 2, 172, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 692, 0, 16, 0, 147, 0, 0, 0, 0, 0, 3, 0, 2, 4017, 4054, 2804, 0, 4, 0, 0, 0, 0, 5, 0, 0, 0, 0, 24, 0, 0, 51, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[4096, 3986, 401, 4096, 0, 0, 0, 0, 4074, 0, 0, 26, 0, 0, 108, 0, 0, 15, 0, 0, 2, 0, 0, 1262, 0, 0, 0, 5, 0, 0, 0, 3418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3739, 3439, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[4096, 3972, 0, 4055, 10, 0, 0, 0, 4095, 0, 0, 99, 0, 37, 1828, 0, 0, 0, 0, 0, 0, 0, 764, 407, 0, 0, 0, 219, 0, 2206, 2956, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 31, 4, 0, 4047, 1, 3870, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 58] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[4089, 4019, 87, 4094, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 14, 2, 0, 0, 0, 0, 0, 0, 0, 3847, 0, 0, 0, 37, 0, 1344, 0, 3150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 5, 3887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[548, 4096, 5, 3417, 0, 0, 0, 0, 4096, 0, 0, 2, 14, 0, 3907, 11, 0, 0, 0, 0, 0, 0, 493, 407, 0, 0, 0, 13, 0, 1, 4087, 0, 1526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3872, 1209, 4096, 0, 684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 284] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[4018, 2936, 127, 4095, 0, 0, 0, 1, 4076, 0, 7, 56, 0, 0, 4074, 0, 0, 0, 0, 0, 2, 0, 0, 1749, 0, 1, 0, 1126, 0, 1, 0, 544, 0, 0, 6, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 2142, 3704, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4089, 3671, 0, 4077, 0, 0, 0, 0, 4034, 1, 122, 0, 0, 1, 6, 0, 0, 12, 0, 0, 4, 0, 439, 4065, 0, 0, 22, 37, 5, 0, 25, 2429, 0, 0, 0, 5, 0, 0, 1, 0, 1, 0, 0, 0, 4092, 2116, 3506, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3629, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2734, 0, 49, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 817, 536, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 23, 0, 4081, 866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 6, 3380, 544, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 277, 2628, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 367, 0, 0, 0, 0, 4003, 6, 6, 0, 0, 0, 0, 0, 2, 4, 0, 0, 0, 0, 2, 7, 4061, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 2785, 5441, 5005, 3967, 2969, 4442, 4076] avg=4096.0000 max=5441 max_rank=2 min=2785 min_rank=1 imbalance_ratio=1.328369 max_load_reduction=27327 max_load_reduction_pct=0.833954 moved_to_echo=15454 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3979, 447, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2724, 0, 334, 2, 3, 23, 251, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 0, 0, 2, 172, 0, 4092, 814, 373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 692, 603, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 147, 0, 0, 0, 0, 3329, 542, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 4054, 1828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 5, 0, 0, 3695, 6, 6, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 51, 0, 0, 4011, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4439, 3338, 5973, 1295, 4034, 5887, 3716, 4086] avg=4096.0000 max=5973 max_rank=2 min=1295 min_rank=3 imbalance_ratio=1.458252 max_load_reduction=26795 max_load_reduction_pct=0.817719 moved_to_echo=17723 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4096, 472, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2720, 0, 0, 26, 0, 0, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 2, 0, 0, 1262, 813, 457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 3418, 739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3508, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3739, 2243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4969, 2854, 2549, 4162, 4049, 5982, 4114, 4089] avg=4096.0000 max=5982 max_rank=5 min=2549 min_rank=2 imbalance_ratio=1.460449 max_load_reduction=26786 max_load_reduction_pct=0.817444 moved_to_echo=18234 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[548, 485, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2734, 0, 0, 2, 14, 0, 3907, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 407, 818, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 1, 4087, 0, 881, 0, 0, 0, 0, 0, 0, 0, 1526, 0, 0, 0, 0, 0, 0, 0, 3605, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1209, 2670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 684, 0, 0, 0, 0, 0, 0, 0, 3417, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 284, 3867, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1038, 6668, 2263, 4982, 5675, 3879, 4112, 4151] avg=4096.0000 max=6668 max_rank=1 min=1038 min_rank=0 imbalance_ratio=1.627930 max_load_reduction=26100 max_load_reduction_pct=0.796509 moved_to_echo=17817 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4096, 470, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2734, 0, 0, 99, 0, 37, 1828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 764, 407, 817, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 219, 0, 2206, 2956, 0, 832, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 3496, 544, 0, 0, 0, 0, 0, 0, 0, 31, 4, 0, 0, 1, 2523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 4055, 6, 5, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 58, 4042, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4576, 4698, 2503, 6213, 4048, 2559, 4070, 4101] avg=4096.0000 max=6213 max_rank=3 min=2503 min_rank=2 imbalance_ratio=1.516846 max_load_reduction=26555 max_load_reduction_pct=0.810394 moved_to_echo=14778 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4018, 348, 127, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2720, 0, 7, 56, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1749, 814, 493, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1126, 0, 1, 0, 544, 796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 1, 0, 2584, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2142, 2415, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4095, 4, 6, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4494, 6857, 3058, 2468, 3133, 4558, 4108, 4092] avg=4096.0000 max=6857 max_rank=1 min=2468 min_rank=3 imbalance_ratio=1.674072 max_load_reduction=25911 max_load_reduction_pct=0.790741 moved_to_echo=15891 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4089, 476, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2734, 0, 4, 0, 0, 0, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3847, 818, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 1344, 0, 3150, 836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3537, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4652, 2754, 5182, 5367, 4081, 2539, 4106, 4087] avg=4096.0000 max=5367 max_rank=3 min=2539 min_rank=5 imbalance_ratio=1.310303 max_load_reduction=27401 max_load_reduction_pct=0.836212 moved_to_echo=14813 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4089, 428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2684, 1, 122, 0, 0, 1, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4, 0, 439, 4065, 809, 470, 0, 0, 0, 0, 0, 0, 0, 0, 22, 37, 5, 0, 25, 2429, 758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 1, 0, 3234, 541, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2116, 2278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4077, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4517, 2814, 5799, 3276, 3781, 4395, 4100, 4086] avg=4096.0000 max=5799 max_rank=2 min=2814 min_rank=1 imbalance_ratio=1.415771 max_load_reduction=26969 max_load_reduction_pct=0.823029 moved_to_echo=16362 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 640, 0, 0, 128, 256, 1792, 19968, 6528, 3968, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[21888, 128, 640, 256, 128, 128, 10240, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 1536, 128, 3584, 7168, 14336, 6400, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 0, 13568, 19200, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[28544, 3584, 640, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1536, 128, 128, 256, 0, 0, 128, 128, 26752, 4352, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[768, 0, 128, 384, 0, 128, 0, 0, 31616, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 0, 128, 128, 384, 32384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 760, 2531, 0, 0, 0, 0, 0, 0, 0, 155, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 49, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 930, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 3887, 0, 0, 3985] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 14, 4096, 0, 0, 513, 0, 0, 0, 0, 55, 0, 0, 182, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4035, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 2, 0, 0, 5, 0, 0, 0, 0, 2918, 4, 656, 0, 1, 0, 5, 0, 0, 3901, 0, 4090, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 68, 56, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 165, 0, 0, 0, 0, 4063, 110, 102, 0, 5, 0, 0, 0, 0, 3842, 0, 3873, 0, 0, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 5, 7, 0, 0, 3299, 0, 3657, 34, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 98, 0, 4082, 0, 0, 0, 0, 4, 4092, 0, 0, 1, 0, 0, 1866, 0, 4078, 3, 0, 0, 0, 0, 6, 39, 0, 0, 0, 0, 0, 3706, 0, 2, 0, 2, 0, 0, 0, 0, 3696, 0, 1, 0, 0, 4086] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4086, 0, 1732, 0, 0, 0, 0, 0, 0, 73, 0, 121, 0, 0, 3, 181, 3959, 0, 4, 8, 0, 0, 4039, 3, 0, 7, 0, 0, 35, 2, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3326, 2, 0, 0, 0, 0, 7, 31, 0, 4046, 0, 2985, 0, 20, 4006] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[18, 0, 0, 0, 0, 0, 3357, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 0, 0, 0, 5, 0, 4093, 0, 3, 0, 0, 0, 746, 0, 4086, 0, 3, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4046, 0, 0, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4053, 0, 3435, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 555, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 884, 0, 0, 0, 2, 0, 0, 0, 0, 3647, 0, 3778, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 1, 4085, 0, 976, 183, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 10, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2937, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 14, 4096, 0, 131, 63, 0, 0, 0, 0, 0, 0, 0, 513, 0, 0, 0, 0, 55, 0, 1807, 0, 0, 0, 0, 0, 0, 0, 0, 182, 0, 0, 0, 0, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 5, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2918, 4, 656, 0, 1, 0, 5, 1209, 273, 0, 0, 0, 0, 0, 0, 0, 0, 885, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4304, 2375, 4205, 4045, 3695, 4103, 5066, 4975] avg=4096.0000 max=5066 max_rank=6 min=2375 min_rank=1 imbalance_ratio=1.236816 max_load_reduction=27702 max_load_reduction_pct=0.845398 moved_to_echo=17839 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 131, 63, 0, 0, 0, 0, 0, 0, 760, 2531, 0, 0, 0, 0, 0, 0, 1893, 0, 0, 0, 0, 0, 0, 0, 0, 155, 0, 0, 0, 0, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 3692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3985, 0, 0, 0, 0, 0, 0, 0, 0, 930, 0, 0, 0, 0, 0, 0, 1267, 273, 0, 0, 0, 0, 0, 0, 0, 0, 927, 0, 3887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4290, 5184, 4178, 4106, 3741, 3985, 2470, 4814] avg=4096.0000 max=5184 max_rank=1 min=2470 min_rank=6 imbalance_ratio=1.265625 max_load_reduction=27584 max_load_reduction_pct=0.841797 moved_to_echo=15010 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 131, 63, 0, 0, 0, 0, 0, 0, 68, 56, 10, 0, 0, 0, 0, 0, 1779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 110, 102, 0, 5, 0, 0, 1191, 272, 0, 0, 0, 0, 0, 0, 0, 0, 872, 0, 3873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4290, 1913, 4021, 4106, 3692, 4258, 5743, 4745] avg=4096.0000 max=5743 max_rank=6 min=1913 min_rank=1 imbalance_ratio=1.402100 max_load_reduction=27025 max_load_reduction_pct=0.824738 moved_to_echo=18438 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 5, 7, 0, 0, 3299, 0, 130, 63, 0, 0, 0, 0, 0, 0, 3657, 34, 0, 0, 0, 0, 0, 4, 1712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 4009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4092, 0, 0, 1, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1866, 0, 3677, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 39, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 3706, 0, 2, 0, 2, 0, 0, 1145, 271, 0, 0, 0, 0, 0, 0, 0, 0, 839, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3504, 5407, 4107, 4107, 5546, 4131, 5126, 840] avg=4096.0000 max=5546 max_rank=4 min=840 min_rank=7 imbalance_ratio=1.354004 max_load_reduction=27222 max_load_reduction_pct=0.830750 moved_to_echo=15643 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4086, 0, 131, 61, 0, 0, 0, 0, 0, 0, 1732, 0, 0, 0, 0, 0, 0, 73, 1874, 1, 0, 0, 0, 0, 0, 0, 0, 121, 0, 0, 3, 181, 3888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8, 0, 0, 4039, 3, 0, 7, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 2, 3689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4005, 0, 0, 0, 0, 0, 0, 0, 0, 3326, 2, 0, 0, 0, 0, 7, 1254, 272, 0, 0, 0, 0, 0, 0, 31, 0, 918, 0, 2985, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4278, 3680, 4193, 4071, 3726, 4005, 4861, 3954] avg=4096.0000 max=4861 max_rank=6 min=3680 min_rank=1 imbalance_ratio=1.186768 max_load_reduction=27907 max_load_reduction_pct=0.851654 moved_to_echo=16546 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[18, 0, 0, 0, 0, 0, 3357, 0, 131, 63, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 1894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4093, 0, 3, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 746, 0, 3683, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 1267, 272, 0, 0, 0, 0, 0, 0, 0, 0, 929, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3569, 5993, 4022, 4111, 4432, 4095, 1571, 4975] avg=4096.0000 max=5993 max_rank=1 min=1571 min_rank=6 imbalance_ratio=1.463135 max_load_reduction=26775 max_load_reduction_pct=0.817108 moved_to_echo=15073 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4053, 0, 131, 63, 0, 0, 0, 0, 0, 0, 3435, 30, 0, 0, 0, 0, 0, 0, 1689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 0, 3691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 884, 0, 0, 0, 2, 0, 0, 1130, 272, 0, 0, 0, 0, 0, 0, 0, 0, 828, 0, 3778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4247, 5154, 4025, 4106, 4246, 4096, 2288, 4606] avg=4096.0000 max=5154 max_rank=1 min=2288 min_rank=6 imbalance_ratio=1.258301 max_load_reduction=27614 max_load_reduction_pct=0.842712 moved_to_echo=15236 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 1, 4085, 0, 134, 65, 0, 0, 0, 0, 0, 0, 976, 183, 2, 0, 0, 0, 0, 0, 1900, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4095, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 1273, 277, 0, 0, 0, 0, 0, 0, 0, 0, 922, 0, 2937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4286, 3062, 4017, 4116, 3690, 4095, 5643, 3859] avg=4096.0000 max=5643 max_rank=6 min=3062 min_rank=1 imbalance_ratio=1.377686 max_load_reduction=27125 max_load_reduction_pct=0.827789 moved_to_echo=17287 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[10752, 7552, 128, 128, 0, 0, 128, 128, 14592, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 19968, 128, 768, 0, 128, 0, 128, 9856, 2304, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 7168, 0, 25600, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 3328, 128, 29568, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 512, 128, 128, 128, 256, 32128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 128, 128, 31232, 0, 1152, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 256, 0, 0, 0, 32640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33792 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 3, 3995, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4092, 0, 1934, 0, 0, 4096, 2971, 0, 0, 0, 0, 0, 0, 0, 1, 3401, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4070, 0, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 10, 0, 0, 0, 13, 0, 1, 0, 150, 4050, 0, 2, 0, 0, 0, 1, 0, 4015, 0, 13, 4095, 0, 4084, 2, 14, 4086, 158, 0, 0, 0, 54, 0, 1, 0, 0, 3374, 4094, 0, 1, 0, 0, 0, 0, 1, 13, 0, 5, 0, 1, 0, 4051, 0, 476] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 0, 0, 61, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 2584, 1390, 0, 0, 0, 58, 0, 0, 0, 4096, 0, 60, 3841, 149, 4071, 0, 0, 3573, 270, 0, 0, 0, 0, 3, 0, 0, 0, 4090, 4033, 0, 0, 0, 0, 0, 0, 0, 7, 0, 9, 3, 0, 0, 4018, 0, 441] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 23, 17, 0, 0, 0, 0, 3, 0, 0, 0, 23, 0, 0, 0, 0, 3375, 3692, 0, 0, 0, 0, 0, 0, 0, 2904, 0, 0, 4061, 0, 4095, 0, 0, 4096, 2473, 0, 0, 0, 0, 0, 0, 0, 0, 114, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 262, 0, 3525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 5, 0, 0, 0, 2, 0, 7, 0, 599, 0, 0, 0, 292, 412, 14, 35, 0, 103, 0, 5, 0, 4065, 0, 5, 3549, 0, 4075, 101, 0, 4045, 3736, 1, 0, 0, 3, 0, 0, 0, 1, 4008, 3848, 0, 0, 19, 20, 1, 1, 0, 0, 53, 0, 1, 66, 35, 3660, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 309, 2905, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 1361, 0, 0, 4096, 3817, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 3917, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 18, 23, 4, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2759, 2, 0, 0, 45, 0, 1, 0, 4033, 0, 0, 4070, 130, 2477, 0, 43, 4096, 3210, 2, 0, 2, 0, 0, 0, 0, 0, 4095, 4046, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 3706, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3997, 3, 0, 0, 6, 0, 0, 0, 4086, 1, 4, 4094, 96, 4080, 0, 10, 4071, 14, 0, 0, 0, 0, 0, 0, 0, 18, 4091, 4089, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 0, 1, 4083, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 3887, 300, 0, 0, 0, 0, 0, 0, 0, 0, 2, 10, 0, 0, 0, 13, 3106, 1161, 0, 0, 0, 0, 0, 0, 0, 1, 0, 150, 4050, 0, 2, 0, 436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4015, 0, 13, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2431, 2, 14, 0, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 1, 0, 0, 3374, 688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 13, 4086, 69, 0, 0, 0, 0, 0, 0, 0, 5, 0, 1, 0, 4051, 0, 476, 56, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4188, 4292, 4639, 4168, 2605, 4117, 4170, 4589] avg=4096.0000 max=4639 max_rank=2 min=2605 min_rank=4 imbalance_ratio=1.132568 max_load_reduction=28129 max_load_reduction_pct=0.858429 moved_to_echo=15481 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 0, 0, 3646, 295, 0, 0, 0, 0, 0, 0, 61, 0, 9, 0, 0, 0, 0, 0, 3060, 1157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2584, 1390, 0, 0, 0, 435, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 4096, 0, 60, 130, 0, 0, 0, 0, 0, 0, 0, 0, 149, 2423, 0, 0, 0, 270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4090, 678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3573, 65, 0, 0, 0, 0, 0, 0, 0, 9, 3, 0, 0, 4018, 0, 441, 56, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3943, 4287, 4409, 4344, 2842, 4771, 3645, 4527] avg=4096.0000 max=4771 max_rank=5 min=2842 min_rank=4 imbalance_ratio=1.164795 max_load_reduction=27997 max_load_reduction_pct=0.854401 moved_to_echo=15785 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 23, 17, 0, 0, 3855, 299, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 23, 0, 3105, 1164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3375, 3692, 0, 0, 0, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2904, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2438, 0, 0, 0, 2473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 262, 0, 3525, 56, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4194, 4295, 7504, 3042, 4911, 803, 4164, 3855] avg=4096.0000 max=7504 max_rank=2 min=803 min_rank=5 imbalance_ratio=1.832031 max_load_reduction=25264 max_load_reduction_pct=0.770996 moved_to_echo=13733 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 5, 3369, 281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7, 0, 599, 2919, 1159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 292, 412, 14, 35, 0, 435, 0, 0, 0, 0, 0, 0, 0, 103, 0, 5, 0, 4065, 0, 5, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2425, 101, 0, 0, 3736, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 4008, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 20, 1, 1, 0, 0, 4045, 60, 0, 0, 0, 0, 0, 0, 53, 0, 1, 66, 35, 3660, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3656, 4686, 1188, 4298, 6263, 4660, 4146, 3871] avg=4096.0000 max=6263 max_rank=4 min=1188 min_rank=2 imbalance_ratio=1.529053 max_load_reduction=26505 max_load_reduction_pct=0.808868 moved_to_echo=18940 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 3884, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3108, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3995, 0, 0, 0, 206, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1152, 0, 0, 0, 2971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3401, 688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 69, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4070, 0, 6, 26, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4184, 3663, 4204, 4236, 4123, 4090, 4165, 4103] avg=4096.0000 max=4236 max_rank=3 min=3663 min_rank=1 imbalance_ratio=1.034180 max_load_reduction=28532 max_load_reduction_pct=0.870728 moved_to_echo=16481 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 18, 23, 3863, 296, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 1, 3070, 704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2759, 2, 0, 0, 264, 0, 0, 0, 0, 0, 0, 0, 45, 0, 1, 0, 4033, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 130, 1475, 0, 43, 0, 3210, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4095, 680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3706, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4201, 3780, 3025, 4217, 4860, 4777, 4168, 3740] avg=4096.0000 max=4860 max_rank=4 min=3025 min_rank=2 imbalance_ratio=1.186523 max_load_reduction=27908 max_load_reduction_pct=0.851685 moved_to_echo=17545 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 3888, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3108, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 309, 2905, 0, 0, 0, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 811, 0, 0, 0, 3817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 69, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3917, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4188, 3495, 3359, 4234, 4628, 4761, 4166, 3937] avg=4096.0000 max=4761 max_rank=5 min=3359 min_rank=2 imbalance_ratio=1.162354 max_load_reduction=28007 max_load_reduction_pct=0.854706 moved_to_echo=17492 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 20, 3890, 303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3106, 1164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3997, 3, 0, 0, 440, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4086, 1, 4, 132, 0, 0, 0, 0, 0, 0, 0, 0, 96, 2416, 0, 10, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 4091, 680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4071, 72, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 4083, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4214, 4270, 4440, 4229, 2536, 4789, 4144, 4146] avg=4096.0000 max=4789 max_rank=5 min=2536 min_rank=4 imbalance_ratio=1.169189 max_load_reduction=27979 max_load_reduction_pct=0.853851 moved_to_echo=15615 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[384, 15616, 128, 128, 0, 16768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 0, 6784, 23296, 128, 128, 0, 2816, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 0, 128, 128, 640, 24704, 7552, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 0, 128, 27264, 5504, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 128, 27776, 0, 4480, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 0, 0, 128, 128, 128, 128, 128, 30336, 2432, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 128, 128, 128, 128, 128, 32256, 640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 0, 128, 0, 31488, 128, 128, 1152, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33792 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[66, 0, 4020, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1037, 0, 0, 0, 0, 0, 1083, 0, 4021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1837, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 221, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[9, 0, 4024, 0, 0, 0, 37, 4057, 0, 0, 0, 3927, 0, 0, 0, 0, 0, 48, 0, 7, 0, 0, 0, 4096, 0, 3664, 1, 0, 0, 0, 0, 590, 0, 4087, 0, 2, 0, 0, 0, 0, 0, 0, 6, 0, 55, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 24, 3, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[3, 0, 4096, 0, 0, 0, 29, 4088, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3755, 0, 0, 0, 46, 0, 571, 0, 3776, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 42, 4083, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 4, 0, 0, 0, 0, 3107, 0, 1255, 0, 1, 0, 0, 3635, 0, 0, 0, 19, 0, 165, 0, 0, 4054, 0, 34, 0, 0, 2, 0, 16, 0, 0, 4076, 0, 0, 1, 4, 0, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[53, 0, 4096, 7, 0, 1, 1831, 4073, 0, 0, 0, 4032, 1, 0, 0, 0, 11, 0, 0, 0, 5, 0, 0, 4096, 0, 2288, 0, 0, 0, 0, 56, 184, 0, 3739, 0, 0, 34, 0, 0, 0, 0, 10, 0, 1, 33, 0, 0, 4083, 3, 0, 0, 0, 0, 5, 0, 0, 2, 4094, 0, 0, 0, 5, 25, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 4096, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4037, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 33, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 42, 3621, 17, 0, 0, 1153, 4051, 0, 0, 0, 4075, 3, 0, 0, 1, 0, 0, 14, 0, 0, 0, 2, 4042, 0, 4049, 5, 0, 0, 5, 6, 3056, 39, 366, 0, 0, 59, 3, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4026, 0, 3, 0, 0, 23, 0, 0, 51, 0, 4035, 0, 0, 20, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[2, 0, 4096, 0, 0, 0, 153, 4067, 0, 0, 0, 4096, 0, 29, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 1504, 0, 0, 0, 125, 5, 3557, 0, 2891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3994, 0, 0, 0, 0, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[53, 0, 3739, 7, 0, 1, 1831, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 1, 0, 0, 0, 21, 5, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 5, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2288, 0, 0, 0, 0, 56, 184, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3739, 0, 0, 34, 0, 0, 0, 357, 245, 0, 0, 0, 0, 0, 0, 0, 10, 0, 1, 33, 0, 0, 3817, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 5, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 2, 4086, 0, 0, 0, 5, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5631, 4059, 4112, 2531, 4375, 3861, 4081, 4118] avg=4096.0000 max=5631 max_rank=0 min=2531 min_rank=3 imbalance_ratio=1.374756 max_load_reduction=27137 max_load_reduction_pct=0.828156 moved_to_echo=16435 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[66, 0, 3670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 21, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 0, 0, 0, 0, 0, 1083, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4021, 0, 0, 0, 0, 0, 0, 350, 246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1837, 1, 0, 3829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 221, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3736, 4122, 4096, 2123, 4617, 5667, 4096, 4311] avg=4096.0000 max=5667 max_rank=5 min=2123 min_rank=3 imbalance_ratio=1.383545 max_load_reduction=27101 max_load_reduction_pct=0.827057 moved_to_echo=18691 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[9, 0, 3673, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3927, 0, 0, 0, 0, 21, 5, 0, 0, 0, 0, 0, 0, 0, 48, 0, 7, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3664, 1, 0, 0, 0, 0, 590, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 2, 0, 0, 0, 0, 351, 245, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 55, 0, 0, 3823, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 24, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3719, 3953, 4151, 4258, 4685, 3884, 4057, 4061] avg=4096.0000 max=4685 max_rank=4 min=3719 min_rank=0 imbalance_ratio=1.143799 max_load_reduction=28083 max_load_reduction_pct=0.857025 moved_to_echo=16687 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 3739, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 21, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3755, 0, 0, 0, 46, 0, 571, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3776, 0, 0, 0, 0, 0, 0, 357, 246, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 3, 0, 0, 3829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3771, 4122, 4096, 4375, 4379, 3853, 4088, 4084] avg=4096.0000 max=4379 max_rank=4 min=3771 min_rank=0 imbalance_ratio=1.069092 max_load_reduction=28389 max_load_reduction_pct=0.866364 moved_to_echo=16404 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 42, 3727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 21, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 3107, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1255, 0, 1, 0, 0, 3635, 0, 356, 243, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 165, 0, 0, 3790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 2, 0, 16, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 1, 4, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3769, 4122, 4071, 3114, 5490, 3974, 4148, 4080] avg=4096.0000 max=5490 max_rank=4 min=3114 min_rank=3 imbalance_ratio=1.340332 max_load_reduction=27278 max_load_reduction_pct=0.832458 moved_to_echo=17692 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 42, 3306, 17, 0, 0, 1153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 3, 0, 0, 1, 20, 5, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 2, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 5, 0, 0, 5, 6, 3056, 3, 0, 0, 0, 0, 0, 0, 0, 39, 366, 0, 0, 59, 3, 0, 0, 315, 241, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 23, 0, 0, 51, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4518, 4104, 4058, 7124, 1023, 3766, 4128, 4047] avg=4096.0000 max=7124 max_rank=3 min=1023 min_rank=4 imbalance_ratio=1.739258 max_load_reduction=25644 max_load_reduction_pct=0.782593 moved_to_echo=12964 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3739, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 21, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 357, 246, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 33, 0, 0, 3829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3740, 4128, 4096, 4040, 4699, 3882, 4096, 4087] avg=4096.0000 max=4699 max_rank=4 min=3740 min_rank=0 imbalance_ratio=1.147217 max_load_reduction=28069 max_load_reduction_pct=0.856598 moved_to_echo=16764 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 3729, 0, 0, 0, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 29, 0, 1, 24, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1504, 0, 0, 0, 125, 5, 3557, 8, 3, 0, 0, 0, 0, 0, 0, 0, 2891, 0, 0, 0, 0, 0, 0, 360, 249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 3822, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 7, 0, 0, 0, 0, 0, 0, 0, 3978, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3884, 4158, 4088, 5203, 3500, 3881, 4074, 3980] avg=4096.0000 max=5203 max_rank=3 min=3500 min_rank=4 imbalance_ratio=1.270264 max_load_reduction=27565 max_load_reduction_pct=0.841217 moved_to_echo=15435 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 16384, 128, 0, 0, 256, 128, 16256, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 24320, 0, 128, 128, 128, 3712, 0, 2816, 2048, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 2304, 128, 0, 30592, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[256, 128, 29440, 128, 0, 128, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 0, 0, 32640, 128, 128, 0, 128, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 0, 0, 128, 128, 128, 128, 32640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 32512, 0, 128, 128, 128, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 4093, 8, 79, 0, 0, 0, 4001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 200, 0, 0, 0, 0, 0, 0, 0, 173, 0, 3927, 0, 7, 0, 1, 49, 0, 0, 0, 0, 0, 0, 142, 0, 4096, 0, 0, 4090, 3711, 0, 4090] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2, 7, 0, 14, 0, 111, 0, 0, 0, 0, 0, 1913, 0, 4060, 0, 0, 0, 3903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 53, 53, 0, 0, 1, 0, 0, 102, 0, 0, 2203, 0, 519, 3, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4092, 3968, 59, 3517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 9, 2, 0, 0, 0, 0, 9, 0, 2, 0, 160, 0, 0, 0, 999, 0, 3986, 0, 0, 0, 2184, 2, 36, 0, 3, 0, 0, 0, 25, 0, 4056, 0, 0, 0, 0, 0, 0, 55, 675, 0, 0, 3873, 0, 116, 0, 668, 0, 0, 0, 0, 0, 0, 85, 0, 0, 4034, 0, 0, 4096, 3997, 44, 3652] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3865, 0, 4092, 0, 0, 1, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 49, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2368, 0, 4095, 0, 0, 0, 1509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 263, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4067, 0, 4083] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 6, 1, 0, 0, 1, 4041, 0, 4051, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 57, 8, 0, 0, 0, 0, 20, 0, 1, 4084, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 46, 4088, 0, 0, 4094, 4092, 4, 4033] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 8, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 285, 0, 0, 2, 3, 0, 4086, 0, 0, 0, 4006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4087, 0, 0, 0, 178, 88, 0, 0, 0, 0, 0, 0, 0, 1, 3961, 0, 0, 4084, 4077, 7, 3743] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 3570, 0, 4096, 0, 0, 0, 509, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4075, 0, 0, 7, 0, 13, 2, 0, 8, 1, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 4096, 0, 0, 4092, 4096, 0, 4086] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2, 7, 0, 4096, 11, 4, 0, 0, 0, 0, 0, 14, 0, 111, 0, 0, 0, 0, 0, 3833, 402, 31, 49, 0, 0, 0, 0, 1511, 0, 0, 0, 0, 0, 3903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 4041, 53, 53, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 0, 2172, 0, 519, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 3800, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 59, 3517, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4440, 5414, 4088, 4148, 2796, 4029, 3733] avg=4096.0000 max=5414 max_rank=2 min=2796 min_rank=5 imbalance_ratio=1.321777 max_load_reduction=27354 max_load_reduction_pct=0.834778 moved_to_echo=14706 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 75, 860, 55, 49, 0, 0, 0, 0, 3233, 8, 0, 0, 0, 0, 4001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 4047, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173, 0, 3872, 0, 7, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 142, 3554, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 1044, 7242, 4085, 4247, 4053, 3749, 4237] avg=4096.0000 max=7242 max_rank=2 min=1044 min_rank=1 imbalance_ratio=1.768066 max_load_reduction=25526 max_load_reduction_pct=0.778992 moved_to_echo=16286 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 9, 2, 0, 0, 0, 0, 4034, 11, 5, 0, 0, 0, 0, 0, 9, 0, 2, 0, 160, 0, 0, 0, 3764, 210, 54, 48, 0, 0, 0, 0, 789, 0, 0, 0, 0, 0, 2184, 2, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 3, 0, 0, 0, 25, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4008, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 675, 0, 0, 3819, 0, 116, 0, 668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 3828, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 44, 3652, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4247, 2975, 4155, 4063, 5278, 4135, 3854] avg=4096.0000 max=5278 max_rank=5 min=2975 min_rank=2 imbalance_ratio=1.288574 max_load_reduction=27490 max_load_reduction_pct=0.838928 moved_to_echo=17330 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 12, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3864, 812, 58, 49, 0, 0, 0, 0, 3053, 0, 0, 0, 0, 1, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3923, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 49, 4094, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4783, 3232, 4091, 4047, 4047, 4151, 4304] avg=4096.0000 max=4783 max_rank=1 min=3232 min_rank=2 imbalance_ratio=1.167725 max_load_reduction=27985 max_load_reduction_pct=0.854034 moved_to_echo=16549 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 12, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3866, 498, 58, 49, 0, 0, 0, 0, 1870, 0, 0, 0, 0, 0, 1509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 263, 0, 0, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3895, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4471, 3379, 4092, 4047, 4300, 4124, 4243] avg=4096.0000 max=4471 max_rank=1 min=3379 min_rank=2 imbalance_ratio=1.091553 max_load_reduction=28297 max_load_reduction_pct=0.863556 moved_to_echo=16714 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 12, 4088, 12, 4, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 0, 0, 1, 3824, 849, 57, 49, 0, 0, 0, 0, 3192, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4043, 0, 57, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 1, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 3919, 227, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 161, 4, 4033, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4787, 3193, 4090, 4108, 4048, 4182, 4244] avg=4096.0000 max=4787 max_rank=1 min=3193 min_rank=2 imbalance_ratio=1.168701 max_load_reduction=27981 max_load_reduction_pct=0.853912 moved_to_echo=16582 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 0, 33, 0, 0, 0, 0, 3961, 12, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 285, 0, 0, 2, 3857, 0, 57, 49, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 4047, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4030, 0, 0, 0, 178, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 3904, 229, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 161, 7, 3743, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4019, 4250, 4009, 4079, 4062, 4216, 4221, 3912] avg=4096.0000 max=4250 max_rank=1 min=3912 min_rank=7 imbalance_ratio=1.037598 max_load_reduction=28518 max_load_reduction_pct=0.870300 moved_to_echo=16411 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 15, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 3868, 755, 62, 51, 0, 0, 0, 0, 2815, 0, 0, 0, 0, 0, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 4024, 0, 0, 7, 0, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 0, 4021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 3926, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4746, 3324, 4088, 4046, 4030, 4177, 4241] avg=4096.0000 max=4746 max_rank=1 min=3324 min_rank=2 imbalance_ratio=1.158691 max_load_reduction=28022 max_load_reduction_pct=0.855164 moved_to_echo=16494 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 0, 128, 128, 512, 0, 128, 128, 27008, 4480, 512, 512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[16512, 128, 0, 128, 0, 128, 16384, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1152, 256, 128, 30080, 0, 768, 128, 896, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 0, 0, 1280, 256, 31360, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[32384, 384, 128, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[256, 128, 0, 0, 128, 0, 128, 256, 30848, 1664, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 0, 0, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[0, 128, 128, 128, 0, 128, 128, 128, 32640, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 13, 8, 3, 8, 0, 2, 0, 2505, 340, 0, 0, 1, 25, 0, 0, 0, 0, 29, 31, 0, 0, 1, 4077, 0, 1, 4095, 4, 177, 0, 0, 164, 0, 136, 0, 0, 0, 0, 4063, 29, 0, 0, 0, 0, 0, 4, 0, 1, 68, 4029, 0, 867, 0, 0, 0, 4085, 0, 3912, 18, 0, 0, 4071, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3, 0, 18, 3918, 123, 0, 0, 0, 3983, 0, 0, 3, 0, 0, 0, 0, 6, 0, 3589, 0, 0, 3212, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 1875, 0, 73, 1, 0, 0, 550, 0, 7, 0, 0, 4084, 0, 0, 0, 0, 0, 4077, 0, 4096, 0, 0, 13, 3105, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 36, 0, 4, 0, 4095, 0, 0, 3594, 0, 0, 0, 0, 10, 4, 0, 0, 0, 0, 0, 3089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1470, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 4090, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4, 0, 4084, 2531, 0, 0, 0, 4064, 29, 0, 0, 0, 0, 4054, 0, 0, 0, 4095, 0, 0, 395, 0, 64, 0, 0, 11, 0, 0, 0, 2, 0, 0, 949, 0, 0, 6, 3, 0, 0, 1, 0, 12, 0, 274, 1, 0, 3, 0, 0, 4086, 0, 4009, 0, 0, 0, 4091, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 72, 0, 0, 0, 3636, 0, 0, 0, 0, 0, 597, 0, 2, 0, 4094, 0, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4001, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 0, 4021, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3993, 0, 0, 0, 0, 0, 1183, 0, 12, 0, 4083, 0, 0, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2997, 0, 225, 0, 0, 0, 18, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 4096, 0, 4083, 0, 0, 0, 3733, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 8, 4041, 546, 0, 0, 0, 4030, 0, 0, 0, 0, 4, 2226, 1, 0, 11, 4071, 0, 0, 2080, 0, 0, 0, 0, 1, 0, 0, 0, 0, 6, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3653, 0, 0, 0, 0, 0, 4096, 0, 4091, 16, 0, 0, 3784, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 35, 0, 0, 0, 4091, 115, 0, 0, 0, 539, 0, 0, 0, 0, 0, 479, 0, 2, 0, 3979, 1, 0, 3581, 1, 116, 0, 0, 0, 0, 0, 0, 0, 1, 9, 3678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 4093, 0, 4089, 8, 0, 0, 3854, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3, 0, 18, 3848, 269, 0, 0, 0, 0, 0, 0, 590, 123, 0, 0, 0, 3983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 6, 0, 2695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3212, 9, 0, 0, 0, 0, 1896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 1875, 0, 894, 489, 0, 0, 0, 0, 0, 0, 73, 1, 0, 0, 550, 0, 7, 0, 3042, 943, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 766, 0, 0, 0, 0, 0, 0, 0, 0, 0, 248, 0, 0, 13, 3105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4138, 4696, 2704, 5117, 3281, 4616, 4850, 3366] avg=4096.0000 max=5117 max_rank=3 min=2704 min_rank=2 imbalance_ratio=1.249268 max_load_reduction=27651 max_load_reduction_pct=0.843842 moved_to_echo=16113 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 13, 8, 3, 8, 0, 2, 0, 3675, 269, 0, 0, 0, 0, 0, 0, 378, 340, 0, 0, 1, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 31, 0, 0, 1, 3061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 4, 177, 0, 0, 164, 1212, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 0, 0, 0, 4063, 29, 1016, 312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 3048, 603, 0, 0, 0, 0, 0, 0, 68, 4029, 0, 867, 0, 0, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 237, 18, 0, 0, 4071, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3978, 744, 3122, 5653, 5556, 3656, 5732, 4327] avg=4096.0000 max=5732 max_rank=6 min=744 min_rank=1 imbalance_ratio=1.399414 max_load_reduction=27036 max_load_reduction_pct=0.825073 moved_to_echo=19271 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 3845, 270, 0, 0, 0, 0, 0, 0, 617, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 4, 0, 3075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3594, 0, 0, 0, 0, 10, 1980, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 3089, 0, 1020, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3056, 985, 0, 0, 0, 0, 0, 0, 0, 1470, 0, 0, 0, 0, 0, 770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 248, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4712, 3115, 5584, 4623, 4041, 2240, 4338] avg=4096.0000 max=5584 max_rank=3 min=2240 min_rank=6 imbalance_ratio=1.363281 max_load_reduction=27184 max_load_reduction_pct=0.829590 moved_to_echo=15242 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 3844, 270, 0, 0, 0, 0, 0, 0, 617, 72, 0, 0, 0, 3636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 597, 0, 2, 0, 3074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 0, 0, 0, 0, 1982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3949, 0, 1020, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3056, 986, 0, 0, 0, 0, 0, 0, 0, 4001, 0, 0, 0, 0, 0, 770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 248, 0, 0, 0, 4021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4325, 3673, 2094, 5480, 4042, 4771, 4269] avg=4096.0000 max=5480 max_rank=4 min=2094 min_rank=3 imbalance_ratio=1.337891 max_load_reduction=27288 max_load_reduction_pct=0.832764 moved_to_echo=18562 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 8, 3844, 270, 0, 0, 0, 0, 0, 0, 609, 546, 0, 0, 0, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2226, 1, 0, 11, 3057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2080, 0, 0, 0, 0, 1, 1955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 101, 0, 1014, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3056, 973, 0, 0, 0, 0, 0, 0, 0, 3653, 0, 0, 0, 0, 0, 770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 16, 0, 0, 3784, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4122, 5185, 5299, 4036, 1625, 4029, 4423, 4049] avg=4096.0000 max=5299 max_rank=2 min=1625 min_rank=4 imbalance_ratio=1.293701 max_load_reduction=27469 max_load_reduction_pct=0.838287 moved_to_echo=14126 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 3836, 270, 0, 0, 0, 0, 0, 0, 617, 0, 0, 0, 0, 3993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1183, 0, 12, 0, 3066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173, 0, 0, 0, 0, 0, 1982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2997, 0, 1017, 511, 0, 0, 0, 0, 0, 0, 225, 0, 0, 0, 18, 0, 0, 0, 3056, 986, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 0, 0, 0, 3733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4610, 4261, 2155, 4525, 4285, 4846, 3980] avg=4096.0000 max=4846 max_rank=6 min=2155 min_rank=3 imbalance_ratio=1.183105 max_load_reduction=27922 max_load_reduction_pct=0.852112 moved_to_echo=17636 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4, 0, 3766, 269, 0, 0, 0, 0, 0, 0, 616, 2531, 0, 0, 0, 4064, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 0, 3075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 395, 0, 64, 0, 0, 11, 1976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 949, 0, 1020, 509, 0, 0, 0, 0, 0, 0, 0, 6, 3, 0, 0, 1, 0, 12, 3048, 983, 0, 0, 0, 0, 0, 0, 0, 274, 1, 0, 3, 0, 0, 769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4039, 7240, 7129, 2446, 2480, 4053, 1047, 4334] avg=4096.0000 max=7240 max_rank=1 min=1047 min_rank=6 imbalance_ratio=1.767578 max_load_reduction=25528 max_load_reduction_pct=0.779053 moved_to_echo=11914 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 35, 0, 0, 0, 3847, 274, 0, 0, 0, 0, 0, 0, 602, 115, 0, 0, 0, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 0, 2, 0, 2984, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3581, 1, 116, 0, 0, 0, 1984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 9, 3678, 0, 995, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3056, 990, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 763, 0, 0, 0, 0, 0, 0, 0, 0, 0, 242, 8, 0, 0, 3854, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4156, 1256, 3465, 5683, 5198, 4046, 4859, 4105] avg=4096.0000 max=5683 max_rank=3 min=1256 min_rank=1 imbalance_ratio=1.387451 max_load_reduction=27085 max_load_reduction_pct=0.826569 moved_to_echo=18208 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[384, 128, 128, 0, 640, 128, 128, 128, 24448, 7552, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 17280, 128, 384, 0, 0, 256, 14976, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4736, 3840, 0, 0, 128, 24448, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 256, 0, 128, 128, 128, 20736, 128, 8064, 3968, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 25728, 128, 896, 128, 0, 0, 6272, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 0, 128, 8704, 128, 128, 128, 24192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[0, 128, 128, 128, 128, 128, 128, 128, 30592, 2176, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 2048, 128, 0, 128, 30848, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[2928, 0, 36, 2, 0, 1751, 0, 0, 0, 0, 4041, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4086, 6, 0, 0, 0, 0, 131, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3787, 458, 3006, 0, 0, 0, 0, 3, 0, 0, 0, 39] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[70, 4, 3307, 519, 0, 0, 5, 0, 0, 343, 2262, 4, 30, 0, 0, 37, 0, 4050, 3776, 6, 1, 0, 9, 0, 57, 1, 0, 0, 4082, 164, 0, 1, 4027, 0, 0, 0, 0, 0, 799, 0, 13, 0, 0, 0, 0, 0, 2, 46, 0, 0, 1, 1, 796, 3495, 4063, 0, 2, 1, 0, 713, 0, 0, 0, 81] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[435, 0, 1926, 0, 0, 764, 0, 0, 0, 0, 1782, 3, 0, 0, 0, 0, 0, 4090, 4096, 70, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 14, 3092, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 4087, 4096, 4033, 0, 0, 0, 0, 19, 0, 0, 0, 144] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[4025, 0, 2250, 0, 0, 0, 0, 0, 0, 230, 1159, 0, 0, 0, 0, 0, 0, 4096, 4088, 120, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3956, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 4095, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[4078, 0, 3255, 0, 0, 3, 0, 4, 0, 174, 3918, 0, 1, 5, 0, 0, 0, 4077, 4093, 24, 0, 0, 0, 0, 57, 0, 0, 0, 4091, 0, 0, 0, 979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 2, 0, 0, 3884, 4090, 2, 0, 0, 11, 0, 0, 0, 2, 13] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[4090, 38, 5, 0, 0, 1338, 0, 10, 10, 125, 3131, 0, 0, 0, 103, 0, 0, 3949, 4046, 993, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 3821, 0, 0, 0, 0, 7, 0, 14, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 9, 14, 13, 2756, 4095, 0, 0, 102, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1470, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 1989, 0, 0, 0, 0, 0, 0, 4096, 3906, 91, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 810, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4040, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[94, 0, 275, 0, 1, 64, 0, 0, 0, 0, 3853, 0, 0, 0, 0, 0, 0, 4037, 4096, 0, 0, 1, 0, 3, 0, 0, 1, 0, 4093, 0, 0, 7, 4087, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3980, 4047, 4091, 0, 0, 2, 0, 33, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2238, 0, 36, 2, 0, 1751, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 0, 0, 0, 0, 0, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 6, 0, 0, 0, 0, 131, 207, 690, 28, 95, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3787, 264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 39, 2991, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4027, 4204, 4001, 4066, 5274, 4109, 4051, 3036] avg=4096.0000 max=5274 max_rank=4 min=3036 min_rank=7 imbalance_ratio=1.287598 max_load_reduction=27494 max_load_reduction_pct=0.839050 moved_to_echo=16470 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[333, 0, 1926, 0, 0, 764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1782, 3, 0, 0, 0, 0, 1458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4001, 70, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 3092, 0, 0, 0, 0, 1, 0, 2, 102, 255, 95, 31, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 4090, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 2354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 144, 4013, 29, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3023, 3243, 4074, 4079, 3578, 4125, 6441, 4205] avg=4096.0000 max=6441 max_rank=6 min=3023 min_rank=0 imbalance_ratio=1.572510 max_load_reduction=26327 max_load_reduction_pct=0.803436 moved_to_echo=18349 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[54, 4, 3307, 519, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 343, 2262, 4, 30, 0, 0, 37, 1244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3689, 6, 1, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 1, 0, 0, 4051, 164, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 0, 0, 0, 0, 0, 799, 0, 16, 218, 87, 31, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 2, 46, 4050, 20, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 796, 2009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 713, 0, 0, 0, 81, 4043, 24, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3889, 3920, 3705, 4274, 5178, 4131, 2807, 4864] avg=4096.0000 max=5178 max_rank=4 min=2807 min_rank=6 imbalance_ratio=1.264160 max_load_reduction=27590 max_load_reduction_pct=0.841980 moved_to_echo=16980 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3076, 0, 2250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 230, 1159, 0, 0, 0, 0, 0, 1458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3993, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3956, 0, 0, 0, 0, 0, 0, 0, 949, 255, 95, 31, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 4096, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 2353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 29, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5326, 2847, 4113, 4065, 5286, 4132, 2902, 4097] avg=4096.0000 max=5326 max_rank=0 min=2847 min_rank=1 imbalance_ratio=1.300293 max_load_reduction=27442 max_load_reduction_pct=0.837463 moved_to_echo=16417 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3117, 0, 3255, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174, 3918, 0, 1, 5, 0, 0, 1382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3998, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 979, 0, 0, 0, 0, 0, 0, 0, 961, 242, 95, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4077, 19, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2233, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 2, 13, 4071, 27, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6379, 5480, 4022, 4117, 2308, 4101, 2237, 4124] avg=4096.0000 max=6379 max_rank=0 min=2237 min_rank=6 imbalance_ratio=1.557373 max_load_reduction=26389 max_load_reduction_pct=0.805328 moved_to_echo=12770 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3126, 38, 5, 0, 0, 1338, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 10, 125, 3131, 0, 0, 0, 103, 0, 981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3952, 993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3821, 0, 0, 0, 0, 7, 0, 14, 964, 172, 94, 31, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 3949, 20, 0, 0, 0, 0, 0, 0, 0, 0, 9, 14, 13, 1584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 0, 0, 0, 0, 0, 4075, 19, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4517, 4350, 4945, 4039, 5103, 3998, 1620, 4196] avg=4096.0000 max=5103 max_rank=4 min=1620 min_rank=6 imbalance_ratio=1.245850 max_load_reduction=27665 max_load_reduction_pct=0.844269 moved_to_echo=14917 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1124, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1989, 0, 0, 0, 0, 0, 1438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3816, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 810, 0, 0, 0, 0, 0, 2, 0, 346, 252, 90, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4077, 28, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5199, 3427, 3907, 4065, 1531, 4115, 6417, 4107] avg=4096.0000 max=6417 max_rank=6 min=1531 min_rank=4 imbalance_ratio=1.566650 max_load_reduction=26351 max_load_reduction_pct=0.804169 moved_to_echo=16170 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[68, 0, 275, 0, 1, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3853, 0, 0, 0, 0, 0, 1444, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3997, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4055, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 3, 26, 257, 99, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3980, 2313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 33, 0, 0, 0, 0, 4071, 33, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[408, 5297, 4001, 4063, 4510, 4057, 6293, 4139] avg=4096.0000 max=6293 max_rank=6 min=408 min_rank=0 imbalance_ratio=1.536377 max_load_reduction=26475 max_load_reduction_pct=0.807953 moved_to_echo=18999 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 896, 22144, 128, 128, 128, 128, 128, 9600, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 128, 128, 17408, 15488, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[13184, 128, 15232, 640, 128, 3968, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[24960, 128, 0, 0, 0, 128, 1024, 256, 4096, 1792, 768, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 0, 31488, 1408, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 0, 32512, 256, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 128, 128, 128, 128, 32512, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 896, 0, 0, 128, 384, 31488, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True + [2026-09-21 08:25:37] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 255.1 | throughput per GPU (TFLOP/s/GPU): 126.3 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.077168E+01 | loss scale: 1.0 | grad norm: 21.307 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1007, 597, 248, 5, 1, 1002, 1279, 152, 0, 1339, 115, 0, 21, 76, 94, 247, 20, 329, 337, 1390, 11, 264, 303, 48, 309, 59, 35, 643, 758, 1459, 308, 1045, 185, 247, 720, 443, 142, 1873, 471, 912, 808, 1237, 1536, 11, 2341, 1224, 5, 247, 760, 478, 982, 44, 1040, 786, 459, 36, 1431, 6, 501, 11, 49, 59, 202, 21] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1106, 591, 118, 7, 9, 943, 1283, 139, 16, 1602, 14, 1, 58, 232, 103, 266, 2, 670, 499, 1678, 13, 381, 164, 38, 533, 32, 33, 354, 909, 1089, 505, 847, 120, 153, 784, 416, 78, 1636, 370, 949, 444, 1104, 912, 28, 2012, 1578, 28, 252, 993, 623, 852, 23, 1090, 958, 494, 60, 1582, 42, 425, 23, 15, 114, 352, 23] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[667, 596, 171, 17, 8, 767, 1312, 155, 6, 1956, 81, 0, 5, 211, 111, 656, 11, 608, 472, 1361, 41, 715, 305, 10, 386, 15, 52, 253, 812, 738, 494, 787, 214, 198, 818, 395, 175, 1976, 444, 780, 553, 962, 1202, 20, 2383, 1362, 5, 426, 961, 806, 965, 20, 669, 1206, 791, 94, 775, 7, 400, 3, 29, 36, 203, 111] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1112, 566, 177, 5, 6, 577, 1369, 218, 16, 1616, 99, 0, 28, 67, 360, 256, 35, 641, 717, 2024, 22, 360, 149, 10, 169, 26, 46, 420, 760, 972, 306, 1085, 233, 235, 506, 303, 111, 1822, 361, 774, 580, 967, 1219, 32, 1773, 1234, 31, 388, 946, 866, 1006, 14, 781, 1175, 489, 83, 1166, 21, 672, 9, 29, 44, 653, 31] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1418, 604, 204, 3, 25, 935, 1161, 232, 16, 2010, 13, 2, 17, 70, 214, 108, 49, 499, 401, 1577, 21, 166, 248, 12, 327, 36, 35, 524, 875, 966, 285, 896, 240, 224, 622, 784, 128, 1621, 272, 791, 399, 727, 1319, 14, 2224, 1360, 19, 193, 605, 738, 967, 58, 1191, 1057, 711, 26, 1320, 18, 362, 8, 9, 160, 591, 61] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1328, 637, 310, 0, 10, 558, 1562, 126, 70, 1972, 23, 5, 11, 172, 182, 329, 77, 801, 468, 1373, 13, 291, 299, 31, 209, 55, 29, 427, 546, 828, 512, 1120, 365, 217, 605, 569, 231, 1229, 140, 720, 404, 1129, 969, 64, 2510, 1504, 32, 230, 777, 593, 828, 138, 1093, 956, 407, 49, 1403, 21, 218, 39, 7, 99, 821, 27] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1292, 490, 239, 2, 3, 783, 1401, 106, 7, 1547, 43, 1, 74, 59, 238, 352, 81, 867, 760, 1580, 22, 238, 301, 15, 238, 21, 61, 313, 969, 893, 479, 1107, 361, 215, 700, 364, 169, 1583, 367, 693, 651, 1019, 1229, 51, 1601, 822, 44, 326, 829, 810, 849, 39, 957, 970, 773, 199, 1003, 18, 531, 44, 10, 176, 701, 82] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1103, 660, 357, 57, 10, 999, 1455, 130, 18, 2407, 37, 2, 14, 148, 108, 155, 52, 177, 290, 1445, 17, 306, 242, 87, 304, 122, 25, 569, 627, 1304, 227, 672, 102, 146, 585, 477, 66, 1245, 297, 599, 1136, 818, 1235, 41, 2232, 1579, 24, 170, 748, 561, 958, 123, 950, 1462, 809, 78, 1016, 9, 551, 14, 53, 102, 428, 28] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1106, 591, 118, 7, 9, 943, 1089, 139, 0, 0, 0, 0, 0, 0, 0, 0, 16, 1602, 14, 1, 58, 232, 103, 266, 958, 386, 124, 0, 0, 0, 0, 0, 2, 670, 499, 1678, 13, 381, 164, 38, 353, 339, 146, 0, 0, 0, 0, 0, 533, 32, 33, 354, 909, 1089, 505, 847, 73, 48, 0, 0, 0, 0, 0, 0, 120, 153, 784, 416, 78, 1283, 370, 949, 0, 0, 0, 0, 0, 0, 0, 0, 444, 1104, 912, 28, 0, 1115, 28, 252, 0, 0, 0, 0, 0, 0, 0, 0, 993, 623, 852, 23, 1017, 0, 494, 60, 0, 0, 0, 0, 0, 0, 0, 0, 1582, 42, 425, 23, 15, 114, 352, 23, 1626, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4002, 3760, 4283, 4423, 4153, 3883, 4062, 4202] avg=4096.0000 max=4423 max_rank=3 min=3760 min_rank=1 imbalance_ratio=1.079834 max_load_reduction=28345 max_load_reduction_pct=0.865021 moved_to_echo=16300 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[667, 596, 171, 17, 8, 767, 1114, 155, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1956, 81, 0, 5, 211, 111, 656, 1206, 458, 107, 0, 0, 0, 0, 0, 11, 608, 472, 1361, 41, 715, 305, 10, 426, 293, 149, 0, 0, 0, 0, 0, 386, 15, 52, 253, 812, 738, 494, 787, 44, 49, 0, 0, 0, 0, 0, 0, 214, 198, 818, 395, 175, 1550, 444, 780, 0, 0, 0, 0, 0, 0, 0, 0, 553, 962, 1202, 20, 0, 962, 5, 426, 0, 0, 0, 0, 0, 0, 0, 0, 961, 806, 965, 20, 625, 0, 791, 94, 0, 0, 0, 0, 0, 0, 0, 0, 775, 7, 400, 3, 29, 36, 203, 111, 1925, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3495, 4797, 4391, 3630, 4574, 4130, 4262, 3489] avg=4096.0000 max=4797 max_rank=1 min=3489 min_rank=7 imbalance_ratio=1.171143 max_load_reduction=27971 max_load_reduction_pct=0.853607 moved_to_echo=16455 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1112, 566, 177, 5, 6, 577, 1162, 218, 0, 0, 0, 0, 0, 0, 0, 0, 16, 1616, 99, 0, 28, 67, 360, 256, 1175, 341, 97, 0, 0, 0, 0, 0, 35, 641, 717, 2024, 22, 360, 149, 10, 393, 265, 156, 0, 0, 0, 0, 0, 169, 26, 46, 420, 760, 972, 306, 1085, 52, 51, 0, 0, 0, 0, 0, 0, 233, 235, 506, 303, 111, 1429, 361, 774, 0, 0, 0, 0, 0, 0, 0, 0, 580, 967, 1219, 32, 0, 872, 31, 388, 0, 0, 0, 0, 0, 0, 0, 0, 946, 866, 1006, 14, 729, 0, 489, 83, 0, 0, 0, 0, 0, 0, 0, 0, 1166, 21, 672, 9, 29, 44, 653, 31, 1432, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3823, 4055, 4772, 3887, 3952, 4089, 4133, 4057] avg=4096.0000 max=4772 max_rank=2 min=3823 min_rank=0 imbalance_ratio=1.165039 max_load_reduction=27996 max_load_reduction_pct=0.854370 moved_to_echo=16231 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1328, 637, 310, 0, 10, 558, 1326, 126, 0, 0, 0, 0, 0, 0, 0, 0, 70, 1972, 23, 5, 11, 172, 182, 329, 956, 483, 118, 0, 0, 0, 0, 0, 77, 801, 468, 1373, 13, 291, 299, 31, 265, 323, 178, 0, 0, 0, 0, 0, 209, 55, 29, 427, 546, 828, 512, 1120, 73, 58, 0, 0, 0, 0, 0, 0, 365, 217, 605, 569, 231, 964, 140, 720, 0, 0, 0, 0, 0, 0, 0, 0, 404, 1129, 969, 64, 0, 1063, 32, 230, 0, 0, 0, 0, 0, 0, 0, 0, 777, 593, 828, 138, 1020, 0, 407, 49, 0, 0, 0, 0, 0, 0, 0, 0, 1403, 21, 218, 39, 7, 99, 821, 27, 2027, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4295, 4321, 4119, 3857, 3811, 3891, 3812, 4662] avg=4096.0000 max=4662 max_rank=7 min=3811 min_rank=4 imbalance_ratio=1.138184 max_load_reduction=28106 max_load_reduction_pct=0.857727 moved_to_echo=16176 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1007, 597, 248, 5, 1, 1002, 1086, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1339, 115, 0, 21, 76, 94, 247, 786, 450, 96, 0, 0, 0, 0, 0, 20, 329, 337, 1390, 11, 264, 303, 48, 404, 263, 145, 0, 0, 0, 0, 0, 309, 59, 35, 643, 758, 1459, 308, 1045, 69, 48, 0, 0, 0, 0, 0, 0, 185, 247, 720, 443, 142, 1469, 471, 912, 0, 0, 0, 0, 0, 0, 0, 0, 808, 1237, 1536, 11, 0, 865, 5, 247, 0, 0, 0, 0, 0, 0, 0, 0, 760, 478, 982, 44, 971, 0, 459, 36, 0, 0, 0, 0, 0, 0, 0, 0, 1431, 6, 501, 11, 49, 59, 202, 21, 1891, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 3224, 3514, 4733, 4589, 4709, 3730, 4171] avg=4096.0000 max=4733 max_rank=3 min=3224 min_rank=1 imbalance_ratio=1.155518 max_load_reduction=28035 max_load_reduction_pct=0.855560 moved_to_echo=17199 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1418, 604, 204, 3, 25, 935, 986, 232, 0, 0, 0, 0, 0, 0, 0, 0, 16, 2010, 13, 2, 17, 70, 214, 108, 1057, 427, 107, 0, 0, 0, 0, 0, 49, 499, 401, 1577, 21, 166, 248, 12, 349, 292, 132, 0, 0, 0, 0, 0, 327, 36, 35, 524, 875, 966, 285, 896, 79, 43, 0, 0, 0, 0, 0, 0, 240, 224, 622, 784, 128, 1272, 272, 791, 0, 0, 0, 0, 0, 0, 0, 0, 399, 727, 1319, 14, 0, 961, 19, 193, 0, 0, 0, 0, 0, 0, 0, 0, 605, 738, 967, 58, 1112, 0, 711, 26, 0, 0, 0, 0, 0, 0, 0, 0, 1320, 18, 362, 8, 9, 160, 591, 61, 1797, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4407, 4041, 3746, 4066, 4333, 3632, 4217, 4326] avg=4096.0000 max=4407 max_rank=0 min=3632 min_rank=5 imbalance_ratio=1.075928 max_load_reduction=28361 max_load_reduction_pct=0.865509 moved_to_echo=16508 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1292, 490, 239, 2, 3, 783, 1190, 106, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1547, 43, 1, 74, 59, 238, 352, 970, 308, 64, 0, 0, 0, 0, 0, 81, 867, 760, 1580, 22, 238, 301, 15, 341, 176, 159, 0, 0, 0, 0, 0, 238, 21, 61, 313, 969, 893, 479, 1107, 64, 52, 0, 0, 0, 0, 0, 0, 361, 215, 700, 364, 169, 1242, 367, 693, 0, 0, 0, 0, 0, 0, 0, 0, 651, 1019, 1229, 51, 0, 582, 44, 326, 0, 0, 0, 0, 0, 0, 0, 0, 829, 810, 849, 39, 893, 0, 773, 199, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 18, 531, 44, 10, 176, 701, 82, 1293, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 3663, 4540, 4197, 4111, 3902, 4392, 3858] avg=4096.0000 max=4540 max_rank=2 min=3663 min_rank=1 imbalance_ratio=1.108398 max_load_reduction=28228 max_load_reduction_pct=0.861450 moved_to_echo=16263 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1103, 660, 357, 57, 10, 999, 1227, 130, 0, 0, 0, 0, 0, 0, 0, 0, 18, 2407, 37, 2, 14, 148, 108, 155, 1462, 430, 126, 0, 0, 0, 0, 0, 52, 177, 290, 1445, 17, 306, 242, 87, 272, 345, 170, 0, 0, 0, 0, 0, 304, 122, 25, 569, 627, 1304, 227, 672, 67, 58, 0, 0, 0, 0, 0, 0, 102, 146, 585, 477, 66, 973, 297, 599, 0, 0, 0, 0, 0, 0, 0, 0, 1136, 818, 1235, 41, 0, 1108, 24, 170, 0, 0, 0, 0, 0, 0, 0, 0, 748, 561, 958, 123, 883, 0, 809, 78, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 9, 551, 14, 53, 102, 428, 28, 1802, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4543, 4907, 3403, 3975, 3245, 4532, 4160, 4003] avg=4096.0000 max=4907 max_rank=1 min=3245 min_rank=4 imbalance_ratio=1.197998 max_load_reduction=27861 max_load_reduction_pct=0.850250 moved_to_echo=15940 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[9088, 4864, 1920, 128, 128, 6656, 9216, 1280, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[256, 14464, 512, 128, 256, 1152, 1536, 2432, 8576, 3328, 896, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[6656, 5504, 7424, 512, 7296, 0, 4992, 640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1920, 1664, 5376, 3840, 1152, 10240, 2816, 6272, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[2560, 384, 384, 3584, 6272, 8320, 3200, 7680, 640, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[384, 4608, 3968, 12544, 256, 2816, 2048, 256, 2816, 2304, 1280, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[9728, 256, 3712, 256, 256, 896, 3968, 384, 13824, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4992, 8064, 9728, 384, 0, 7552, 256, 2304, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[339, 0, 4078, 0, 0, 0, 3, 0, 9, 2, 4, 4, 4095, 0, 99, 0, 619, 0, 787, 0, 0, 1690, 14, 463, 0, 295, 0, 204, 2489, 4, 0, 3407, 154, 0, 1, 0, 0, 2, 105, 4074, 1918, 3042, 442, 1, 103, 0, 0, 2, 555, 3336, 0, 1, 5, 1, 0, 210, 196, 0, 9, 2, 3, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 1, 3808, 0, 0, 0, 157, 0, 129, 2, 0, 1412, 3922, 20, 15, 1, 2553, 1, 0, 5, 400, 119, 11, 11, 0, 0, 0, 12, 3196, 0, 2, 3656, 985, 0, 0, 0, 2, 0, 40, 3597, 0, 4018, 62, 0, 544, 0, 5, 15, 101, 1747, 17, 0, 0, 1, 26, 70, 837, 0, 1018, 0, 237, 0, 13, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 3969, 0, 0, 0, 0, 0, 67, 0, 0, 87, 3614, 0, 3, 1, 1268, 0, 473, 2, 0, 142, 20, 1, 0, 1, 0, 0, 3732, 0, 72, 3280, 143, 0, 0, 0, 0, 0, 178, 3660, 83, 4019, 1766, 0, 268, 0, 1, 0, 4, 3179, 0, 0, 8, 0, 6, 136, 747, 1, 1766, 0, 71, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 13, 3621, 0, 0, 69, 1, 0, 4, 0, 115, 56, 4083, 0, 1, 0, 2985, 0, 538, 2, 0, 38, 236, 77, 0, 15, 0, 0, 3449, 0, 3, 391, 1244, 0, 0, 0, 0, 0, 330, 3969, 115, 4063, 1308, 0, 17, 0, 4, 0, 168, 3477, 4, 1, 0, 0, 0, 151, 1368, 0, 805, 1, 46, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 496, 3734, 0, 0, 0, 2, 0, 0, 4, 0, 1112, 4081, 1, 2, 0, 901, 17, 138, 24, 0, 46, 10, 77, 4, 2, 0, 0, 3576, 0, 129, 3564, 51, 0, 0, 0, 0, 0, 181, 4089, 30, 3928, 852, 0, 75, 0, 9, 0, 73, 2850, 0, 3, 1, 0, 0, 13, 1733, 0, 921, 4, 34, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1, 231, 4025, 1, 0, 1, 0, 0, 1, 1, 4, 82, 4075, 0, 781, 0, 3568, 0, 120, 2, 0, 910, 0, 2, 0, 66, 1, 0, 1115, 0, 7, 2816, 180, 18, 0, 0, 0, 0, 85, 3929, 316, 3412, 1569, 0, 64, 0, 0, 1, 373, 3543, 0, 0, 0, 0, 0, 193, 428, 0, 68, 0, 774, 0, 5, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 1, 3943, 0, 0, 13, 0, 20, 56, 0, 164, 259, 4021, 6, 594, 0, 1624, 0, 31, 2, 15, 1999, 0, 0, 0, 2, 0, 161, 1663, 3, 0, 1993, 95, 1, 1, 0, 0, 0, 197, 3873, 1040, 4063, 706, 0, 4, 0, 55, 0, 9, 3940, 2, 0, 0, 0, 0, 1710, 305, 0, 169, 0, 28, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1, 43, 3289, 3, 1, 12, 19, 0, 27, 18, 1, 1839, 3868, 0, 210, 24, 610, 1, 426, 71, 1316, 2976, 26, 73, 0, 104, 187, 46, 810, 69, 59, 236, 94, 81, 7, 72, 2, 43, 58, 3728, 565, 4061, 107, 70, 1772, 3, 0, 0, 9, 3532, 158, 184, 0, 2, 0, 107, 373, 4, 779, 0, 547, 21, 23, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 43, 3289, 3, 1, 12, 19, 0, 108, 0, 0, 0, 0, 0, 0, 0, 27, 18, 1, 1839, 3107, 0, 210, 24, 0, 0, 0, 0, 0, 0, 0, 0, 610, 1, 426, 71, 1316, 2976, 26, 73, 608, 19, 0, 0, 0, 0, 0, 0, 0, 104, 187, 46, 476, 69, 59, 236, 0, 0, 0, 0, 0, 0, 0, 0, 94, 81, 7, 72, 2, 43, 58, 3419, 0, 0, 0, 0, 0, 0, 0, 0, 565, 2501, 107, 70, 1772, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 3532, 158, 184, 0, 2, 0, 107, 201, 153, 0, 0, 0, 0, 0, 0, 373, 4, 779, 0, 547, 21, 23, 1, 1560, 315, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3476, 5226, 6126, 1177, 3776, 5018, 4346, 3623] avg=4096.0000 max=6126 max_rank=2 min=1177 min_rank=3 imbalance_ratio=1.495605 max_load_reduction=26642 max_load_reduction_pct=0.813049 moved_to_echo=16763 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 13, 3621, 0, 0, 69, 1, 0, 112, 0, 0, 0, 0, 0, 0, 0, 4, 0, 115, 56, 3286, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2985, 0, 538, 2, 0, 38, 236, 77, 639, 60, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 2064, 0, 3, 391, 0, 0, 0, 0, 0, 0, 0, 0, 1244, 0, 0, 0, 0, 0, 330, 3649, 0, 0, 0, 0, 0, 0, 0, 0, 115, 2506, 1308, 0, 17, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 3477, 4, 1, 0, 0, 0, 151, 208, 158, 0, 0, 0, 0, 0, 0, 1368, 0, 805, 1, 46, 0, 0, 0, 1557, 1325, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3816, 3462, 4575, 2473, 5223, 3950, 4167, 5102] avg=4096.0000 max=5223 max_rank=4 min=2473 min_rank=3 imbalance_ratio=1.275146 max_load_reduction=27545 max_load_reduction_pct=0.840607 moved_to_echo=18442 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 496, 3734, 0, 0, 0, 2, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1112, 3284, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 901, 17, 138, 24, 0, 46, 10, 77, 639, 63, 0, 0, 0, 0, 0, 0, 4, 2, 0, 0, 2139, 0, 129, 3564, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 181, 3759, 0, 0, 0, 0, 0, 0, 0, 0, 30, 2423, 852, 0, 75, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 2850, 0, 3, 1, 0, 0, 13, 215, 158, 0, 0, 0, 0, 0, 0, 1733, 0, 921, 4, 34, 0, 1, 0, 1505, 1374, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4347, 4403, 1915, 5838, 3991, 3389, 3313, 5572] avg=4096.0000 max=5838 max_rank=3 min=1915 min_rank=2 imbalance_ratio=1.425293 max_load_reduction=26930 max_load_reduction_pct=0.821838 moved_to_echo=16265 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3969, 0, 0, 0, 0, 0, 103, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 87, 2908, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1268, 0, 473, 2, 0, 142, 20, 1, 566, 65, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2233, 0, 72, 3280, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 0, 0, 0, 0, 178, 3365, 0, 0, 0, 0, 0, 0, 0, 0, 83, 2479, 1766, 0, 268, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3179, 0, 0, 8, 0, 6, 136, 192, 140, 0, 0, 0, 0, 0, 0, 747, 1, 1766, 0, 71, 0, 0, 0, 1540, 1434, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 3066, 2537, 5586, 3686, 4597, 3665, 5559] avg=4096.0000 max=5586 max_rank=3 min=2537 min_rank=2 imbalance_ratio=1.363770 max_load_reduction=27182 max_load_reduction_pct=0.829529 moved_to_echo=17507 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 3808, 0, 0, 0, 157, 0, 101, 0, 0, 0, 0, 0, 0, 0, 129, 2, 0, 1412, 3156, 20, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2553, 1, 0, 5, 400, 119, 11, 11, 614, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1912, 0, 2, 3656, 0, 0, 0, 0, 0, 0, 0, 0, 985, 0, 0, 0, 2, 0, 40, 3307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2479, 62, 0, 544, 0, 5, 15, 0, 0, 0, 0, 0, 0, 0, 0, 101, 1747, 17, 0, 0, 1, 26, 70, 189, 152, 0, 0, 0, 0, 0, 0, 837, 0, 1018, 0, 237, 0, 13, 0, 1539, 1228, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4735, 3770, 5582, 4334, 3105, 2303, 4872] avg=4096.0000 max=5582 max_rank=3 min=2303 min_rank=6 imbalance_ratio=1.362793 max_load_reduction=27186 max_load_reduction_pct=0.829651 moved_to_echo=14614 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[339, 0, 4078, 0, 0, 0, 3, 0, 115, 0, 0, 0, 0, 0, 0, 0, 9, 2, 4, 4, 3295, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 619, 0, 787, 0, 0, 1690, 14, 463, 641, 43, 0, 0, 0, 0, 0, 0, 0, 295, 0, 204, 1490, 4, 0, 3407, 0, 0, 0, 0, 0, 0, 0, 0, 154, 0, 1, 0, 0, 2, 105, 3745, 0, 0, 0, 0, 0, 0, 0, 0, 1918, 1877, 442, 1, 103, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 555, 3336, 0, 1, 5, 1, 0, 210, 214, 159, 0, 0, 0, 0, 0, 0, 196, 0, 9, 2, 3, 0, 1, 0, 1165, 956, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4535, 3413, 4257, 5400, 4007, 4343, 4481, 2332] avg=4096.0000 max=5400 max_rank=3 min=2332 min_rank=7 imbalance_ratio=1.318359 max_load_reduction=27368 max_load_reduction_pct=0.835205 moved_to_echo=15163 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 231, 4025, 1, 0, 1, 0, 0, 110, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4, 82, 3279, 0, 781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3568, 0, 120, 2, 0, 910, 0, 2, 638, 19, 0, 0, 0, 0, 0, 0, 0, 66, 1, 0, 668, 0, 7, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 180, 18, 0, 0, 0, 0, 85, 3613, 0, 0, 0, 0, 0, 0, 0, 0, 316, 2105, 1569, 0, 64, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 373, 3543, 0, 0, 0, 0, 0, 193, 206, 158, 0, 0, 0, 0, 0, 0, 428, 0, 68, 0, 774, 0, 5, 0, 1307, 428, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4369, 4148, 5259, 3558, 3896, 4055, 4473, 3010] avg=4096.0000 max=5259 max_rank=2 min=3010 min_rank=7 imbalance_ratio=1.283936 max_load_reduction=27509 max_load_reduction_pct=0.839508 moved_to_echo=15434 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 3943, 0, 0, 13, 0, 20, 109, 0, 0, 0, 0, 0, 0, 0, 56, 0, 164, 259, 3236, 6, 594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1624, 0, 31, 2, 15, 1999, 0, 0, 629, 29, 0, 0, 0, 0, 0, 0, 0, 2, 0, 161, 995, 3, 0, 1993, 0, 0, 0, 0, 0, 0, 0, 0, 95, 1, 1, 0, 0, 0, 197, 3561, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 2506, 706, 0, 4, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 3940, 2, 0, 0, 0, 0, 1710, 203, 156, 0, 0, 0, 0, 0, 0, 305, 0, 169, 0, 28, 0, 0, 0, 1557, 639, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4315, 4329, 3154, 3855, 4311, 6020, 2698] avg=4096.0000 max=6020 max_rank=6 min=2698 min_rank=7 imbalance_ratio=1.469727 max_load_reduction=26748 max_load_reduction_pct=0.816284 moved_to_echo=16884 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[6016, 128, 5632, 128, 1792, 128, 128, 128, 11776, 7808, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 512, 256, 512, 12032, 128, 384, 19456, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[14208, 128, 2560, 128, 1792, 7936, 384, 768, 4992, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 18944, 6912, 128, 2944, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[384, 128, 384, 4864, 25600, 128, 1792, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[1408, 25728, 256, 256, 128, 128, 128, 2688, 1664, 1280, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[3072, 128, 128, 128, 128, 128, 1280, 28544, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[384, 896, 30592, 128, 128, 128, 256, 128, 896, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[199, 1, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 19, 0, 0, 0, 0, 8, 3932, 4, 4042, 2, 0, 0, 0, 5, 0, 4, 0, 0, 0, 0, 4085, 0, 0, 404, 0, 7, 0, 0, 0, 0, 0, 0, 0, 2, 3739, 4052, 0, 0, 4081, 0, 32, 0, 0, 122, 0, 3839, 0, 0, 0, 4, 0, 100, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[136, 0, 46, 0, 0, 1, 7, 0, 0, 3883, 0, 1529, 0, 0, 0, 211, 6, 3564, 15, 1119, 1, 0, 0, 0, 0, 0, 3, 14, 816, 345, 0, 4075, 0, 3, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3741, 0, 107, 3986, 0, 676, 2, 137, 0, 0, 4058, 0, 1, 1, 0, 0, 46, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[4, 0, 0, 0, 200, 12, 0, 1, 0, 4096, 0, 41, 0, 0, 0, 0, 2772, 308, 1, 850, 12, 0, 0, 0, 0, 0, 0, 0, 70, 531, 2, 3866, 69, 77, 0, 0, 0, 140, 0, 0, 1, 0, 0, 0, 0, 4088, 4087, 0, 50, 4066, 0, 3909, 0, 1, 0, 0, 3514, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[12, 0, 1, 0, 200, 0, 4, 0, 0, 3949, 0, 0, 0, 0, 6, 0, 579, 3595, 0, 2821, 5, 0, 0, 0, 0, 0, 2, 0, 0, 530, 0, 3715, 0, 0, 0, 0, 51, 0, 1, 0, 0, 0, 0, 0, 0, 4054, 4045, 0, 13, 4035, 1, 927, 0, 1, 49, 0, 3880, 68, 0, 0, 0, 42, 179, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[10, 0, 98, 0, 53, 1, 0, 0, 0, 4046, 0, 65, 0, 0, 0, 0, 72, 4096, 27, 1893, 0, 0, 0, 0, 0, 0, 2, 0, 224, 269, 0, 4092, 0, 4, 14, 2, 0, 0, 0, 0, 0, 0, 0, 34, 0, 3097, 4028, 0, 1, 4001, 0, 2196, 3, 0, 0, 1, 4078, 81, 0, 0, 0, 0, 274, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 6, 0, 0, 4086, 0, 1182, 0, 0, 0, 0, 25, 3390, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 0, 3785, 0, 37, 0, 0, 76, 4, 1, 0, 0, 0, 0, 0, 7, 4040, 4080, 0, 0, 4063, 9, 75, 0, 0, 0, 0, 3716, 0, 6, 0, 16, 1, 25, 62] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 119, 0, 188, 0, 0, 0, 0, 4046, 0, 21, 0, 0, 0, 0, 3998, 333, 0, 1563, 41, 0, 0, 0, 0, 0, 1, 0, 236, 384, 0, 4032, 0, 0, 19, 3, 0, 0, 124, 0, 0, 0, 0, 0, 0, 4025, 3966, 0, 0, 4090, 0, 2147, 0, 0, 0, 0, 3369, 4, 0, 0, 0, 0, 58, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[8, 0, 0, 28, 0, 0, 1, 0, 26, 4054, 0, 918, 0, 0, 0, 54, 37, 2949, 0, 2189, 7, 181, 0, 0, 0, 0, 439, 0, 39, 0, 62, 3955, 0, 0, 0, 0, 0, 0, 41, 0, 1, 221, 45, 0, 3, 3534, 4054, 6, 695, 4096, 2, 54, 7, 25, 89, 0, 2193, 5, 0, 0, 93, 0, 2657, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 0, 0, 200, 12, 0, 1, 200, 1377, 450, 297, 0, 0, 0, 0, 0, 3646, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2772, 91, 1, 850, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 531, 2, 3522, 0, 0, 0, 0, 0, 0, 0, 0, 69, 77, 0, 0, 0, 140, 0, 0, 3860, 17, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4088, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 2689, 0, 3909, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3514, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2541, 3687, 3726, 4125, 4163, 4316, 6649, 3561] avg=4096.0000 max=6649 max_rank=6 min=2541 min_rank=0 imbalance_ratio=1.623291 max_load_reduction=26119 max_load_reduction_pct=0.797089 moved_to_echo=18689 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[136, 0, 46, 0, 0, 1, 7, 0, 2325, 1350, 427, 313, 0, 0, 0, 0, 0, 3456, 0, 1529, 0, 0, 0, 211, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1035, 15, 1119, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 14, 816, 345, 0, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 143, 0, 0, 0, 0, 0, 3533, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 2636, 0, 676, 2, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 1, 1, 0, 0, 46, 0, 50, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4605, 5196, 2176, 4890, 3883, 4304, 3558, 4156] avg=4096.0000 max=5196 max_rank=1 min=2176 min_rank=2 imbalance_ratio=1.268555 max_load_reduction=27572 max_load_reduction_pct=0.841431 moved_to_echo=15901 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[199, 1, 0, 0, 0, 0, 0, 0, 2565, 1382, 449, 314, 0, 0, 0, 0, 0, 3636, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1142, 4, 4042, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4, 0, 0, 0, 0, 3721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 404, 0, 7, 0, 0, 0, 3827, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3739, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2699, 0, 32, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3839, 0, 0, 0, 4, 0, 100, 0, 50, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4910, 3655, 5198, 3730, 4463, 3966, 2853, 3993] avg=4096.0000 max=5198 max_rank=2 min=2853 min_rank=6 imbalance_ratio=1.269043 max_load_reduction=27570 max_load_reduction_pct=0.841370 moved_to_echo=15275 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 1, 0, 200, 0, 4, 0, 2346, 1366, 434, 286, 0, 0, 0, 0, 0, 3515, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 579, 1044, 0, 2821, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 530, 0, 3384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 1, 0, 3820, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 2669, 1, 927, 0, 1, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3880, 68, 0, 0, 0, 42, 179, 3, 45, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4649, 3521, 4449, 3916, 4077, 4279, 3660, 4217] avg=4096.0000 max=4649 max_rank=0 min=3521 min_rank=1 imbalance_ratio=1.135010 max_load_reduction=28119 max_load_reduction_pct=0.858124 moved_to_echo=16233 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 98, 0, 53, 1, 0, 0, 2673, 1355, 445, 315, 0, 0, 0, 0, 0, 3601, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 1189, 27, 1893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 224, 269, 0, 3727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 14, 2, 0, 0, 0, 0, 3804, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 3097, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2646, 0, 2196, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 81, 0, 0, 0, 0, 274, 6, 50, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4950, 3666, 3181, 4222, 4058, 3355, 4847, 4489] avg=4096.0000 max=4950 max_rank=0 min=3181 min_rank=2 imbalance_ratio=1.208496 max_load_reduction=27818 max_load_reduction_pct=0.848938 moved_to_echo=16749 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 6, 0, 2212, 1376, 449, 291, 0, 0, 0, 0, 0, 3637, 0, 1182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 984, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 0, 3448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 76, 4, 1, 0, 3854, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4040, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2687, 9, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3716, 0, 6, 0, 16, 1, 25, 62, 46, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4335, 4819, 5076, 3456, 4166, 4273, 2771, 3872] avg=4096.0000 max=5076 max_rank=2 min=2771 min_rank=6 imbalance_ratio=1.239258 max_load_reduction=27692 max_load_reduction_pct=0.845093 moved_to_echo=15082 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 119, 0, 188, 0, 0, 0, 217, 1385, 445, 310, 0, 0, 0, 0, 0, 3601, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3998, 97, 0, 1563, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 236, 384, 0, 3673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 3, 0, 0, 124, 0, 3746, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2705, 0, 2147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3369, 4, 0, 0, 0, 0, 58, 0, 49, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2665, 3622, 5699, 4294, 3911, 4245, 4852, 3480] avg=4096.0000 max=5699 max_rank=2 min=2665 min_rank=0 imbalance_ratio=1.391357 max_load_reduction=27069 max_load_reduction_pct=0.826080 moved_to_echo=16488 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 0, 28, 0, 0, 1, 0, 1928, 1390, 450, 308, 0, 0, 0, 0, 26, 3604, 0, 918, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 37, 849, 0, 2189, 7, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 439, 0, 39, 0, 62, 3595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 3834, 172, 0, 0, 0, 0, 0, 0, 1, 221, 45, 0, 3, 3534, 220, 6, 0, 0, 0, 0, 0, 0, 0, 0, 695, 2706, 2, 54, 7, 25, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2193, 5, 0, 0, 93, 0, 2657, 0, 52, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4602, 3263, 4135, 4047, 4030, 3578, 5000] avg=4096.0000 max=5000 max_rank=7 min=3263 min_rank=2 imbalance_ratio=1.220703 max_load_reduction=27768 max_load_reduction_pct=0.847412 moved_to_echo=16655 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33920 observed_loss=-1152 tokens_per_expert=[384, 128, 384, 128, 768, 128, 128, 128, 14592, 11008, 3584, 2560, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[896, 21504, 128, 10112, 128, 256, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[7552, 6528, 128, 18560, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 28800, 0, 3840, 0, 0, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[28672, 256, 128, 128, 128, 128, 3456, 128, 512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 512, 128, 1408, 2176, 128, 28800, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 640, 128, 256, 256, 256, 0, 30336, 1280, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 256, 128, 128, 128, 30720, 1792, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33920 slack_slots=1792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4094, 0, 0, 2458, 4083, 1, 0, 0, 3, 0, 0, 133, 0, 2773, 4047, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 217, 0, 1, 2132, 4091, 32, 0, 0, 0, 0, 4089, 0, 0, 0, 497, 0, 0, 0, 1, 0, 0, 3, 0, 5, 0, 0, 0, 0, 0, 0, 4089, 3, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4040, 0, 0, 448, 3915, 0, 0, 0, 0, 0, 0, 952, 0, 2014, 3931, 0, 0, 8, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2632, 4084, 0, 0, 2, 0, 0, 4096, 6, 0, 10, 2476, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4076, 74, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4091, 0, 0, 3, 3933, 0, 0, 9, 0, 0, 0, 2735, 0, 1716, 3984, 0, 63, 0, 1, 19, 0, 0, 14, 0, 5, 24, 0, 3, 0, 145, 386, 3839, 1, 0, 0, 0, 0, 3555, 0, 0, 28, 4057, 34, 0, 1, 0, 1, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 4007, 36, 4, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[4, 0, 4096, 0, 7, 0, 4094, 0, 0, 0, 0, 0, 0, 40, 2, 3995, 4083, 0, 32, 0, 0, 0, 0, 22, 1, 0, 22, 0, 0, 9, 0, 1, 1303, 4089, 1, 3, 0, 16, 0, 4096, 0, 0, 9, 2723, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 1, 0, 0, 0, 0, 4088, 13, 0, 0, 15] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4086, 0, 0, 0, 3817, 42, 0, 0, 140, 0, 0, 1537, 0, 3157, 4049, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 3950, 0, 0, 0, 0, 0, 2851, 0, 0, 113, 3940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 898, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[6, 0, 3877, 0, 0, 23, 3601, 18, 0, 0, 1, 0, 0, 86, 0, 4030, 3173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1561, 0, 462, 63, 4032, 156, 17, 0, 0, 0, 4089, 0, 0, 263, 3114, 0, 0, 0, 0, 8, 0, 0, 0, 0, 48, 1, 0, 0, 5, 0, 4084, 4, 0, 0, 46] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 2566, 0, 283, 2434, 3793, 2, 0, 0, 0, 0, 4, 9, 0, 3164, 3928, 13, 1, 8, 111, 76, 0, 0, 634, 0, 16, 45, 3, 1, 0, 0, 3216, 4083, 1, 0, 0, 0, 0, 2818, 0, 0, 0, 55, 0, 1, 0, 0, 137, 0, 0, 0, 0, 53, 1712, 0, 0, 0, 0, 3539, 59, 0, 0, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[3, 0, 3880, 0, 0, 14, 4038, 137, 0, 91, 0, 1, 0, 3036, 2, 1513, 3894, 0, 1, 0, 0, 0, 0, 0, 1, 1, 104, 3, 0, 844, 0, 1, 649, 4094, 15, 0, 0, 0, 0, 2849, 0, 0, 0, 3505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4091, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 3393, 0, 0, 23, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 86, 0, 4030, 205, 0, 0, 0, 0, 0, 0, 0, 3173, 0, 0, 0, 0, 0, 0, 0, 98, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1561, 0, 462, 3117, 192, 0, 0, 0, 0, 0, 0, 63, 0, 156, 17, 0, 0, 0, 3184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 263, 3114, 0, 0, 0, 0, 905, 484, 181, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 48, 1, 0, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4025, 4, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3440, 4322, 3330, 5332, 3420, 4947, 3897, 4080] avg=4096.0000 max=5332 max_rank=3 min=3330 min_rank=2 imbalance_ratio=1.301758 max_load_reduction=27436 max_load_reduction_pct=0.837280 moved_to_echo=16344 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3583, 0, 0, 2458, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 133, 0, 2773, 216, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 16, 0, 0, 0, 0, 104, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 217, 0, 1, 3535, 195, 0, 0, 0, 0, 0, 0, 2132, 0, 32, 0, 0, 0, 0, 3184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 0, 0, 0, 1, 905, 548, 191, 0, 0, 0, 0, 0, 0, 0, 3, 0, 5, 0, 0, 0, 3896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4030, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6042, 3125, 4226, 3948, 5348, 2142, 3904, 4033] avg=4096.0000 max=6042 max_rank=0 min=2142 min_rank=5 imbalance_ratio=1.475098 max_load_reduction=26726 max_load_reduction_pct=0.815613 moved_to_echo=15427 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3580, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 2735, 0, 1716, 216, 0, 0, 0, 0, 0, 0, 0, 3984, 0, 63, 0, 1, 19, 0, 0, 104, 57, 0, 0, 0, 0, 0, 0, 14, 0, 5, 24, 0, 3, 0, 145, 3405, 183, 0, 0, 0, 0, 0, 0, 386, 0, 1, 0, 0, 0, 0, 2768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 4057, 34, 0, 1, 0, 787, 528, 191, 0, 0, 0, 0, 0, 1, 0, 73, 0, 0, 0, 0, 0, 3656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3950, 36, 4, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3583, 4676, 4228, 3779, 3155, 5626, 3730, 3991] avg=4096.0000 max=5626 max_rank=5 min=3155 min_rank=4 imbalance_ratio=1.373535 max_load_reduction=27142 max_load_reduction_pct=0.828308 moved_to_echo=16502 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 3585, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 2, 3995, 216, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 32, 0, 0, 0, 0, 22, 104, 59, 0, 0, 0, 0, 0, 0, 1, 0, 22, 0, 0, 9, 0, 1, 3545, 196, 0, 0, 0, 0, 0, 0, 1303, 0, 1, 3, 0, 16, 0, 3189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2723, 0, 0, 0, 0, 907, 549, 191, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 1, 0, 3893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 13, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3596, 4253, 4300, 3774, 4512, 4379, 3897, 4057] avg=4096.0000 max=4512 max_rank=4 min=3596 min_rank=0 imbalance_ratio=1.101562 max_load_reduction=28256 max_load_reduction_pct=0.862305 moved_to_echo=16845 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3536, 0, 0, 448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 952, 0, 2014, 213, 0, 0, 0, 0, 0, 0, 0, 3931, 0, 0, 8, 0, 0, 0, 0, 102, 58, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3389, 194, 0, 0, 0, 0, 0, 0, 2632, 0, 0, 0, 2, 0, 0, 3189, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 10, 2476, 1, 0, 0, 0, 907, 526, 189, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4018, 74, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3984, 3179, 4099, 3584, 5823, 4115, 3891, 4093] avg=4096.0000 max=5823 max_rank=4 min=3179 min_rank=1 imbalance_ratio=1.421631 max_load_reduction=26945 max_load_reduction_pct=0.822296 moved_to_echo=17922 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3576, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 1537, 0, 3157, 216, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 62, 0, 0, 0, 0, 0, 103, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 3305, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 3940, 0, 0, 0, 0, 631, 512, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3618, 5050, 4273, 3525, 2220, 5387, 3761, 4934] avg=4096.0000 max=5387 max_rank=5 min=2220 min_rank=4 imbalance_ratio=1.315186 max_load_reduction=27381 max_load_reduction_pct=0.835602 moved_to_echo=16302 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 3396, 0, 0, 14, 0, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 1, 0, 3036, 2, 1513, 205, 0, 0, 0, 0, 0, 0, 0, 3894, 0, 1, 0, 0, 0, 0, 0, 98, 59, 0, 0, 0, 0, 0, 0, 1, 1, 104, 3, 0, 844, 0, 1, 3497, 196, 0, 0, 0, 0, 0, 0, 649, 0, 15, 0, 0, 0, 0, 2218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3505, 0, 0, 0, 0, 631, 541, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3898, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3550, 4848, 4052, 4647, 2882, 4858, 3898, 4033] avg=4096.0000 max=4858 max_rank=5 min=2882 min_rank=4 imbalance_ratio=1.186035 max_load_reduction=27910 max_load_reduction_pct=0.851746 moved_to_echo=15671 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2236, 0, 283, 2434, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 9, 0, 3164, 138, 0, 0, 0, 0, 0, 0, 0, 3928, 13, 1, 8, 111, 76, 0, 0, 69, 54, 0, 0, 0, 0, 0, 0, 634, 0, 16, 45, 3, 1, 0, 0, 3285, 195, 0, 0, 0, 0, 0, 0, 3216, 0, 1, 0, 0, 0, 0, 2191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 1, 0, 0, 627, 508, 123, 0, 0, 0, 0, 0, 137, 0, 0, 0, 0, 53, 1712, 0, 3888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3485, 59, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4955, 3315, 4260, 4179, 5408, 1314, 5790, 3547] avg=4096.0000 max=5790 max_rank=6 min=1314 min_rank=5 imbalance_ratio=1.413574 max_load_reduction=26978 max_load_reduction_pct=0.823303 moved_to_echo=16059 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[128, 0, 512, 20480, 128, 128, 128, 128, 6400, 4224, 1536, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[10496, 0, 256, 128, 128, 128, 0, 22144, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 27008, 0, 384, 5504, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 256, 128, 128, 8576, 128, 22400, 1664, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[768, 128, 256, 128, 128, 2688, 0, 640, 27136, 1664, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[31104, 128, 256, 128, 128, 128, 0, 128, 896, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[256, 128, 128, 128, 128, 128, 1792, 0, 30848, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 31616, 1152, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33792 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4028, 0, 0, 4068, 0, 0, 0, 0, 0, 835, 0, 0, 3, 4095, 0, 1, 0, 0, 4082, 3946, 0, 0, 0, 0, 81, 35, 0, 0, 0, 0, 1, 31, 4, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 2, 1881, 4091, 0, 0, 0, 0, 0, 191, 0, 0, 0, 4094, 0, 0, 0, 0, 1267] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 3989, 205, 6, 3936, 2, 231, 0, 0, 0, 0, 0, 0, 0, 4090, 14, 0, 0, 0, 4096, 4071, 0, 0, 0, 0, 0, 30, 0, 0, 4, 0, 0, 0, 1, 0, 0, 1, 2, 0, 0, 325, 0, 0, 0, 8, 3137, 3822, 0, 0, 1, 1, 0, 1246, 30, 0, 0, 3505, 0, 0, 15, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[15, 1, 11, 3287, 7, 0, 4084, 0, 441, 0, 0, 0, 1, 0, 0, 1, 4023, 0, 0, 0, 0, 4003, 3527, 0, 0, 0, 0, 35, 27, 0, 0, 264, 0, 0, 0, 0, 1, 0, 3, 11, 0, 1, 14, 0, 1, 7, 4, 3694, 3837, 0, 0, 0, 1, 46, 1284, 0, 0, 0, 4078, 0, 0, 20, 0, 39] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[2, 0, 0, 4087, 0, 8, 4091, 0, 13, 0, 0, 0, 486, 0, 0, 0, 4036, 0, 3, 0, 5, 4078, 3912, 9, 47, 0, 0, 492, 40, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 100, 0, 0, 0, 0, 3144, 4095, 0, 7, 3, 0, 7, 16, 0, 0, 0, 4056, 0, 0, 4, 0, 13] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 4, 1, 3539, 175, 0, 4048, 0, 46, 0, 3, 0, 37, 0, 0, 0, 4091, 0, 0, 0, 0, 4095, 4067, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 26, 0, 1, 0, 1, 0, 2, 0, 3, 4053, 4087, 12, 0, 0, 6, 0, 0, 0, 21, 0, 4094, 0, 0, 0, 0, 351] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 3993, 0, 0, 3376, 0, 78, 0, 0, 0, 17, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4078, 1, 0, 0, 0, 2, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 6, 0, 0, 0, 887, 4096, 1, 8, 0, 0, 0, 3836, 0, 0, 0, 4095, 0, 0, 0, 0, 76] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[2, 0, 0, 4093, 4, 1, 4055, 20, 0, 0, 0, 1, 98, 0, 0, 0, 4039, 0, 3, 0, 0, 3959, 4087, 1, 0, 0, 0, 59, 0, 0, 0, 438, 0, 0, 6, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 2, 3491, 3913, 0, 0, 0, 0, 0, 372, 0, 4, 0, 4095, 1, 0, 0, 0, 18] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[2, 0, 2, 0, 14, 0, 3584, 0, 337, 0, 0, 293, 3182, 0, 0, 334, 3841, 0, 4, 2, 2, 4096, 3631, 12, 229, 0, 5, 7, 525, 200, 0, 151, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 13, 4, 6, 0, 0, 3988, 3750, 0, 0, 2, 0, 366, 93, 0, 22, 10, 3428, 124, 0, 504, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3989, 205, 6, 665, 2, 0, 0, 0, 0, 0, 0, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 3230, 87, 0, 0, 0, 0, 0, 0, 178, 14, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 4, 3806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 2, 4096, 19, 0, 0, 0, 0, 0, 0, 0, 0, 325, 0, 0, 0, 8, 3137, 778, 130, 41, 0, 0, 0, 0, 0, 3044, 0, 0, 1, 1, 0, 1246, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3375, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4867, 3548, 4263, 3840, 4119, 4419, 4322, 3390] avg=4096.0000 max=4867 max_rank=0 min=3390 min_rank=7 imbalance_ratio=1.188232 max_load_reduction=27901 max_load_reduction_pct=0.851471 moved_to_echo=16250 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4028, 0, 0, 688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 835, 0, 0, 3, 3338, 87, 0, 0, 0, 0, 0, 0, 178, 0, 1, 0, 0, 0, 3946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 35, 0, 0, 0, 3811, 0, 0, 0, 0, 0, 0, 0, 0, 1, 31, 4, 0, 0, 0, 0, 4082, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 2, 1881, 832, 151, 42, 0, 0, 0, 0, 0, 3259, 0, 0, 0, 0, 0, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3943, 0, 0, 0, 0, 1267, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4716, 4263, 4125, 3927, 4137, 2940, 3450, 5210] avg=4096.0000 max=5210 max_rank=7 min=2940 min_rank=5 imbalance_ratio=1.271973 max_load_reduction=27558 max_load_reduction_pct=0.841003 moved_to_echo=15737 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 4087, 0, 8, 691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 486, 0, 0, 0, 3357, 85, 0, 0, 0, 0, 0, 0, 176, 0, 3, 0, 5, 0, 3912, 9, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 492, 40, 1, 0, 0, 3756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 19, 0, 0, 0, 0, 0, 0, 13, 0, 100, 0, 0, 0, 0, 3144, 833, 150, 43, 0, 0, 0, 0, 0, 3262, 0, 7, 3, 0, 7, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3906, 0, 0, 4, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4788, 3941, 4105, 4336, 4097, 4283, 3295, 3923] avg=4096.0000 max=4788 max_rank=0 min=3295 min_rank=6 imbalance_ratio=1.168945 max_load_reduction=27980 max_load_reduction_pct=0.853882 moved_to_echo=15598 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[15, 1, 11, 3287, 7, 0, 689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 441, 0, 0, 0, 1, 0, 0, 1, 3352, 85, 0, 0, 0, 0, 0, 0, 175, 0, 0, 0, 0, 0, 3527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 27, 0, 0, 264, 3744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 11, 4003, 19, 0, 0, 0, 0, 0, 0, 0, 1, 14, 0, 1, 7, 4, 3694, 781, 151, 43, 0, 0, 0, 0, 0, 3056, 0, 0, 0, 1, 46, 1284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3927, 0, 0, 20, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4010, 3880, 3702, 4070, 4037, 4696, 4387, 3986] avg=4096.0000 max=4696 max_rank=5 min=3702 min_rank=2 imbalance_ratio=1.146484 max_load_reduction=28072 max_load_reduction_pct=0.856689 moved_to_echo=17106 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 4093, 4, 1, 685, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 98, 0, 0, 0, 3328, 86, 0, 0, 0, 0, 0, 0, 175, 0, 3, 0, 0, 0, 4087, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 438, 3759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 6, 0, 3959, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3491, 796, 151, 42, 0, 0, 0, 0, 0, 3117, 0, 0, 0, 0, 0, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3944, 1, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4805, 3513, 4266, 4256, 3990, 4482, 3489, 3967] avg=4096.0000 max=4805 max_rank=0 min=3489 min_rank=6 imbalance_ratio=1.173096 max_load_reduction=27963 max_load_reduction_pct=0.853363 moved_to_echo=15928 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 1, 3539, 175, 0, 684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 3, 0, 37, 0, 0, 0, 3322, 87, 0, 0, 0, 0, 0, 0, 178, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 26, 0, 4095, 19, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 2, 0, 3, 4053, 832, 151, 42, 0, 0, 0, 0, 0, 3255, 12, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 3943, 0, 0, 0, 0, 351, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4403, 3495, 4245, 3808, 4144, 5085, 3273, 4315] avg=4096.0000 max=5085 max_rank=5 min=3273 min_rank=6 imbalance_ratio=1.241455 max_load_reduction=27683 max_load_reduction_pct=0.844818 moved_to_echo=16817 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3993, 0, 0, 571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 17, 0, 0, 0, 2770, 87, 0, 0, 0, 0, 0, 0, 178, 0, 0, 0, 0, 0, 4078, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 21, 0, 0, 0, 3812, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 887, 833, 151, 35, 0, 0, 0, 0, 0, 3263, 1, 8, 0, 0, 0, 3836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3944, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4564, 2952, 4257, 3835, 4120, 1912, 7108, 4020] avg=4096.0000 max=7108 max_rank=6 min=1912 min_rank=5 imbalance_ratio=1.735352 max_load_reduction=25660 max_load_reduction_pct=0.783081 moved_to_echo=17160 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 2, 0, 14, 0, 597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 337, 0, 0, 293, 3182, 0, 0, 334, 2946, 84, 0, 0, 0, 0, 0, 0, 154, 0, 4, 2, 2, 0, 3631, 12, 0, 0, 0, 0, 0, 0, 0, 0, 229, 0, 5, 7, 525, 200, 0, 151, 3579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 24, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4, 6, 0, 0, 3988, 767, 132, 41, 0, 0, 0, 0, 0, 2983, 0, 0, 2, 0, 366, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 10, 3296, 124, 0, 504, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[615, 7176, 3805, 4696, 4124, 4951, 3444, 3957] avg=4096.0000 max=7176 max_rank=1 min=615 min_rank=0 imbalance_ratio=1.751953 max_load_reduction=25592 max_load_reduction_pct=0.781006 moved_to_echo=16476 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 27136, 512, 128, 5376, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[128, 128, 512, 128, 128, 128, 128, 24320, 6528, 1280, 384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1152, 0, 128, 384, 4736, 0, 0, 384, 25728, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[25344, 128, 128, 128, 128, 512, 7040, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1408, 128, 128, 128, 128, 0, 31360, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 30336, 128, 0, 640, 0, 1792, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[384, 0, 128, 768, 768, 256, 0, 896, 30080, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 128, 0, 128, 128, 32512, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33792 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 16, 4096, 3984, 0, 1460, 0, 0, 0, 0, 0, 0, 0, 3336, 43, 0, 7, 2288, 76, 570, 0, 0, 2547, 14, 0, 65, 2365, 0, 0, 0, 0, 3821, 0, 0, 0, 0, 4008, 0, 44, 3891, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4072, 4005, 0, 4075, 0, 0, 0, 0, 0, 0, 1, 944, 11, 0, 0, 3, 110, 0, 0, 0, 3825, 3, 415, 8, 3397, 0, 2, 153, 0, 3612, 0, 0, 0, 0, 3955, 0, 0, 4043, 39, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 25, 0, 2, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 3, 4088, 4012, 0, 2419, 3, 0, 0, 1, 0, 0, 33, 394, 0, 0, 0, 0, 0, 4, 0, 0, 4033, 64, 171, 803, 3964, 0, 0, 470, 0, 4054, 13, 6, 0, 18, 4080, 14, 0, 4096, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 1, 1, 1, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4094, 4086, 0, 4085, 0, 0, 0, 1, 0, 0, 0, 4049, 0, 0, 0, 571, 1, 1, 0, 0, 4062, 510, 1700, 171, 1079, 0, 3, 27, 0, 119, 0, 0, 0, 22, 4069, 0, 2, 4095, 0, 3, 0, 1, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4093, 4070, 0, 4014, 0, 0, 0, 3, 0, 0, 0, 4075, 2, 0, 0, 82, 0, 1, 0, 0, 4091, 0, 1742, 0, 1386, 0, 0, 0, 0, 1093, 0, 0, 0, 0, 3963, 0, 1, 4078, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 6, 0, 0, 0, 0, 0, 46, 8, 12, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 4090, 0, 0, 0, 0, 0, 32, 0, 809, 0, 0, 0, 0, 0, 102, 0, 0, 3989, 60, 3095, 6, 8, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 4090, 0, 72, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 57, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4085, 0, 347, 2, 0, 4, 0, 0, 31, 0, 4051, 0, 0, 0, 90, 0, 0, 0, 0, 4078, 1129, 3643, 0, 662, 0, 0, 0, 0, 2294, 5, 0, 0, 0, 4086, 0, 0, 4089, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 3, 3, 0, 0, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4013, 4067, 0, 0, 37, 0, 69, 138, 0, 0, 0, 3924, 204, 0, 0, 0, 0, 0, 0, 0, 1508, 51, 3, 0, 3830, 0, 5, 0, 0, 3992, 0, 0, 33, 90, 3814, 0, 0, 3921, 7, 0, 189, 0, 1, 0, 0, 0, 0, 0, 0, 2, 32, 0, 0, 0, 9, 2798, 30, 0, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 16, 0, 1503, 0, 1460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3336, 43, 0, 1576, 99, 0, 0, 0, 0, 0, 0, 7, 2288, 76, 570, 0, 0, 2448, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 2365, 0, 0, 0, 0, 597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 0, 44, 47, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 3837, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2428, 1648, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2979, 5054, 5403, 3027, 4099, 4026, 4103, 4077] avg=4096.0000 max=5403 max_rank=2 min=2979 min_rank=0 imbalance_ratio=1.319092 max_load_reduction=27365 max_load_reduction_pct=0.835114 moved_to_echo=16305 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1511, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 944, 11, 0, 1490, 149, 0, 0, 0, 0, 0, 0, 0, 3, 110, 0, 0, 0, 3676, 3, 0, 0, 0, 0, 0, 0, 0, 0, 415, 8, 3397, 0, 2, 153, 0, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3955, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 3987, 54, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 4072, 7, 0, 0, 0, 0, 0, 0, 0, 0, 59, 25, 0, 2, 0, 1, 2440, 1558, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5586, 2595, 3792, 4539, 4004, 4080, 4087, 4085] avg=4096.0000 max=5586 max_rank=0 min=2595 min_rank=1 imbalance_ratio=1.363770 max_load_reduction=27182 max_load_reduction_pct=0.829529 moved_to_echo=16256 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 1513, 0, 2419, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 33, 394, 0, 0, 1672, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 3876, 64, 0, 0, 0, 0, 0, 0, 0, 0, 171, 803, 3964, 0, 0, 470, 0, 633, 0, 0, 0, 0, 0, 0, 0, 0, 13, 6, 0, 18, 4080, 14, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 4039, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4088, 7, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 2445, 1749, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3938, 2257, 3944, 6041, 4181, 4106, 4104, 4197] avg=4096.0000 max=6041 max_rank=3 min=2257 min_rank=1 imbalance_ratio=1.474854 max_load_reduction=26727 max_load_reduction_pct=0.815643 moved_to_echo=16588 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1541, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4049, 0, 0, 49, 159, 0, 0, 0, 0, 0, 0, 0, 571, 1, 1, 0, 0, 3903, 510, 0, 0, 0, 0, 0, 0, 0, 0, 1700, 171, 1079, 0, 3, 27, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4069, 0, 2, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 8, 0, 0, 4038, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 1, 2490, 51, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5626, 4258, 4986, 2999, 4143, 4105, 4101, 2550] avg=4096.0000 max=5626 max_rank=0 min=2550 min_rank=7 imbalance_ratio=1.373535 max_load_reduction=27142 max_load_reduction_pct=0.828308 moved_to_echo=14899 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1545, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 809, 0, 0, 1677, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 0, 3833, 60, 0, 0, 0, 0, 0, 0, 0, 0, 3095, 6, 8, 0, 0, 0, 0, 635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 72, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 4, 0, 0, 57, 0, 0, 0, 0, 2496, 1754, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5635, 2674, 3995, 3744, 4212, 4094, 4103, 4311] avg=4096.0000 max=5635 max_rank=0 min=2674 min_rank=1 imbalance_ratio=1.375732 max_load_reduction=27133 max_load_reduction_pct=0.828033 moved_to_echo=16720 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1535, 0, 4014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4075, 2, 0, 450, 160, 0, 0, 0, 0, 0, 0, 0, 82, 0, 1, 0, 0, 3931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1742, 0, 1386, 0, 0, 0, 0, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3963, 0, 1, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 4022, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4093, 7, 0, 0, 0, 0, 0, 0, 0, 46, 8, 12, 0, 0, 0, 0, 2480, 471, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5549, 4690, 4014, 3300, 4013, 4079, 4106, 3017] avg=4096.0000 max=5549 max_rank=0 min=3017 min_rank=7 imbalance_ratio=1.354736 max_load_reduction=27219 max_load_reduction_pct=0.830658 moved_to_echo=15215 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1541, 0, 347, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 31, 0, 4051, 0, 0, 946, 159, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 3919, 1129, 0, 0, 0, 0, 0, 0, 0, 0, 3643, 0, 662, 0, 0, 0, 0, 359, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4086, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 4033, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 2, 0, 2489, 989, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1890, 5191, 5138, 4664, 4140, 4152, 4107, 3486] avg=4096.0000 max=5191 max_rank=1 min=1890 min_rank=0 imbalance_ratio=1.267334 max_load_reduction=27577 max_load_reduction_pct=0.841583 moved_to_echo=15885 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1528, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 138, 0, 0, 0, 3924, 204, 0, 1651, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1445, 51, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3830, 0, 5, 0, 0, 616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 90, 3814, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 189, 0, 1, 0, 0, 0, 3872, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 32, 0, 0, 0, 4013, 10, 0, 0, 0, 0, 0, 0, 9, 2798, 30, 0, 0, 0, 0, 1, 2482, 1725, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1565, 6049, 1496, 4454, 3976, 4126, 4057, 7045] avg=4096.0000 max=7045 max_rank=7 min=1496 min_rank=2 imbalance_ratio=1.719971 max_load_reduction=25723 max_load_reduction_pct=0.785004 moved_to_echo=19204 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 0, 12288, 0, 20608, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 2944, 128, 128, 128, 128, 128, 128, 19840, 9984, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[10880, 1152, 16768, 0, 128, 768, 0, 3712, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 256, 0, 128, 128, 21632, 384, 0, 9600, 1152, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 3072, 256, 768, 0, 0, 27136, 1920, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[256, 128, 256, 128, 128, 128, 0, 128, 31872, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 0, 128, 128, 0, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 256, 32128, 128, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 1588, 50, 0, 0, 473, 0, 0, 0, 0, 0, 0, 298, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 13, 1667, 0, 0, 0, 0, 118, 0, 0, 62, 0, 0, 4096, 0, 4096, 0, 3719, 0, 0, 0, 4044, 0, 0, 32, 232, 0, 4089, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 4022, 1731, 0, 0, 3, 0, 0, 0, 10, 0, 0, 1, 0, 3570, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3979, 0, 0, 0, 3, 3546, 99, 0, 72, 6, 0, 107, 0, 0, 203, 230, 1, 4077, 0, 4087, 0, 86, 0, 60, 0, 4088, 12, 0, 2561, 0, 0, 212, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 4020, 944, 0, 8, 0, 94, 0, 0, 3, 0, 0, 43, 0, 3997, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 2, 0, 2907, 159, 0, 0, 0, 0, 9, 0, 0, 896, 0, 0, 4039, 0, 4013, 1, 79, 0, 61, 0, 4046, 0, 0, 5, 0, 0, 3340, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 3656, 461, 0, 555, 0, 0, 0, 0, 10, 0, 0, 189, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3097, 4, 0, 0, 2, 28, 10, 3, 0, 0, 0, 163, 0, 0, 468, 0, 0, 4082, 0, 4083, 0, 3401, 37, 4, 0, 3515, 89, 0, 943, 2, 0, 3845, 0, 0, 0, 0, 0, 0, 25] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[10, 2693, 0, 0, 2, 167, 0, 2, 0, 0, 0, 0, 12, 4, 4078, 47, 48, 0, 0, 14, 0, 0, 199, 0, 0, 3522, 0, 5, 0, 0, 21, 3660, 0, 0, 0, 0, 0, 0, 0, 2086, 0, 0, 4085, 0, 4017, 6, 1133, 0, 0, 0, 4026, 0, 0, 2650, 1, 0, 278, 0, 1, 0, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 2276, 132, 0, 0, 1, 7, 0, 0, 0, 0, 0, 3, 0, 4084, 1, 2, 0, 0, 0, 1, 0, 1, 0, 0, 4087, 0, 0, 0, 0, 4060, 210, 0, 0, 2, 0, 407, 0, 0, 1409, 0, 0, 4096, 0, 4093, 0, 2, 0, 0, 0, 4094, 27, 2, 2763, 876, 0, 123, 0, 0, 0, 1, 0, 0, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 2046, 25, 1, 6, 0, 0, 0, 0, 0, 0, 0, 1021, 0, 4030, 15, 0, 1, 0, 0, 6, 0, 0, 24, 0, 4065, 0, 0, 0, 0, 2340, 6, 0, 0, 0, 0, 1, 0, 0, 2777, 1, 2, 4095, 1, 4095, 4, 64, 0, 844, 0, 4086, 8, 1, 0, 3167, 0, 33, 0, 0, 1, 0, 1, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 3383, 270, 0, 12, 3038, 30, 0, 0, 0, 0, 0, 1450, 259, 3553, 0, 5, 0, 0, 0, 0, 0, 1714, 0, 0, 1232, 0, 14, 0, 0, 1244, 3358, 8, 191, 0, 0, 0, 0, 445, 105, 0, 0, 3635, 0, 4093, 0, 47, 0, 449, 0, 3554, 262, 0, 0, 232, 0, 165, 0, 0, 0, 16, 1, 0, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3656, 461, 0, 555, 0, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 189, 0, 3822, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3854, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 4, 0, 0, 2, 28, 10, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 163, 0, 0, 468, 2062, 295, 229, 0, 0, 0, 0, 0, 0, 0, 3037, 0, 0, 0, 3401, 37, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1955, 89, 0, 943, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3845, 0, 0, 0, 0, 0, 0, 25, 1265, 1045, 136, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4810, 4021, 3854, 1079, 3220, 6475, 2993, 6316] avg=4096.0000 max=6475 max_rank=5 min=1079 min_rank=3 imbalance_ratio=1.580811 max_load_reduction=26293 max_load_reduction_pct=0.802399 moved_to_echo=19004 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4020, 944, 0, 8, 0, 94, 0, 134, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 43, 0, 3730, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3788, 0, 0, 0, 0, 0, 0, 0, 0, 1367, 0, 0, 2, 0, 2907, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 896, 2722, 340, 225, 0, 0, 0, 0, 0, 0, 0, 3005, 0, 0, 1, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 2250, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3340, 0, 0, 0, 0, 0, 0, 0, 1456, 1034, 133, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5200, 3789, 3788, 4435, 4192, 3085, 2316, 5963] avg=4096.0000 max=5963 max_rank=7 min=2316 min_rank=6 imbalance_ratio=1.455811 max_load_reduction=26805 max_load_reduction_pct=0.818024 moved_to_echo=15556 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1588, 50, 0, 0, 473, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, 0, 3821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3867, 0, 0, 0, 0, 0, 0, 0, 0, 1369, 0, 0, 0, 0, 13, 1667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 0, 0, 62, 2727, 340, 229, 0, 0, 0, 0, 0, 0, 0, 3047, 0, 0, 0, 3719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2249, 0, 0, 32, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 1455, 1049, 136, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2249, 4119, 3867, 3049, 3476, 6766, 2513, 6729] avg=4096.0000 max=6766 max_rank=5 min=2249 min_rank=0 imbalance_ratio=1.651855 max_load_reduction=26002 max_load_reduction_pct=0.793518 moved_to_echo=19484 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4022, 1731, 0, 0, 3, 0, 0, 120, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 1, 0, 3331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3859, 0, 0, 0, 0, 0, 0, 0, 0, 1330, 0, 0, 0, 3, 3546, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 6, 0, 107, 0, 0, 203, 2649, 344, 228, 0, 0, 0, 0, 0, 230, 1, 3033, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 2273, 12, 0, 2561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 0, 0, 0, 0, 0, 0, 1, 1471, 1044, 119, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5876, 3342, 3860, 4978, 3609, 3350, 4906, 2847] avg=4096.0000 max=5876 max_rank=0 min=2847 min_rank=7 imbalance_ratio=1.434570 max_load_reduction=26892 max_load_reduction_pct=0.820679 moved_to_echo=14712 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 2046, 25, 1, 6, 0, 0, 0, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 0, 3761, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6, 0, 0, 24, 3866, 0, 0, 0, 0, 0, 0, 0, 0, 1359, 0, 0, 0, 0, 2340, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2777, 2706, 343, 229, 0, 0, 0, 0, 0, 1, 2, 3046, 1, 0, 4, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 844, 0, 2273, 8, 1, 0, 3167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 1, 0, 1, 1, 0, 1470, 1049, 134, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2213, 4797, 3897, 3705, 6056, 3118, 6293, 2689] avg=4096.0000 max=6293 max_rank=6 min=2213 min_rank=0 imbalance_ratio=1.536377 max_load_reduction=26475 max_load_reduction_pct=0.807953 moved_to_echo=18156 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[10, 2693, 0, 0, 2, 167, 0, 2, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4, 3805, 47, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 14, 0, 0, 199, 0, 3791, 0, 0, 0, 0, 0, 0, 0, 0, 1177, 0, 5, 0, 0, 21, 3660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2086, 2345, 338, 226, 0, 0, 0, 0, 0, 0, 0, 3039, 0, 0, 6, 1133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2240, 0, 0, 2650, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 278, 0, 1, 0, 0, 1, 0, 0, 1448, 1046, 136, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3011, 3868, 4052, 4863, 4995, 4178, 4891, 2910] avg=4096.0000 max=4995 max_rank=4 min=2910 min_rank=7 imbalance_ratio=1.219482 max_load_reduction=27773 max_load_reduction_pct=0.847565 moved_to_echo=16974 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 2276, 132, 0, 0, 1, 7, 0, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3811, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 1, 0, 3863, 0, 0, 0, 0, 0, 0, 0, 0, 1366, 0, 0, 0, 0, 4060, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 407, 0, 0, 1409, 2721, 344, 230, 0, 0, 0, 0, 0, 0, 0, 3047, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2277, 27, 2, 2763, 876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 0, 0, 0, 1, 0, 0, 8, 1473, 1049, 136, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2553, 3815, 3867, 5636, 5113, 3049, 5945, 2790] avg=4096.0000 max=5945 max_rank=6 min=2553 min_rank=0 imbalance_ratio=1.451416 max_load_reduction=26823 max_load_reduction_pct=0.818573 moved_to_echo=16897 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3383, 270, 0, 12, 3038, 30, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1450, 259, 3308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1714, 0, 3864, 0, 0, 0, 0, 0, 0, 0, 0, 407, 0, 14, 0, 0, 1244, 3358, 0, 0, 0, 0, 0, 0, 0, 0, 8, 191, 0, 0, 0, 0, 445, 105, 825, 304, 229, 0, 0, 0, 0, 0, 0, 0, 2700, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 449, 0, 1968, 262, 0, 0, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 0, 0, 0, 16, 1, 0, 3, 1282, 935, 122, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6856, 5017, 5583, 5023, 2107, 2747, 2911, 2524] avg=4096.0000 max=6856 max_rank=0 min=2107 min_rank=4 imbalance_ratio=1.673828 max_load_reduction=25912 max_load_reduction_pct=0.790771 moved_to_echo=10289 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[12160, 0, 128, 128, 128, 128, 128, 128, 11392, 8320, 1152, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 9472, 128, 128, 128, 128, 14208, 9216, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1536, 0, 17536, 512, 128, 8960, 4608, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 384, 128, 0, 896, 0, 512, 8064, 18816, 2688, 1920, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 23808, 3712, 128, 640, 3712, 256, 128, 1152, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[256, 128, 24064, 128, 0, 128, 8576, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 128, 0, 1920, 128, 30848, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 3072, 384, 29440, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33792 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4080, 12, 3100, 0, 0, 4096, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 1, 11, 0, 0, 1, 0, 0, 0, 0, 597, 0, 0, 4086, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 0, 3, 0, 0, 0, 0, 1035, 0, 0, 0, 0, 22, 3361, 0, 116] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4029, 8, 4094, 0, 0, 4096, 0, 3129, 0, 1, 0, 0, 0, 0, 0, 2, 33, 0, 0, 0, 0, 0, 0, 0, 0, 139, 0, 0, 4094, 0, 4095, 6, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 8, 0, 1, 4089, 0, 0, 0, 0, 0, 0, 0, 855, 0, 1, 0, 0, 0, 3847, 0, 236] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4096, 9, 738, 0, 62, 4096, 0, 164, 0, 0, 0, 0, 0, 0, 0, 0, 559, 0, 0, 0, 2955, 0, 0, 0, 0, 125, 0, 0, 4092, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 4095, 0, 3547] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4092, 0, 4095, 0, 2153, 4024, 0, 2409, 9, 0, 0, 0, 15, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 3, 71, 3, 0, 4071, 0, 4094, 2, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 134, 3969, 0, 0, 120, 0, 0, 0, 0, 261, 8, 0, 1, 0, 0, 2705, 0, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 38, 3918, 0, 4021, 0, 219, 4055, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 4094, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 3286, 0, 0, 1, 0, 0, 0, 13, 51, 0, 0, 0, 0, 0, 3852, 1, 835] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 1, 3715, 142, 19, 0, 767, 4004, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2646, 0, 1, 0, 0, 6, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 3488, 0, 0, 0, 0, 0, 0, 250, 3161, 0, 0, 0, 0, 0, 1271, 0, 919] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4087, 47, 3422, 0, 2412, 4072, 3, 3237, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 444, 0, 0, 0, 0, 66, 0, 0, 4028, 0, 4092, 411, 0, 10, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 223, 4025, 0, 10, 0, 1, 1, 0, 0, 136, 0, 1, 0, 0, 0, 1830, 0, 200] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[16, 145, 4014, 0, 4095, 0, 177, 918, 0, 3975, 0, 12, 0, 0, 0, 0, 0, 0, 0, 258, 2, 0, 72, 0, 0, 0, 0, 248, 0, 0, 3619, 0, 4096, 1933, 0, 1, 0, 0, 285, 0, 0, 0, 0, 0, 2, 0, 2775, 3926, 0, 0, 34, 0, 2, 0, 0, 68, 0, 0, 0, 0, 17, 1884, 1, 193] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 142, 19, 0, 767, 407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 809, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2646, 0, 2788, 618, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6, 0, 0, 4012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 3314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 250, 3161, 3097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1271, 0, 919, 174, 136, 84, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1336, 5092, 6052, 4019, 3753, 3424, 6508, 2584] avg=4096.0000 max=6508 max_rank=6 min=1336 min_rank=0 imbalance_ratio=1.588867 max_load_reduction=26260 max_load_reduction_pct=0.801392 moved_to_echo=16269 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 12, 3100, 0, 0, 416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 828, 207, 0, 0, 0, 0, 0, 0, 0, 0, 1, 11, 0, 0, 1, 0, 2852, 679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 597, 0, 0, 4003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3753, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1035, 3401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 3361, 0, 116, 203, 136, 83, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3528, 5136, 3544, 4600, 3754, 3846, 4439, 3921] avg=4096.0000 max=5136 max_rank=1 min=3528 min_rank=0 imbalance_ratio=1.253906 max_load_reduction=27632 max_load_reduction_pct=0.843262 moved_to_echo=15960 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 9, 738, 0, 62, 416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 0, 0, 0, 0, 0, 0, 828, 207, 0, 0, 0, 0, 0, 0, 0, 0, 559, 0, 0, 0, 2955, 0, 2852, 681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 0, 0, 4009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3753, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 3415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3547, 204, 136, 83, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1225, 1199, 7047, 4134, 3754, 3863, 3481, 8065] avg=4096.0000 max=8065 max_rank=7 min=1199 min_rank=1 imbalance_ratio=1.968994 max_load_reduction=24703 max_load_reduction_pct=0.753876 moved_to_echo=19163 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 4094, 0, 0, 416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3129, 0, 1, 0, 0, 0, 0, 828, 207, 0, 0, 0, 0, 0, 0, 0, 2, 33, 0, 0, 0, 0, 0, 2852, 670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 0, 0, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3752, 6, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 1, 3884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 855, 3359, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3847, 0, 236, 205, 136, 83, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4518, 4165, 3557, 4150, 3763, 3893, 4214, 4508] avg=4096.0000 max=4518 max_rank=0 min=3557 min_rank=2 imbalance_ratio=1.103027 max_load_reduction=28250 max_load_reduction_pct=0.862122 moved_to_echo=16378 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 2153, 409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2409, 9, 0, 0, 0, 15, 1, 813, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 2802, 681, 0, 0, 0, 0, 0, 0, 0, 0, 3, 71, 3, 0, 3988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3751, 2, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 3770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 0, 0, 0, 0, 261, 3411, 0, 0, 0, 0, 0, 0, 0, 8, 0, 1, 0, 0, 2705, 0, 523, 199, 136, 83, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6657, 3454, 3485, 4065, 3756, 3904, 3792, 3655] avg=4096.0000 max=6657 max_rank=0 min=3454 min_rank=1 imbalance_ratio=1.625244 max_load_reduction=26111 max_load_reduction_pct=0.796844 moved_to_echo=15107 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 47, 3422, 0, 2412, 413, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3237, 0, 0, 0, 0, 0, 0, 823, 207, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 444, 0, 2836, 681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 3946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3749, 411, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 223, 3824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 1, 1, 0, 0, 136, 3406, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1830, 0, 200, 201, 136, 82, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6294, 4270, 3962, 4012, 4170, 4056, 3554, 2450] avg=4096.0000 max=6294 max_rank=0 min=2450 min_rank=7 imbalance_ratio=1.536621 max_load_reduction=26474 max_load_reduction_pct=0.807922 moved_to_echo=14230 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 38, 0, 0, 4021, 0, 219, 412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 0, 0, 0, 0, 0, 0, 819, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2824, 652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3751, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 3122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 13, 51, 3266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3852, 1, 835, 164, 136, 83, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4690, 5067, 3477, 4000, 3853, 3279, 3331, 5071] avg=4096.0000 max=5071 max_rank=7 min=3279 min_rank=5 imbalance_ratio=1.238037 max_load_reduction=27697 max_load_reduction_pct=0.845245 moved_to_echo=15534 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[16, 145, 0, 0, 4095, 0, 177, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3975, 0, 12, 0, 0, 0, 0, 188, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 258, 2, 0, 72, 0, 643, 669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 248, 0, 0, 3540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3746, 1933, 0, 1, 0, 0, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2775, 3726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 2, 0, 0, 68, 3345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 1884, 1, 193, 200, 140, 79, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4520, 4385, 1644, 3788, 5965, 6503, 3449, 2514] avg=4096.0000 max=6503 max_rank=5 min=1644 min_rank=2 imbalance_ratio=1.587646 max_load_reduction=26265 max_load_reduction_pct=0.801544 moved_to_echo=18431 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 256, 0, 256, 23680, 0, 5888, 3072, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 0, 128, 22912, 128, 6656, 1664, 1152, 768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 25216, 128, 128, 0, 0, 128, 128, 6016, 1664, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 640, 384, 128, 0, 6144, 0, 20480, 5376, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 256, 128, 128, 0, 384, 5760, 26752, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 128, 0, 3456, 29440, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[30080, 2560, 0, 128, 128, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 1280, 128, 0, 31616, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33792 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 3752, 8, 0, 0, 0, 0, 9, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 13, 0, 5, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4085, 0, 0, 0, 0, 253, 4096, 0, 0, 0, 0, 0, 0, 0, 7, 91, 0, 0, 0, 0, 4058, 12, 4, 0, 0, 0, 4094, 0, 0, 4088] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 3985, 49, 0, 0, 0, 0, 110, 0, 0, 51, 0, 0, 0, 4082, 0, 0, 0, 0, 73, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 4009, 0, 4087, 0, 0, 0, 0, 3349, 4096, 0, 0, 2, 0, 17, 0, 0, 0, 358, 328, 0, 0, 0, 4056, 0, 0, 0, 0, 0, 3972, 0, 0, 142] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 65, 2185, 0, 0, 0, 0, 3, 0, 0, 249, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 4096, 0, 0, 0, 0, 1538, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 98, 17, 1, 0, 0, 4094, 0, 0, 0, 0, 0, 4076, 0, 0, 4091] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 346, 2769, 0, 5, 0, 0, 0, 1, 0, 35, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 17, 0, 0, 3152, 4038, 0, 0, 0, 0, 0, 0, 0, 1, 136, 0, 0, 0, 1, 1847, 0, 0, 0, 0, 0, 4068, 0, 0, 4065] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 5, 0, 1, 0, 0, 0, 4088, 0, 0, 3, 0, 6, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3974, 17, 4088, 5, 0, 0, 0, 3979, 4090, 1, 0, 46, 0, 2, 0, 0, 0, 3394, 1162, 2, 0, 0, 3727, 0, 1, 0, 0, 0, 4091, 0, 1, 45] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4046, 0, 0, 0, 0, 0, 0, 7, 0, 60, 0, 0, 0, 3938, 0, 6, 0, 0, 0, 0, 45, 0, 0, 3277, 0, 279, 0, 0, 0, 4042, 242, 3831, 0, 0, 0, 0, 1083, 4089, 0, 0, 10, 0, 0, 0, 0, 0, 92, 10, 93, 0, 0, 3336, 0, 14, 30, 0, 0, 4096, 0, 0, 142] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 78, 2600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 57, 4075, 0, 0, 0, 0, 3246, 4096, 0, 40, 0, 0, 10, 0, 0, 25, 0, 0, 0, 0, 0, 3441, 26, 0, 0, 0, 0, 4037, 0, 0, 2879] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 3132, 0, 0, 3, 0, 0, 0, 0, 0, 38, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4085, 0, 0, 0, 0, 1092, 4093, 6, 0, 0, 0, 1, 0, 0, 0, 8, 30, 0, 2, 0, 4084, 0, 0, 0, 2, 0, 4096, 0, 0, 3900] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3752, 8, 0, 0, 0, 995, 250, 5, 0, 0, 0, 0, 0, 0, 9, 0, 0, 1, 0, 0, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 13, 0, 5, 6, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2412, 1235, 31, 0, 0, 0, 0, 0, 1860, 0, 0, 0, 0, 0, 0, 253, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 91, 0, 0, 0, 0, 3808, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1682, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5010, 4064, 4122, 3678, 2113, 4089, 3918, 5774] avg=4096.0000 max=5774 max_rank=7 min=2113 min_rank=4 imbalance_ratio=1.409668 max_load_reduction=26994 max_load_reduction_pct=0.823792 moved_to_echo=15894 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 78, 2600, 0, 0, 0, 988, 212, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2378, 1227, 31, 0, 0, 0, 0, 0, 1847, 57, 0, 0, 0, 0, 0, 3246, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 40, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 3229, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1659, 0, 0, 2879, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3883, 4044, 4098, 3636, 5150, 4139, 3280, 4538] avg=4096.0000 max=5150 max_rank=4 min=3280 min_rank=6 imbalance_ratio=1.257324 max_load_reduction=27618 max_load_reduction_pct=0.842834 moved_to_echo=17107 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3985, 49, 0, 0, 0, 975, 250, 5, 0, 0, 0, 0, 0, 0, 110, 0, 0, 51, 0, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 73, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 2340, 1211, 31, 0, 0, 0, 0, 0, 1823, 0, 0, 0, 0, 0, 0, 3349, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 2, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 358, 328, 0, 0, 0, 3806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1632, 0, 0, 142, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5264, 4217, 4157, 3584, 5172, 4108, 4492, 1774] avg=4096.0000 max=5264 max_rank=0 min=1774 min_rank=7 imbalance_ratio=1.285156 max_load_reduction=27504 max_load_reduction_pct=0.839355 moved_to_echo=15546 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 65, 2185, 0, 0, 0, 988, 252, 5, 0, 0, 0, 0, 0, 0, 3, 0, 0, 249, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2401, 1227, 31, 0, 0, 0, 0, 0, 1848, 0, 0, 0, 0, 0, 0, 1538, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 17, 1, 0, 0, 3842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1675, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3495, 4317, 4098, 3659, 3386, 4089, 3958, 5766] avg=4096.0000 max=5766 max_rank=7 min=3386 min_rank=4 imbalance_ratio=1.407715 max_load_reduction=27002 max_load_reduction_pct=0.824036 moved_to_echo=17199 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 39, 0, 0, 0, 0, 967, 230, 5, 0, 0, 0, 0, 0, 0, 0, 5, 0, 1, 0, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 3, 0, 6, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2410, 1200, 30, 0, 0, 0, 0, 0, 1807, 17, 0, 5, 0, 0, 0, 3979, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 1, 0, 46, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3394, 1162, 2, 0, 0, 3497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1681, 0, 1, 45, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1241, 4064, 4099, 3641, 5808, 4132, 8055, 1728] avg=4096.0000 max=8055 max_rank=6 min=1241 min_rank=0 imbalance_ratio=1.966553 max_load_reduction=24713 max_load_reduction_pct=0.754181 moved_to_echo=19723 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 346, 2769, 0, 5, 0, 996, 114, 5, 0, 0, 0, 0, 0, 0, 0, 1, 0, 35, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2396, 1237, 31, 0, 0, 0, 0, 0, 1863, 0, 0, 0, 17, 0, 0, 3152, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 136, 0, 0, 0, 1, 1733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1672, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4235, 4100, 4097, 3664, 5032, 4032, 1871, 5737] avg=4096.0000 max=5737 max_rank=7 min=1871 min_rank=6 imbalance_ratio=1.400635 max_load_reduction=27031 max_load_reduction_pct=0.824921 moved_to_echo=16672 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3132, 0, 0, 3, 0, 996, 252, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2413, 1236, 31, 0, 0, 0, 0, 0, 1861, 0, 0, 0, 0, 0, 0, 1092, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 6, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 30, 0, 2, 0, 3832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1683, 0, 0, 3900, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4388, 4092, 4105, 3680, 2953, 4093, 3872, 5585] avg=4096.0000 max=5585 max_rank=7 min=2953 min_rank=4 imbalance_ratio=1.363525 max_load_reduction=27183 max_load_reduction_pct=0.829559 moved_to_echo=16503 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4046, 0, 0, 0, 0, 987, 209, 10, 0, 0, 0, 0, 0, 0, 0, 7, 0, 60, 0, 0, 0, 3803, 0, 0, 0, 0, 0, 0, 0, 3938, 0, 6, 0, 0, 0, 0, 45, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3277, 0, 279, 0, 0, 0, 2416, 1226, 28, 0, 0, 0, 0, 0, 1829, 242, 0, 0, 0, 0, 0, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 10, 93, 0, 0, 3127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 30, 0, 0, 1680, 0, 0, 142, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5252, 3870, 3992, 7226, 3154, 4086, 3322, 1866] avg=4096.0000 max=7226 max_rank=3 min=1866 min_rank=7 imbalance_ratio=1.764160 max_load_reduction=25542 max_load_reduction_pct=0.779480 moved_to_echo=12428 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 3328, 0, 384, 0, 0, 0, 19200, 9856, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[14848, 384, 0, 128, 128, 0, 0, 17792, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 13440, 0, 128, 19456, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 0, 15488, 7680, 0, 128, 0, 7936, 1792, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 4224, 1664, 128, 128, 128, 26880, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[32640, 0, 128, 128, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32640, 128, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 0, 512, 0, 0, 0, 32256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 3958, 0, 0, 0, 0, 0, 4085, 0, 216, 0, 2, 2018, 0, 4095, 91, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1908, 0, 0, 0, 3, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[10, 0, 4095, 4, 0, 2, 0, 0, 4050, 0, 34, 30, 0, 3493, 0, 4080, 0, 0, 0, 0, 4096, 4093, 4, 0, 54, 0, 0, 2, 0, 4, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 145, 0, 0, 0, 0, 1, 0, 0, 0, 6, 383, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4095, 6, 3, 0, 0, 0, 4088, 0, 10, 0, 25, 222, 0, 4093, 0, 0, 0, 0, 4082, 3869, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 4058, 3, 4085, 0, 0, 0, 9, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1, 0, 3966, 0, 0, 0, 0, 0, 3714, 0, 0, 48, 0, 3036, 0, 3955, 300, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1287, 0, 0, 0, 69, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 2, 4089, 0, 0, 0, 0, 7, 4080, 0, 0, 63, 18, 2672, 0, 3906, 4, 0, 0, 0, 4096, 3968, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 4078, 0, 1447, 0, 35, 0, 0, 0, 0, 0, 3, 4, 193, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 13, 3172, 0, 0, 0, 0, 0, 3940, 0, 5, 0, 0, 394, 0, 4044, 148, 0, 0, 1, 4096, 4010, 246, 0, 52, 0, 0, 0, 0, 55, 575, 0, 4027, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 43, 4096, 0, 3770, 0, 6, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1, 0, 3328, 0, 207, 0, 0, 40, 4080, 0, 0, 408, 13, 3433, 0, 3997, 0, 0, 0, 0, 4096, 4091, 0, 0, 115, 0, 56, 0, 0, 94, 0, 0, 2991, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4096, 0, 722, 0, 0, 0, 0, 949, 0, 0, 0, 17, 22, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 1219, 0, 0, 2, 0, 0, 4095, 0, 44, 0, 0, 3067, 1, 4092, 0, 15, 0, 4, 4094, 4088, 14, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4081, 0, 0, 0, 0, 1, 0, 0, 0, 3, 1, 0, 4078, 0, 3853, 0, 0, 0, 6, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3958, 0, 0, 0, 0, 0, 569, 0, 0, 0, 0, 0, 0, 0, 1719, 0, 216, 0, 2, 2018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 3954, 73, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 103, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1940, 0, 1908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 2366, 1484, 141, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4022, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4527, 3955, 4141, 4030, 4249, 3848, 3995, 4023] avg=4096.0000 max=4527 max_rank=0 min=3848 min_rank=5 imbalance_ratio=1.105225 max_load_reduction=28241 max_load_reduction_pct=0.861847 moved_to_echo=16115 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 6, 3, 0, 0, 0, 564, 0, 0, 0, 0, 0, 0, 0, 1720, 0, 10, 0, 25, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3825, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7, 3951, 72, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 102, 44, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 1922, 3, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 10, 2368, 1470, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4010, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4668, 1977, 3825, 4031, 4234, 6024, 3999, 4010] avg=4096.0000 max=6024 max_rank=5 min=1977 min_rank=1 imbalance_ratio=1.470703 max_load_reduction=26744 max_load_reduction_pct=0.816162 moved_to_echo=18267 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 4095, 4, 0, 2, 0, 0, 569, 0, 0, 0, 0, 0, 0, 0, 1704, 0, 34, 30, 0, 3493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 2, 0, 4, 0, 0, 3939, 73, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 103, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1940, 0, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6, 2346, 1484, 141, 0, 0, 0, 0, 0, 383, 0, 0, 0, 0, 0, 0, 0, 4023, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4680, 5261, 4051, 4072, 4235, 2085, 3978, 4406] avg=4096.0000 max=5261 max_rank=1 min=2085 min_rank=5 imbalance_ratio=1.284424 max_load_reduction=27507 max_load_reduction_pct=0.839447 moved_to_echo=14704 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 13, 3172, 0, 0, 0, 0, 0, 569, 0, 0, 0, 0, 0, 0, 0, 1658, 0, 5, 0, 0, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 0, 0, 1, 0, 3965, 246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 55, 575, 0, 3904, 73, 0, 0, 0, 0, 0, 0, 4027, 0, 0, 0, 0, 0, 0, 6, 103, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 1940, 0, 3770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 69, 0, 0, 0, 0, 0, 0, 2282, 1484, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3754, 2057, 4360, 4659, 4181, 5753, 3981, 4023] avg=4096.0000 max=5753 max_rank=5 min=2057 min_rank=1 imbalance_ratio=1.404541 max_load_reduction=27015 max_load_reduction_pct=0.824432 moved_to_echo=17938 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 3966, 0, 0, 0, 0, 0, 569, 0, 0, 0, 0, 0, 0, 0, 1563, 0, 0, 48, 0, 3036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 0, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 3819, 73, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 103, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1940, 0, 1287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 2151, 1484, 136, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4023, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4536, 4647, 4350, 3897, 4245, 3227, 3840, 4026] avg=4096.0000 max=4647 max_rank=1 min=3227 min_rank=5 imbalance_ratio=1.134521 max_load_reduction=28121 max_load_reduction_pct=0.858185 moved_to_echo=15338 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4089, 0, 0, 0, 0, 7, 567, 0, 0, 0, 0, 0, 0, 0, 1717, 0, 0, 63, 18, 2672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 3923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 3771, 73, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 103, 45, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 1930, 0, 1447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 3, 4, 2363, 1478, 135, 0, 0, 0, 0, 0, 193, 0, 0, 0, 0, 0, 0, 0, 4023, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4665, 4470, 3927, 3846, 4244, 3382, 4018, 4216] avg=4096.0000 max=4665 max_rank=0 min=3382 min_rank=5 imbalance_ratio=1.138916 max_load_reduction=28103 max_load_reduction_pct=0.857635 moved_to_echo=15860 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1219, 0, 0, 2, 0, 0, 567, 0, 0, 0, 0, 0, 0, 0, 1723, 0, 44, 0, 0, 3067, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 4, 0, 4042, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 3951, 74, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 1, 0, 0, 103, 46, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 1930, 0, 3853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 1, 0, 0, 2372, 1478, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4020, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1788, 4835, 4075, 4034, 4231, 5787, 3998, 4020] avg=4096.0000 max=5787 max_rank=5 min=1788 min_rank=0 imbalance_ratio=1.412842 max_load_reduction=26981 max_load_reduction_pct=0.823395 moved_to_echo=18036 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 3328, 0, 207, 0, 0, 40, 574, 0, 0, 0, 0, 0, 0, 0, 1712, 0, 0, 408, 13, 3433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 56, 0, 0, 94, 0, 0, 3860, 74, 0, 0, 0, 0, 0, 0, 2991, 0, 0, 0, 0, 0, 0, 0, 106, 52, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 1928, 0, 722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 949, 0, 0, 0, 17, 2368, 1488, 137, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 4022, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4150, 5566, 4039, 4199, 3149, 2662, 4959, 4044] avg=4096.0000 max=5566 max_rank=1 min=2662 min_rank=5 imbalance_ratio=1.358887 max_load_reduction=27202 max_load_reduction_pct=0.830139 moved_to_echo=14814 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 1024, 0, 128, 128, 128, 18688, 11904, 1152, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 15488, 128, 17280, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[13568, 0, 384, 640, 128, 18432, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 28032, 128, 256, 128, 0, 128, 4608, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[31616, 0, 0, 128, 0, 128, 0, 128, 896, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[640, 0, 0, 0, 0, 0, 0, 0, 32256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[640, 128, 0, 128, 0, 32000, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[256, 0, 128, 128, 0, 256, 640, 128, 31232, 640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=32896 slack_slots=2816 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 9, 0, 0, 0, 7, 0, 2, 0, 4082, 1, 0, 7, 0, 0, 0, 3808, 0, 0, 0, 4096, 4092, 0, 7, 0, 0, 0, 4073, 0, 20, 0, 0, 0, 1513, 0, 635, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 2097, 209, 0, 0, 0, 0, 0, 0, 4000, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 0, 1, 0, 0, 0, 2, 4096, 0, 3, 6, 4075, 4093, 0, 111, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 88, 0, 3986, 0, 0, 0, 1, 0, 0, 0, 18, 0, 0, 0, 0, 0, 8, 10, 0, 0, 0, 0, 0, 0, 4038, 0, 0, 0, 3, 4090] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 30, 0, 97, 0, 8, 0, 3116, 0, 0, 1, 0, 0, 0, 4013, 0, 0, 0, 3897, 4053, 0, 0, 0, 0, 0, 4088, 0, 2, 0, 0, 0, 15, 27, 3054, 0, 0, 0, 0, 0, 0, 0, 176, 0, 0, 0, 0, 1, 2589, 0, 0, 0, 0, 0, 0, 0, 3508, 5, 0, 0, 0, 4088] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 270, 3, 4094, 1, 0, 0, 0, 0, 0, 4044, 0, 0, 1, 4079, 4095, 0, 0, 0, 3, 0, 4091, 0, 5, 0, 0, 0, 1763, 0, 840, 0, 0, 0, 0, 0, 0, 1, 893, 2, 0, 0, 0, 10, 403, 0, 0, 1, 8, 0, 0, 0, 4081, 0, 0, 1, 0, 4079] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 5, 0, 24, 6, 83, 0, 32, 0, 895, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 4047, 4096, 0, 7, 8, 0, 0, 4096, 1, 0, 0, 0, 0, 64, 0, 3984, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 3523, 0, 0, 0, 0, 0, 0, 0, 3721, 0, 0, 0, 0, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 440, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 3972, 4010, 0, 2641, 46, 0, 0, 4087, 0, 0, 0, 0, 0, 3764, 0, 2749, 0, 0, 1, 0, 0, 0, 0, 201, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 2872, 0, 0, 0, 0, 3816] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 22, 2120, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 6, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 2037, 0, 0, 0, 0, 0, 0, 0, 4008, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 6, 0, 0, 0, 72, 0, 571, 0, 1690, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4038, 4081, 0, 44, 73, 0, 0, 4096, 0, 0, 0, 0, 0, 1591, 0, 3652, 0, 0, 4, 0, 0, 0, 0, 10, 0, 0, 0, 5, 0, 691, 4, 0, 0, 0, 0, 0, 0, 3953, 0, 0, 0, 0, 4090] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 9, 0, 0, 0, 7, 0, 4013, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4082, 1, 0, 7, 0, 0, 1231, 246, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 3711, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1513, 0, 635, 0, 0, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 3630, 79, 0, 0, 0, 0, 0, 0, 0, 0, 2097, 209, 0, 0, 0, 0, 2538, 123, 0, 0, 0, 0, 0, 0, 0, 0, 4000, 0, 0, 0, 0, 327, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4029, 5569, 4151, 3738, 2264, 3723, 4967, 4327] avg=4096.0000 max=5569 max_rank=1 min=2264 min_rank=4 imbalance_ratio=1.359619 max_load_reduction=27199 max_load_reduction_pct=0.830048 moved_to_echo=15281 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 0, 1, 0, 0, 0, 1229, 248, 0, 0, 0, 0, 0, 0, 2, 59, 0, 3, 6, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 0, 0, 0, 3731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 3986, 0, 0, 0, 117, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 18, 0, 0, 0, 3904, 79, 0, 0, 0, 0, 0, 0, 0, 0, 8, 10, 0, 0, 0, 0, 2535, 133, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 0, 0, 0, 3, 326, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4014, 5521, 4145, 3842, 4191, 4002, 2686, 4367] avg=4096.0000 max=5521 max_rank=1 min=2686 min_rank=6 imbalance_ratio=1.347900 max_load_reduction=27247 max_load_reduction_pct=0.831512 moved_to_echo=15246 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 5, 0, 24, 6, 83, 0, 4016, 0, 0, 0, 0, 0, 0, 0, 32, 0, 895, 0, 0, 0, 0, 0, 1231, 248, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8, 0, 0, 3731, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 3984, 0, 0, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 3883, 80, 0, 0, 0, 0, 0, 0, 0, 0, 3523, 0, 0, 0, 0, 0, 2537, 132, 0, 0, 0, 0, 0, 0, 0, 0, 3721, 0, 0, 0, 0, 326, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4134, 2406, 4105, 3747, 4165, 3972, 6192, 4047] avg=4096.0000 max=6192 max_rank=6 min=2406 min_rank=1 imbalance_ratio=1.511719 max_load_reduction=26576 max_load_reduction_pct=0.811035 moved_to_echo=18376 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4016, 0, 0, 0, 0, 0, 0, 0, 270, 3, 4094, 1, 0, 0, 0, 0, 1226, 247, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 1, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3727, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1763, 0, 840, 0, 0, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 893, 2, 0, 0, 3855, 79, 0, 0, 0, 0, 0, 0, 0, 10, 403, 0, 0, 1, 8, 0, 2528, 131, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 1, 0, 325, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4016, 5841, 4138, 3735, 2720, 4830, 3081, 4407] avg=4096.0000 max=5841 max_rank=1 min=2720 min_rank=4 imbalance_ratio=1.426025 max_load_reduction=26927 max_load_reduction_pct=0.821747 moved_to_echo=15038 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 30, 0, 97, 0, 3974, 0, 0, 0, 0, 0, 0, 0, 8, 0, 3116, 0, 0, 1, 0, 0, 1229, 247, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 3897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3724, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 27, 3054, 0, 0, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 0, 0, 3825, 79, 0, 0, 0, 0, 0, 0, 0, 1, 2589, 0, 0, 0, 0, 0, 2534, 130, 0, 0, 0, 0, 0, 0, 0, 0, 3508, 5, 0, 0, 0, 325, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4601, 3955, 3726, 3213, 4080, 5254, 3838] avg=4096.0000 max=5254 max_rank=6 min=3213 min_rank=4 imbalance_ratio=1.282715 max_load_reduction=27514 max_load_reduction_pct=0.839661 moved_to_echo=16385 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6, 0, 0, 0, 72, 0, 4003, 0, 0, 0, 0, 0, 0, 0, 571, 0, 1690, 0, 0, 0, 0, 0, 1229, 248, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 1, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 73, 0, 0, 3731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1591, 0, 3652, 0, 0, 4, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 3904, 78, 0, 0, 0, 0, 0, 0, 5, 0, 691, 4, 0, 0, 0, 0, 2535, 133, 0, 0, 0, 0, 0, 0, 0, 0, 3953, 0, 0, 0, 0, 326, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 3738, 4098, 3848, 5364, 3992, 3368, 4279] avg=4096.0000 max=5364 max_rank=4 min=3368 min_rank=6 imbalance_ratio=1.309570 max_load_reduction=27404 max_load_reduction_pct=0.836304 moved_to_echo=17003 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 4017, 0, 0, 0, 0, 0, 0, 0, 0, 22, 2120, 0, 0, 0, 0, 0, 1231, 247, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4095, 0, 0, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3904, 79, 0, 0, 0, 0, 0, 0, 0, 0, 2037, 0, 0, 0, 0, 0, 2538, 133, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 0, 0, 0, 0, 327, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4018, 3620, 4155, 3728, 4218, 3986, 4708, 4335] avg=4096.0000 max=4708 max_rank=6 min=3620 min_rank=1 imbalance_ratio=1.149414 max_load_reduction=28060 max_load_reduction_pct=0.856323 moved_to_echo=17247 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 440, 0, 3933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 1151, 251, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 3972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2641, 46, 0, 0, 3717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3764, 0, 2749, 0, 0, 1, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 201, 0, 0, 0, 3905, 77, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 2369, 137, 0, 0, 0, 0, 0, 0, 0, 0, 2872, 0, 0, 0, 0, 296, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4375, 1472, 4021, 6404, 6633, 4183, 2512, 3168] avg=4096.0000 max=6633 max_rank=4 min=1472 min_rank=1 imbalance_ratio=1.619385 max_load_reduction=26135 max_load_reduction_pct=0.797577 moved_to_echo=16496 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 11392, 256, 0, 128, 128, 0, 20224, 1152, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[896, 128, 20224, 128, 128, 128, 0, 0, 9856, 2048, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 8832, 128, 23040, 0, 0, 128, 1024, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 30208, 128, 0, 128, 128, 2688, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2816, 128, 128, 0, 29824, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 512, 0, 128, 128, 32384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 0, 128, 1408, 128, 0, 0, 30848, 640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 128, 128, 128, 128, 768, 0, 32000, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1584, 75, 2245, 0, 0, 0, 4, 0, 0, 0, 58, 0, 0, 6, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3455, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4003, 3955, 0, 0, 0, 0, 0, 0, 0, 1291, 0, 0, 0, 0, 0, 3866, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 818, 0, 3209, 0, 2, 0, 3, 0, 0, 0, 95, 0, 0, 1, 0, 4089, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4094, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4093, 4086, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[5, 0, 0, 0, 780, 5, 3146, 0, 0, 1, 253, 0, 0, 0, 0, 32, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 73, 41, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4084, 4065, 4096, 0, 0, 0, 1, 0, 0, 0, 12, 0, 0, 0, 0, 0, 3884, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 13, 0, 1446, 0, 0, 0, 878, 0, 0, 0, 484, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 38, 0, 8, 0, 0, 0, 4094, 4062, 0, 230, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4059, 4096, 4019, 0, 0, 0, 45, 0, 0, 1, 1935, 0, 0, 0, 0, 0, 3263, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1081, 0, 2972, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 4091, 0, 0, 0, 5, 0, 0, 0, 0, 3, 0, 4063, 4076, 0, 130, 0, 0, 1, 0, 0, 0, 0, 14, 0, 0, 0, 4095, 4079, 4094, 1, 0, 0, 33, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4021, 0, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7, 0, 3001, 8, 0, 0, 7, 0, 0, 0, 62, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 4096, 0, 744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 341, 0, 0, 0, 0, 0, 4024, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 19, 0, 3083, 0, 0, 0, 1048, 2, 1, 0, 26, 0, 0, 0, 0, 4095, 0, 2, 0, 3, 0, 0, 8, 0, 64, 1, 3712, 4092, 11, 208, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 4092, 4092, 0, 1, 1, 3, 1, 0, 0, 32, 0, 1, 0, 21, 0, 4066, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1470, 0, 4081, 0, 0, 0, 187, 0, 14, 0, 0, 51, 0, 21, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 917, 4039, 347, 3218, 0, 0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 4032, 3404, 3719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3169, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1584, 75, 2245, 0, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 58, 0, 3703, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4042, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3455, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2811, 656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4003, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1291, 3196, 385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3866, 0, 0, 297, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4370, 3765, 4051, 3974, 3467, 4106, 4872, 4163] avg=4096.0000 max=4872 max_rank=6 min=3467 min_rank=4 imbalance_ratio=1.189453 max_load_reduction=27896 max_load_reduction_pct=0.851318 moved_to_echo=16608 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 0, 0, 780, 5, 3146, 0, 466, 0, 0, 0, 0, 0, 0, 0, 0, 1, 253, 0, 0, 0, 0, 32, 3700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 518, 0, 0, 0, 0, 0, 0, 0, 0, 73, 41, 0, 0, 0, 6, 0, 0, 2808, 680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 12, 3310, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3884, 0, 0, 297, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4402, 3986, 4099, 4614, 3608, 4171, 3707, 4181] avg=4096.0000 max=4614 max_rank=3 min=3608 min_rank=4 imbalance_ratio=1.126465 max_load_reduction=28154 max_load_reduction_pct=0.859192 moved_to_echo=15667 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1081, 0, 2972, 0, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3709, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4091, 0, 0, 0, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4063, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 0, 1, 0, 0, 0, 2797, 679, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 4079, 106, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 33, 0, 0, 3, 0, 3309, 386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4021, 0, 4, 296, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4517, 3710, 4100, 4582, 3607, 4199, 3732, 4321] avg=4096.0000 max=4582 max_rank=3 min=3607 min_rank=4 imbalance_ratio=1.118652 max_load_reduction=28186 max_load_reduction_pct=0.860168 moved_to_echo=15859 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 818, 0, 3209, 0, 466, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 0, 0, 0, 95, 0, 3708, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4089, 2, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 2810, 678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 3302, 386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 3, 297, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4493, 3808, 4095, 4610, 3493, 4199, 3695, 4375] avg=4096.0000 max=4610 max_rank=3 min=3493 min_rank=4 imbalance_ratio=1.125488 max_load_reduction=28158 max_load_reduction_pct=0.859314 moved_to_echo=15762 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7, 0, 3001, 8, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 62, 0, 3711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 744, 0, 0, 0, 0, 0, 0, 2811, 679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 341, 3308, 385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 0, 0, 297, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3482, 3780, 4099, 4616, 4234, 4202, 4034, 4321] avg=4096.0000 max=4616 max_rank=3 min=3482 min_rank=0 imbalance_ratio=1.126953 max_load_reduction=28152 max_load_reduction_pct=0.859131 moved_to_echo=16791 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 19, 0, 3083, 0, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 2, 1, 0, 26, 0, 3698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 64, 1, 3712, 518, 0, 0, 0, 0, 0, 0, 0, 0, 11, 208, 1, 1, 0, 0, 0, 0, 2808, 679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 3, 1, 0, 0, 32, 3307, 383, 0, 0, 0, 0, 0, 0, 0, 1, 0, 21, 0, 4066, 0, 0, 297, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3568, 4775, 4103, 4303, 3708, 4198, 3728, 4385] avg=4096.0000 max=4775 max_rank=1 min=3568 min_rank=0 imbalance_ratio=1.165771 max_load_reduction=27993 max_load_reduction_pct=0.854279 moved_to_echo=16019 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 13, 0, 1446, 0, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 878, 0, 0, 0, 484, 0, 3677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 38, 0, 8, 0, 0, 0, 4094, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 230, 0, 0, 0, 0, 1, 0, 2788, 667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 1, 1935, 3248, 382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3263, 0, 0, 295, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1921, 5039, 4099, 4654, 3686, 4200, 5611, 3558] avg=4096.0000 max=5611 max_rank=6 min=1921 min_rank=0 imbalance_ratio=1.369873 max_load_reduction=27157 max_load_reduction_pct=0.828766 moved_to_echo=17055 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1470, 0, 4081, 0, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 187, 0, 14, 0, 0, 51, 3653, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 917, 498, 0, 0, 0, 0, 0, 0, 0, 0, 347, 3218, 0, 0, 1, 0, 0, 0, 2777, 622, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3404, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3010, 379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3169, 0, 0, 295, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6015, 3905, 4122, 1415, 6965, 3493, 3389, 3464] avg=4096.0000 max=6965 max_rank=4 min=1415 min_rank=3 imbalance_ratio=1.700439 max_load_reduction=25803 max_load_reduction_pct=0.787445 moved_to_echo=17311 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[512, 4608, 128, 128, 128, 128, 128, 0, 22528, 5376, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 0, 128, 3712, 26112, 3072, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 0, 128, 128, 28544, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 2432, 128, 128, 0, 768, 128, 29568, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 5888, 128, 23296, 128, 3840, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 0, 128, 0, 30464, 0, 128, 2432, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 0, 0, 32000, 896, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 0, 32768, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[251, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3981, 0, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 3962, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[432, 2, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 3608, 0, 1, 0, 0, 0, 0, 4092, 2, 0, 0, 0, 0, 4096, 0, 0, 4096, 141, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 3913, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 7, 0, 0, 3959, 0, 4034, 0, 0, 0, 0, 1, 2699, 0, 0, 0, 22, 0, 0, 4095, 0, 0, 0, 4, 3, 4096, 155, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 8, 0, 0, 0, 4095, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[3389, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 10, 0, 0, 0, 4091, 0, 3704, 0, 0, 0, 0, 0, 1031, 0, 5, 0, 4, 0, 0, 4088, 0, 0, 0, 0, 0, 4096, 0, 0, 4091, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4085, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[398, 0, 0, 1583, 0, 0, 0, 28, 0, 274, 0, 0, 0, 5, 0, 0, 0, 2, 0, 0, 0, 0, 0, 3900, 0, 4096, 0, 0, 0, 0, 0, 2066, 0, 235, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4089, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4068, 1, 0, 0, 0, 3831, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1189, 1, 0, 0, 0, 0, 0, 0, 1, 15, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 0, 2999, 0, 1, 0, 0, 0, 4018, 0, 16, 0, 0, 0, 0, 4094, 0, 2, 0, 1, 0, 4094, 16, 0, 4095, 20, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 10, 0, 0, 0, 4076, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[3655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 13, 0, 0, 3, 0, 0, 4012, 3, 4093, 0, 0, 0, 0, 0, 519, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 3, 1, 0, 4094, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 4085, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[4060, 0, 0, 0, 0, 0, 28, 0, 0, 28, 0, 0, 0, 0, 0, 0, 4, 33, 0, 0, 0, 0, 0, 3710, 0, 2670, 0, 0, 0, 0, 96, 3801, 0, 1335, 0, 22, 0, 0, 4096, 0, 0, 0, 0, 0, 4078, 0, 0, 3987, 0, 0, 1, 0, 0, 42, 0, 0, 0, 4096, 33, 0, 0, 0, 648, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[432, 2, 0, 0, 0, 0, 0, 0, 2128, 99, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 3593, 476, 23, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 4096, 103, 8, 0, 0, 0, 0, 0, 0, 0, 1490, 0, 0, 0, 0, 0, 3608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3890, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 141, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 503, 6, 0, 0, 0, 3913, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2661, 4161, 4231, 5098, 3893, 4088, 4214, 4422] avg=4096.0000 max=5098 max_rank=3 min=2661 min_rank=0 imbalance_ratio=1.244629 max_load_reduction=27670 max_load_reduction_pct=0.844421 moved_to_echo=16617 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[251, 0, 0, 0, 0, 0, 0, 0, 2129, 99, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3593, 476, 23, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3981, 103, 8, 0, 0, 0, 0, 0, 0, 0, 1491, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 503, 1, 0, 0, 0, 3962, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2479, 4093, 4093, 5581, 3894, 4088, 4074, 4466] avg=4096.0000 max=5581 max_rank=3 min=2479 min_rank=0 imbalance_ratio=1.362549 max_load_reduction=27187 max_load_reduction_pct=0.829681 moved_to_echo=16522 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[398, 0, 0, 1583, 0, 0, 0, 28, 2129, 99, 0, 0, 0, 0, 0, 0, 0, 274, 0, 0, 0, 5, 0, 0, 3569, 476, 22, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 3900, 103, 8, 0, 0, 0, 0, 0, 0, 0, 1491, 0, 0, 0, 0, 0, 2066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 235, 0, 0, 0, 0, 3894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 499, 1, 0, 0, 0, 3831, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4237, 4346, 4013, 3557, 4129, 4087, 4068, 4331] avg=4096.0000 max=4346 max_rank=1 min=3557 min_rank=3 imbalance_ratio=1.061035 max_load_reduction=28422 max_load_reduction_pct=0.867371 moved_to_echo=16615 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3389, 0, 0, 71, 0, 0, 0, 0, 1925, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3591, 431, 22, 0, 0, 0, 0, 0, 0, 9, 0, 10, 0, 0, 0, 4091, 103, 8, 0, 0, 0, 0, 0, 0, 0, 1348, 0, 0, 0, 0, 0, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4, 0, 0, 3886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 502, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5484, 4044, 4221, 2379, 3895, 4083, 4075, 4587] avg=4096.0000 max=5484 max_rank=0 min=2379 min_rank=3 imbalance_ratio=1.338867 max_load_reduction=27284 max_load_reduction_pct=0.832642 moved_to_echo=16640 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1399, 0, 0, 0, 0, 0, 0, 0, 2097, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 3582, 469, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 3959, 103, 8, 0, 0, 0, 0, 0, 0, 0, 1468, 0, 0, 0, 0, 1, 2699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 3893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 0, 155, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 501, 8, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3595, 4087, 4077, 4168, 3915, 4248, 4074, 4604] avg=4096.0000 max=4604 max_rank=7 min=3595 min_rank=0 imbalance_ratio=1.124023 max_load_reduction=28164 max_load_reduction_pct=0.859497 moved_to_echo=16841 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1189, 1, 0, 0, 0, 0, 0, 0, 1559, 99, 0, 0, 0, 0, 0, 0, 1, 15, 1, 0, 0, 0, 1, 0, 3592, 348, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 103, 8, 0, 0, 0, 0, 0, 0, 0, 1092, 0, 1, 0, 0, 0, 4018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 3892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 16, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 1, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 503, 10, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2848, 3981, 4133, 5111, 3908, 4106, 4092, 4589] avg=4096.0000 max=5111 max_rank=3 min=2848 min_rank=0 imbalance_ratio=1.247803 max_load_reduction=27657 max_load_reduction_pct=0.844025 moved_to_echo=16695 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3655, 0, 0, 0, 0, 0, 0, 0, 2127, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 3592, 476, 22, 0, 0, 0, 0, 0, 0, 13, 0, 0, 3, 0, 0, 4012, 103, 8, 0, 0, 0, 0, 0, 0, 3, 1490, 0, 0, 0, 0, 0, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 503, 0, 0, 1, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5881, 4094, 4139, 2012, 3891, 4089, 4073, 4589] avg=4096.0000 max=5881 max_rank=0 min=2012 min_rank=3 imbalance_ratio=1.435791 max_load_reduction=26887 max_load_reduction_pct=0.820526 moved_to_echo=16642 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4060, 0, 0, 0, 0, 0, 28, 0, 1391, 104, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 3597, 314, 23, 0, 0, 0, 0, 0, 4, 33, 0, 0, 0, 0, 0, 3710, 106, 8, 0, 0, 0, 0, 0, 0, 0, 965, 0, 0, 0, 0, 96, 3801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1335, 0, 22, 0, 0, 3886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 42, 0, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 499, 33, 0, 0, 0, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5583, 3962, 3861, 4862, 5243, 3979, 4098, 1180] avg=4096.0000 max=5583 max_rank=0 min=1180 min_rank=7 imbalance_ratio=1.363037 max_load_reduction=27185 max_load_reduction_pct=0.829620 moved_to_echo=14500 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[14848, 128, 0, 1664, 0, 0, 128, 128, 15488, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 10880, 0, 128, 0, 0, 128, 21888, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 4096, 128, 0, 128, 0, 28800, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 512, 128, 0, 0, 128, 128, 0, 28800, 3584, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 128, 0, 0, 31872, 896, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 0, 256, 0, 32640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[256, 0, 128, 128, 0, 128, 0, 128, 32640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 1664, 0, 128, 0, 0, 31232, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3, 9, 6, 0, 0, 0, 0, 4092, 1, 0, 0, 4070, 0, 0, 0, 0, 0, 14, 5, 2, 0, 123, 0, 0, 4092, 1, 0, 3965, 3, 0, 1, 1, 0, 1, 0, 4094, 0, 0, 1, 0, 0, 0, 4070, 17, 0, 0, 0, 0, 0, 0, 7, 7, 0, 0, 0, 4, 4092, 0, 4087] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4007, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 61, 0, 0, 0, 1, 0, 4, 4096, 0, 0, 4096, 12, 0, 0, 0, 0, 0, 0, 4096, 54, 0, 0, 0, 0, 0, 4090, 4, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 4080, 0, 4029] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3413, 1, 0, 0, 0, 0, 0, 0, 0, 0, 438, 0, 0, 4096, 0, 0, 3926, 30, 0, 0, 0, 436, 0, 0, 4096, 0, 0, 76, 1, 0, 0, 3845, 0, 0, 0, 0, 0, 0, 0, 125, 8, 1, 0, 2, 0, 4095, 0, 4083] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 3896, 0, 0, 0, 0, 4, 0, 0, 0, 0, 176, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 419, 0, 0, 4095, 0, 0, 21, 0, 0, 0, 4065, 0, 0, 2, 0, 0, 0, 0, 3, 4, 0, 2, 0, 0, 4096, 0, 3706] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 4095, 0, 0, 0, 4018, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 80, 0, 0, 4090, 0, 0, 18, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 4051, 0, 4091] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 407, 0, 0, 0, 0, 0, 1, 0, 0, 0, 58, 0, 0, 4096, 0, 0, 4037, 0, 0, 0, 0, 3684, 0, 0, 4096, 0, 0, 10, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 4091] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 2659, 0, 0, 0, 0, 0, 1698, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3824, 180, 0, 0, 0, 0, 0, 0, 4096, 444, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 30, 0, 4096, 0, 3432] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 3684, 33, 0, 0, 0, 0, 24, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 3998, 10, 0, 0, 0, 0, 0, 0, 4096, 103, 0, 372, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4007, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 61, 0, 413, 100, 81, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3569, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 3993, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 3955, 114, 2, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 125, 0, 4029, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4023, 4748, 4001, 3581, 4047, 4075, 4198] avg=4096.0000 max=4748 max_rank=2 min=3581 min_rank=4 imbalance_ratio=1.159180 max_load_reduction=28020 max_load_reduction_pct=0.855103 moved_to_echo=15901 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3, 9, 6, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 14, 5, 400, 100, 80, 0, 0, 0, 0, 0, 2, 0, 123, 0, 0, 3992, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3455, 3, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3974, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 7, 3966, 110, 2, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4, 126, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4109, 4669, 4118, 3461, 3975, 4102, 4224] avg=4096.0000 max=4669 max_rank=2 min=3461 min_rank=4 imbalance_ratio=1.139893 max_load_reduction=28099 max_load_reduction_pct=0.857513 moved_to_echo=15762 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 3413, 1, 0, 0, 0, 0, 0, 0, 396, 100, 76, 0, 0, 0, 0, 0, 0, 0, 438, 0, 0, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3421, 30, 0, 0, 0, 436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 1, 0, 0, 3754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 3969, 109, 2, 0, 0, 0, 0, 0, 8, 1, 0, 2, 0, 126, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4111, 3986, 4434, 3887, 3831, 4205, 4220] avg=4096.0000 max=4434 max_rank=3 min=3831 min_rank=5 imbalance_ratio=1.082520 max_load_reduction=28334 max_load_reduction_pct=0.864685 moved_to_echo=16143 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 3684, 33, 0, 0, 0, 0, 24, 0, 403, 100, 81, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3484, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 0, 372, 0, 0, 0, 3990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3970, 111, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4086, 4325, 3999, 3494, 4465, 4083, 4222] avg=4096.0000 max=4465 max_rank=5 min=3494 min_rank=4 imbalance_ratio=1.090088 max_load_reduction=28303 max_load_reduction_pct=0.863739 moved_to_echo=16264 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 2, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 4018, 5, 0, 0, 0, 0, 0, 0, 413, 100, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3569, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 1, 3998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3927, 114, 2, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 124, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4111, 4617, 3996, 3649, 4017, 4043, 4244] avg=4096.0000 max=4617 max_rank=2 min=3649 min_rank=4 imbalance_ratio=1.127197 max_load_reduction=28151 max_load_reduction_pct=0.859100 moved_to_echo=15953 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 407, 0, 0, 0, 0, 0, 1, 0, 407, 100, 81, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3518, 0, 0, 0, 0, 3684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 3998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3970, 112, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 126, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4111, 996, 4054, 7202, 4008, 4084, 4219] avg=4096.0000 max=7202 max_rank=4 min=996 min_rank=2 imbalance_ratio=1.758301 max_load_reduction=25566 max_load_reduction_pct=0.780212 moved_to_echo=19513 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 3896, 0, 0, 0, 0, 4, 0, 0, 413, 100, 80, 0, 0, 0, 0, 0, 0, 0, 176, 0, 0, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3569, 0, 0, 0, 0, 419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 3969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3, 3970, 114, 2, 0, 0, 0, 0, 0, 4, 0, 2, 0, 0, 126, 0, 3706, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4103, 4493, 4172, 3988, 3990, 4091, 3838] avg=4096.0000 max=4493 max_rank=2 min=3838 min_rank=7 imbalance_ratio=1.096924 max_load_reduction=28275 max_load_reduction_pct=0.862885 moved_to_echo=15907 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 2, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 2659, 0, 0, 0, 0, 0, 1698, 0, 390, 102, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3326, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 444, 0, 0, 0, 0, 0, 3991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3976, 108, 1, 0, 0, 0, 0, 0, 0, 21, 0, 30, 0, 120, 0, 3432, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4114, 4934, 3994, 3506, 4435, 4085, 3603] avg=4096.0000 max=4934 max_rank=2 min=3506 min_rank=4 imbalance_ratio=1.204590 max_load_reduction=27834 max_load_reduction_pct=0.849426 moved_to_echo=15629 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[28032, 256, 0, 128, 128, 4736, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[26240, 128, 0, 0, 0, 128, 1920, 128, 3328, 896, 768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 640, 0, 512, 128, 0, 128, 31744, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 896, 0, 128, 32000, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 0, 0, 0, 0, 256, 31744, 896, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 0, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 32640, 128, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 128, 1024, 0, 31616, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 4093, 4094, 0, 0, 0, 4058, 0, 0, 4094, 0, 0, 4091, 0, 0, 0, 13, 0, 1, 3894, 1, 0, 0, 1, 4061, 0, 30, 0, 0, 0, 157, 0, 1, 92, 1, 1, 1, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 2, 0, 0, 4069, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3, 0, 0, 4096, 4096, 0, 0, 0, 4081, 0, 0, 4094, 9, 0, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4091, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 15, 0, 4044, 0, 0, 4096, 0, 0, 4070, 0, 0, 0, 0, 1, 0, 3948, 0, 0, 0, 0, 4090, 0, 66, 0, 0, 0, 132, 0, 0, 2, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 4043, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3771, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 1, 0, 0, 325, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 4088, 0, 0, 4096, 5, 0, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4042, 3, 0, 0, 0, 9, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 62, 0, 4064, 0, 0, 4096, 0, 0, 4096, 0, 0, 16, 0, 0, 0, 3964, 0, 0, 0, 0, 4052, 0, 28, 0, 0, 0, 131, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 54, 0, 4096, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 2524, 0, 118, 0, 0, 0, 0, 0, 0, 1421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 209, 1, 0, 0, 1, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 157, 0, 1, 3997, 12, 0, 0, 0, 0, 0, 0, 92, 1, 1, 1, 4, 1, 0, 0, 3675, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4082, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 4069, 1, 0, 10, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4068, 4097, 4273, 4197, 3861, 4086, 4091] avg=4096.0000 max=4273 max_rank=3 min=3861 min_rank=5 imbalance_ratio=1.043213 max_load_reduction=28495 max_load_reduction_pct=0.869598 moved_to_echo=16235 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 15, 0, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 212, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 132, 0, 0, 3999, 11, 0, 0, 0, 0, 0, 0, 2, 0, 0, 18, 0, 0, 0, 0, 3726, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 0, 0, 10, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4069, 4063, 4303, 4208, 3833, 4136, 4060] avg=4096.0000 max=4303 max_rank=3 min=3833 min_rank=5 imbalance_ratio=1.050537 max_load_reduction=28465 max_load_reduction_pct=0.868683 moved_to_echo=16237 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 220, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3999, 11, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3864, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 11, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4106, 4089, 4316, 4011, 3953, 4085, 4112] avg=4096.0000 max=4316 max_rank=3 min=3953 min_rank=5 imbalance_ratio=1.053711 max_load_reduction=28452 max_load_reduction_pct=0.868286 moved_to_echo=16161 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 220, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3999, 11, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 3861, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 11, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4091, 4098, 4316, 4012, 3955, 4088, 4109] avg=4096.0000 max=4316 max_rank=3 min=3955 min_rank=5 imbalance_ratio=1.053711 max_load_reduction=28452 max_load_reduction_pct=0.868286 moved_to_echo=16164 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 1, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 220, 0, 0, 0, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 9, 0, 0, 0, 3999, 11, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 3865, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 11, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4099, 4087, 4262, 4022, 4003, 4086, 4114] avg=4096.0000 max=4262 max_rank=3 min=4003 min_rank=5 imbalance_ratio=1.040527 max_load_reduction=28506 max_load_reduction_pct=0.869934 moved_to_echo=16225 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 202, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 3999, 12, 0, 0, 0, 0, 0, 0, 325, 0, 0, 5, 0, 0, 0, 0, 3559, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 10, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4101, 4089, 4298, 4013, 3976, 4084, 4111] avg=4096.0000 max=4298 max_rank=3 min=3976 min_rank=5 imbalance_ratio=1.049316 max_load_reduction=28470 max_load_reduction_pct=0.868835 moved_to_echo=16184 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 62, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 213, 0, 0, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 131, 0, 0, 3999, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3741, 87, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 4085, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 0, 10, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4136, 4105, 4265, 4169, 3831, 4117, 4050] avg=4096.0000 max=4265 max_rank=3 min=3831 min_rank=5 imbalance_ratio=1.041260 max_load_reduction=28503 max_load_reduction_pct=0.869843 moved_to_echo=16167 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 211, 0, 0, 0, 0, 2524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 0, 0, 0, 0, 0, 0, 4006, 12, 0, 0, 0, 0, 0, 0, 1421, 0, 0, 0, 0, 0, 0, 0, 3847, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 15, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4098, 4140, 2735, 4136, 5356, 4086, 4121] avg=4096.0000 max=5356 max_rank=5 min=2735 min_rank=3 imbalance_ratio=1.307617 max_load_reduction=27412 max_load_reduction_pct=0.836548 moved_to_echo=17699 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 1792, 128, 0, 0, 128, 31104, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 0, 0, 128, 0, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 256, 0, 0, 128, 512, 0, 128, 32000, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 0, 0, 32640, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[1920, 128, 128, 128, 128, 128, 128, 0, 30208, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 0, 32768, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=32896 slack_slots=2816 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 4061, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4062, 0, 0, 0, 0, 0, 0, 3, 3910, 0, 0, 0, 0, 0, 0, 14, 4096, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 186, 0, 0, 316, 79, 0, 1, 0, 0, 0, 0, 0, 186, 0, 8, 3536, 0, 0, 4071, 0, 4085, 0, 22, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4068, 0, 2, 0, 0, 0, 0, 39, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1170, 0, 0, 4096, 0, 4095, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[3, 4022, 11, 0, 0, 0, 1, 3, 0, 0, 0, 4034, 4093, 0, 0, 0, 0, 0, 0, 9, 4087, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 48, 0, 2, 0, 0, 0, 6, 1535, 0, 0, 26, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2562, 0, 0, 4080, 3, 4093, 0, 69, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 0, 0, 4048, 0, 4096, 0, 0, 1, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 169, 0, 4013, 0, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 1892, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 7, 23, 0, 0, 2085, 0, 0, 4056, 0, 4095, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1, 4078, 0, 0, 0, 1, 0, 0, 35, 0, 0, 4087, 4095, 0, 11, 0, 0, 0, 2, 0, 3994, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 0, 2, 0, 3, 0, 0, 0, 0, 0, 32, 0, 0, 4011, 0, 0, 4089, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[5, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4083, 4044, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 44, 0, 0, 1, 0, 0, 0, 305, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 3777, 0, 0, 4096, 16, 4096, 0, 0, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 4077, 0, 0, 0, 0, 0, 0, 10, 0, 0, 3930, 4092, 0, 7, 0, 0, 0, 4033, 0, 365, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 190, 0, 0, 0, 0, 0, 1053, 0, 0, 0, 0, 0, 14, 0, 0, 2729, 0, 0, 4074, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4061, 1, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 186, 2966, 188, 0, 0, 0, 0, 0, 0, 0, 0, 316, 79, 0, 1, 0, 0, 3891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 0, 8, 3536, 0, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 6, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4120, 3894, 4113, 3375, 4287, 4802, 4099] avg=4096.0000 max=4802 max_rank=6 min=3375 min_rank=4 imbalance_ratio=1.172363 max_load_reduction=27966 max_load_reduction_pct=0.853455 moved_to_echo=16563 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 24, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2914, 2970, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1170, 0, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4122, 4115, 4099, 6072, 3901, 2244, 4102] avg=4096.0000 max=6072 max_rank=4 min=2244 min_rank=6 imbalance_ratio=1.482422 max_load_reduction=26696 max_load_reduction_pct=0.814697 moved_to_echo=16319 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3, 4022, 11, 0, 0, 0, 1, 3, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 48, 0, 2, 0, 0, 0, 6, 1535, 2988, 188, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 2, 0, 1, 0, 3900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2562, 0, 1080, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 3, 5, 0, 69, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4057, 4059, 4076, 4078, 4767, 3929, 3642, 4160] avg=4096.0000 max=4767 max_rank=4 min=3642 min_rank=6 imbalance_ratio=1.163818 max_load_reduction=28001 max_load_reduction_pct=0.854523 moved_to_echo=16498 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 4078, 0, 0, 0, 1, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 4087, 24, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 2990, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 0, 0, 3902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 4011, 0, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4157, 3976, 4097, 3315, 3907, 5124, 4095] avg=4096.0000 max=5124 max_rank=6 min=3315 min_rank=4 imbalance_ratio=1.250977 max_load_reduction=27644 max_load_reduction_pct=0.843628 moved_to_echo=16441 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4089, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 0, 3993, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 1892, 2990, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 3901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 23, 0, 0, 2085, 0, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4121, 4165, 4099, 5073, 3946, 3196, 4062] avg=4096.0000 max=5073 max_rank=4 min=3196 min_rank=6 imbalance_ratio=1.238525 max_load_reduction=27695 max_load_reduction_pct=0.845184 moved_to_echo=16277 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 1, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 2990, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 3902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 0, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 6, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4120, 4076, 4083, 3300, 3929, 5090, 4056] avg=4096.0000 max=5090 max_rank=6 min=3300 min_rank=4 imbalance_ratio=1.242676 max_load_reduction=27678 max_load_reduction_pct=0.844666 moved_to_echo=16375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[5, 4096, 0, 1, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 1, 0, 0, 0, 305, 2952, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 3902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3777, 0, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 16, 6, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4108, 4076, 4099, 3490, 3912, 4844, 4120] avg=4096.0000 max=4844 max_rank=6 min=3490 min_rank=4 imbalance_ratio=1.182617 max_load_reduction=27924 max_load_reduction_pct=0.852173 moved_to_echo=16366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4077, 0, 0, 0, 0, 0, 0, 4, 3, 0, 0, 0, 0, 0, 0, 10, 0, 0, 3930, 14, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 190, 2992, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 0, 0, 3904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 2729, 0, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 3961, 4390, 4100, 3376, 4957, 3826, 4074] avg=4096.0000 max=4957 max_rank=5 min=3376 min_rank=4 imbalance_ratio=1.210205 max_load_reduction=27811 max_load_reduction_pct=0.848724 moved_to_echo=16233 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 0, 0, 128, 7296, 23936, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 128, 256, 0, 128, 23936, 0, 8704, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 384, 256, 128, 1152, 128, 0, 31232, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 0, 32640, 256, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 32768, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 32640, 128, 128, 0, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 4224, 128, 28544, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 32640, 128, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True + [2026-09-21 08:25:37] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 254.5 | throughput per GPU (TFLOP/s/GPU): 126.6 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.666759E+00 | loss scale: 1.0 | grad norm: 8.476 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[263, 519, 491, 1, 2, 845, 984, 243, 8, 1471, 16, 0, 26, 154, 459, 323, 4, 1010, 386, 2195, 13, 281, 201, 16, 195, 20, 65, 311, 894, 896, 720, 1385, 335, 354, 534, 612, 87, 1928, 247, 1019, 682, 932, 1339, 31, 1873, 768, 21, 162, 750, 520, 669, 29, 726, 1181, 1308, 62, 1369, 14, 250, 2, 11, 49, 471, 36] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[300, 429, 343, 3, 2, 534, 1470, 215, 2, 1306, 33, 0, 59, 39, 124, 154, 52, 1027, 755, 2376, 29, 377, 240, 27, 210, 28, 8, 268, 1103, 1394, 524, 1417, 352, 242, 447, 476, 159, 1239, 218, 779, 958, 1268, 1782, 41, 1573, 553, 27, 262, 688, 641, 750, 47, 452, 1466, 1285, 54, 1082, 16, 439, 16, 21, 123, 436, 28] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[422, 301, 473, 3, 0, 728, 1008, 165, 1, 1438, 9, 5, 3, 104, 63, 153, 1, 971, 533, 1842, 27, 459, 246, 11, 111, 5, 20, 414, 923, 739, 714, 1567, 206, 178, 397, 849, 67, 1874, 223, 1058, 997, 682, 1543, 20, 1819, 1128, 12, 159, 920, 538, 784, 9, 689, 1033, 1660, 51, 1626, 15, 337, 3, 14, 10, 385, 23] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[368, 373, 324, 3, 1, 497, 1244, 165, 3, 1055, 33, 2, 12, 36, 189, 158, 5, 1264, 709, 1538, 24, 548, 208, 5, 148, 4, 68, 251, 1173, 780, 651, 1085, 242, 197, 534, 647, 76, 1856, 282, 830, 1001, 940, 1539, 27, 1670, 1173, 25, 178, 988, 683, 707, 47, 780, 1521, 1597, 103, 1311, 17, 246, 1, 18, 98, 479, 31] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[354, 602, 279, 2, 0, 470, 1573, 72, 8, 1048, 63, 4, 17, 10, 188, 317, 15, 912, 523, 1680, 36, 63, 316, 20, 242, 12, 21, 374, 1379, 391, 779, 1450, 301, 216, 555, 423, 177, 1590, 369, 571, 961, 864, 2016, 12, 1422, 1082, 8, 100, 639, 1233, 626, 62, 984, 874, 1444, 15, 1544, 9, 360, 37, 9, 131, 903, 11] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[287, 514, 343, 1, 0, 585, 1577, 109, 2, 1363, 27, 0, 2, 23, 167, 257, 11, 913, 395, 2249, 18, 197, 266, 15, 237, 40, 72, 258, 1393, 866, 633, 1437, 377, 174, 430, 784, 136, 1705, 242, 597, 951, 1027, 1544, 42, 1366, 817, 8, 246, 778, 862, 489, 37, 671, 1252, 1560, 21, 1627, 9, 244, 12, 8, 36, 435, 24] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[313, 314, 581, 19, 0, 547, 1406, 176, 14, 1324, 40, 0, 19, 12, 294, 261, 8, 822, 592, 1715, 21, 77, 181, 2, 79, 11, 62, 328, 1192, 958, 553, 1521, 586, 288, 519, 635, 74, 2267, 176, 699, 995, 800, 2101, 29, 1640, 797, 27, 121, 922, 568, 702, 45, 449, 1020, 1331, 74, 1702, 39, 352, 15, 12, 22, 304, 15] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[204, 519, 415, 4, 1, 706, 1207, 63, 1, 1674, 38, 1, 6, 101, 305, 255, 11, 734, 392, 2058, 17, 160, 299, 26, 164, 27, 45, 570, 963, 439, 892, 1688, 315, 332, 575, 586, 230, 1328, 464, 948, 835, 757, 1709, 15, 1825, 892, 18, 52, 643, 706, 584, 39, 739, 962, 1672, 44, 1742, 15, 211, 2, 8, 31, 475, 29] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[300, 429, 343, 3, 2, 534, 1470, 215, 410, 413, 0, 0, 0, 0, 0, 0, 2, 1306, 33, 0, 59, 39, 124, 154, 1782, 424, 0, 0, 0, 0, 0, 0, 52, 1027, 755, 2376, 29, 377, 240, 27, 57, 25, 0, 0, 0, 0, 0, 0, 210, 28, 8, 268, 1103, 1394, 524, 878, 0, 0, 0, 0, 0, 0, 0, 0, 352, 242, 447, 476, 159, 840, 218, 779, 0, 0, 0, 0, 0, 0, 0, 0, 958, 1268, 0, 41, 1106, 553, 27, 262, 0, 0, 0, 0, 0, 0, 0, 0, 688, 641, 750, 47, 452, 1441, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 1082, 16, 439, 16, 21, 123, 436, 28, 861, 399, 126, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 3923, 4965, 4413, 3513, 4215, 4073, 3547] avg=4096.0000 max=4965 max_rank=2 min=3513 min_rank=4 imbalance_ratio=1.212158 max_load_reduction=27803 max_load_reduction_pct=0.848480 moved_to_echo=15348 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[263, 519, 491, 1, 2, 845, 984, 243, 488, 403, 0, 0, 0, 0, 0, 0, 8, 1471, 16, 0, 26, 154, 459, 323, 1339, 432, 0, 0, 0, 0, 0, 0, 4, 1010, 386, 2195, 13, 281, 201, 16, 68, 20, 0, 0, 0, 0, 0, 0, 195, 20, 65, 311, 894, 896, 720, 859, 0, 0, 0, 0, 0, 0, 0, 0, 335, 354, 534, 612, 87, 1307, 247, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 682, 932, 0, 31, 1317, 768, 21, 162, 0, 0, 0, 0, 0, 0, 0, 0, 750, 520, 669, 29, 726, 1161, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 1369, 14, 250, 2, 11, 49, 471, 36, 876, 621, 123, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4239, 4228, 4194, 3960, 4495, 3913, 3917, 3822] avg=4096.0000 max=4495 max_rank=4 min=3822 min_rank=7 imbalance_ratio=1.097412 max_load_reduction=28273 max_load_reduction_pct=0.862823 moved_to_echo=16147 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[354, 602, 279, 2, 0, 470, 1573, 72, 370, 422, 0, 0, 0, 0, 0, 0, 8, 1048, 63, 4, 17, 10, 188, 317, 2016, 476, 0, 0, 0, 0, 0, 0, 15, 912, 523, 1680, 36, 63, 316, 20, 52, 15, 0, 0, 0, 0, 0, 0, 242, 12, 21, 374, 1379, 391, 779, 899, 0, 0, 0, 0, 0, 0, 0, 0, 301, 216, 555, 423, 177, 1078, 369, 571, 0, 0, 0, 0, 0, 0, 0, 0, 961, 864, 0, 12, 1000, 1082, 8, 100, 0, 0, 0, 0, 0, 0, 0, 0, 639, 1233, 626, 62, 984, 859, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 1544, 9, 360, 37, 9, 131, 903, 11, 968, 512, 129, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4144, 4147, 3632, 4097, 3690, 4027, 4418, 4613] avg=4096.0000 max=4613 max_rank=7 min=3632 min_rank=2 imbalance_ratio=1.126221 max_load_reduction=28155 max_load_reduction_pct=0.859222 moved_to_echo=16748 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[368, 373, 324, 3, 1, 497, 1244, 165, 435, 316, 0, 0, 0, 0, 0, 0, 3, 1055, 33, 2, 12, 36, 189, 158, 1539, 527, 0, 0, 0, 0, 0, 0, 5, 1264, 709, 1538, 24, 548, 208, 5, 61, 26, 0, 0, 0, 0, 0, 0, 148, 4, 68, 251, 1173, 780, 651, 673, 0, 0, 0, 0, 0, 0, 0, 0, 242, 197, 534, 647, 76, 1258, 282, 830, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 940, 0, 27, 1174, 1173, 25, 178, 0, 0, 0, 0, 0, 0, 0, 0, 988, 683, 707, 47, 780, 1495, 0, 103, 0, 0, 0, 0, 0, 0, 0, 0, 1311, 17, 246, 1, 18, 98, 479, 31, 1070, 598, 96, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3726, 3554, 4388, 3748, 4066, 4518, 4803, 3965] avg=4096.0000 max=4803 max_rank=6 min=3554 min_rank=1 imbalance_ratio=1.172607 max_load_reduction=27965 max_load_reduction_pct=0.853424 moved_to_echo=17352 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[313, 314, 581, 19, 0, 547, 1406, 176, 427, 443, 0, 0, 0, 0, 0, 0, 14, 1324, 40, 0, 19, 12, 294, 261, 2101, 440, 0, 0, 0, 0, 0, 0, 8, 822, 592, 1715, 21, 77, 181, 2, 60, 17, 0, 0, 0, 0, 0, 0, 79, 11, 62, 328, 1192, 958, 553, 943, 0, 0, 0, 0, 0, 0, 0, 0, 586, 288, 519, 635, 74, 1537, 176, 699, 0, 0, 0, 0, 0, 0, 0, 0, 995, 800, 0, 29, 1153, 797, 27, 121, 0, 0, 0, 0, 0, 0, 0, 0, 922, 568, 702, 45, 449, 1003, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 1702, 39, 352, 15, 12, 22, 304, 15, 891, 730, 135, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4226, 4505, 3495, 4126, 4514, 3922, 3763, 4217] avg=4096.0000 max=4514 max_rank=4 min=3495 min_rank=2 imbalance_ratio=1.102051 max_load_reduction=28254 max_load_reduction_pct=0.862244 moved_to_echo=16416 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[287, 514, 343, 1, 0, 585, 1577, 109, 356, 418, 0, 0, 0, 0, 0, 0, 2, 1363, 27, 0, 2, 23, 167, 257, 1544, 516, 0, 0, 0, 0, 0, 0, 11, 913, 395, 2249, 18, 197, 266, 15, 50, 21, 0, 0, 0, 0, 0, 0, 237, 40, 72, 258, 1393, 866, 633, 891, 0, 0, 0, 0, 0, 0, 0, 0, 377, 174, 430, 784, 136, 1156, 242, 597, 0, 0, 0, 0, 0, 0, 0, 0, 951, 1027, 0, 42, 960, 817, 8, 246, 0, 0, 0, 0, 0, 0, 0, 0, 778, 862, 489, 37, 671, 1231, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 1627, 9, 244, 12, 8, 36, 435, 24, 1044, 549, 128, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4190, 3901, 4135, 4390, 3896, 4051, 4089, 4116] avg=4096.0000 max=4390 max_rank=3 min=3896 min_rank=4 imbalance_ratio=1.071777 max_load_reduction=28378 max_load_reduction_pct=0.866028 moved_to_echo=16152 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[422, 301, 473, 3, 0, 728, 1008, 165, 474, 456, 0, 0, 0, 0, 0, 0, 1, 1438, 9, 5, 3, 104, 63, 153, 1543, 548, 0, 0, 0, 0, 0, 0, 1, 971, 533, 1842, 27, 459, 246, 11, 66, 17, 0, 0, 0, 0, 0, 0, 111, 5, 20, 414, 923, 739, 714, 971, 0, 0, 0, 0, 0, 0, 0, 0, 206, 178, 397, 849, 67, 1270, 223, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 997, 682, 0, 20, 1279, 1128, 12, 159, 0, 0, 0, 0, 0, 0, 0, 0, 920, 538, 784, 9, 689, 1016, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 1626, 15, 337, 3, 14, 10, 385, 23, 1112, 604, 140, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4030, 3867, 4173, 3897, 4248, 4277, 4007, 4269] avg=4096.0000 max=4277 max_rank=5 min=3867 min_rank=1 imbalance_ratio=1.044189 max_load_reduction=28491 max_load_reduction_pct=0.869476 moved_to_echo=16801 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[204, 519, 415, 4, 1, 706, 1207, 63, 479, 496, 0, 0, 0, 0, 0, 0, 1, 1674, 38, 1, 6, 101, 305, 255, 1709, 553, 0, 0, 0, 0, 0, 0, 11, 734, 392, 2058, 17, 160, 299, 26, 70, 19, 0, 0, 0, 0, 0, 0, 164, 27, 45, 570, 963, 439, 892, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 315, 332, 575, 586, 230, 896, 464, 948, 0, 0, 0, 0, 0, 0, 0, 0, 835, 757, 0, 15, 1276, 892, 18, 52, 0, 0, 0, 0, 0, 0, 0, 0, 643, 706, 584, 39, 739, 943, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 1742, 15, 211, 2, 8, 31, 475, 29, 1119, 432, 155, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4643, 3786, 4137, 4346, 3845, 3698, 4219] avg=4096.0000 max=4643 max_rank=1 min=3698 min_rank=6 imbalance_ratio=1.133545 max_load_reduction=28125 max_load_reduction_pct=0.858307 moved_to_echo=16108 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[2560, 3584, 3328, 128, 128, 4992, 10496, 1280, 3456, 3456, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[7424, 7296, 0, 256, 9344, 7296, 256, 1280, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 7680, 4352, 15744, 256, 2176, 2048, 128, 512, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[12032, 256, 2560, 128, 128, 512, 3968, 256, 8064, 4480, 1152, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[2816, 2048, 4096, 5120, 1024, 9344, 2304, 6528, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 10752, 384, 128, 256, 512, 1792, 1920, 13696, 3968, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1408, 256, 384, 2816, 9088, 6528, 5504, 7168, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[6400, 5760, 5376, 384, 5504, 9216, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[97, 0, 1118, 3, 16, 0, 0, 0, 2, 0, 137, 988, 0, 0, 3361, 0, 1, 3, 3862, 0, 4, 3, 153, 0, 0, 444, 0, 1, 0, 0, 3226, 1457, 0, 0, 11, 0, 6, 143, 1163, 6, 3850, 0, 3963, 0, 6, 0, 89, 0, 945, 0, 0, 0, 73, 0, 0, 0, 3515, 1, 1105, 2590, 417, 3, 6, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[87, 0, 1732, 1, 0, 0, 0, 0, 0, 0, 1319, 151, 2, 0, 1107, 6, 0, 0, 4094, 0, 0, 14, 1929, 0, 0, 673, 0, 0, 34, 0, 2437, 1493, 0, 1, 0, 0, 0, 0, 2596, 1, 4096, 0, 4078, 0, 0, 0, 8, 0, 946, 0, 1, 0, 2, 0, 0, 17, 2418, 0, 1773, 1464, 284, 0, 4, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1, 8, 404, 0, 0, 2, 0, 0, 9, 0, 1269, 1138, 151, 0, 243, 0, 0, 0, 4075, 0, 28, 31, 1588, 14, 0, 7, 0, 3, 1, 0, 4073, 841, 4, 0, 0, 0, 0, 20, 2576, 3, 4085, 0, 4008, 1, 6, 0, 0, 0, 463, 5, 188, 1, 0, 0, 0, 0, 3983, 2, 1791, 1268, 467, 0, 10, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[48, 1, 825, 0, 0, 79, 0, 0, 145, 0, 1553, 482, 61, 0, 25, 0, 3, 0, 4053, 0, 755, 16, 3201, 1, 0, 414, 0, 10, 533, 19, 2244, 671, 5, 0, 0, 0, 7, 0, 4004, 38, 4095, 164, 2170, 0, 0, 0, 105, 3, 65, 1, 14, 1, 0, 0, 1, 1, 1155, 0, 3566, 883, 110, 0, 1241, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 1242, 0, 0, 5, 0, 1, 11, 50, 168, 121, 6, 0, 3445, 0, 69, 0, 4022, 5, 0, 67, 824, 16, 1, 76, 0, 0, 14, 0, 3513, 1371, 0, 0, 0, 0, 6, 0, 3457, 2, 4089, 9, 3649, 1, 5, 0, 0, 6, 695, 406, 0, 0, 0, 0, 2, 0, 3271, 14, 1484, 632, 12, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1, 4, 132, 1, 5, 4, 0, 0, 123, 0, 2789, 94, 0, 0, 1593, 0, 1, 0, 3928, 3, 78, 1, 1735, 8, 0, 1147, 0, 0, 236, 0, 2447, 1626, 9, 1, 1, 1, 41, 0, 3338, 0, 3967, 1, 2991, 1, 0, 0, 0, 0, 267, 3, 4, 18, 3, 1, 0, 0, 2705, 1, 692, 309, 2131, 4, 323, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1, 1, 485, 0, 4, 108, 0, 16, 1, 175, 2979, 277, 3, 0, 2865, 0, 0, 0, 3969, 0, 0, 4, 2644, 60, 0, 37, 0, 1, 13, 0, 3988, 582, 0, 0, 10, 0, 63, 0, 1898, 4, 3920, 0, 3848, 0, 1, 0, 2, 2, 5, 0, 3, 5, 0, 1, 0, 0, 1069, 16, 851, 1767, 1061, 0, 29, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[17, 1, 2010, 0, 0, 2, 0, 2, 3, 2, 3282, 1519, 9, 0, 1041, 14, 4, 0, 4090, 7, 0, 0, 764, 12, 0, 165, 0, 3, 12, 125, 2939, 1024, 2, 13, 0, 0, 0, 0, 340, 73, 3964, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 103, 3974, 1, 1133, 653, 1193, 0, 179, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[48, 1, 825, 0, 0, 79, 0, 0, 981, 698, 0, 0, 0, 0, 0, 0, 145, 0, 1553, 482, 61, 0, 25, 0, 4, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2365, 0, 755, 16, 3201, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 414, 0, 10, 533, 19, 1782, 671, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 7, 0, 4004, 38, 990, 458, 0, 0, 0, 0, 0, 0, 456, 164, 2170, 0, 0, 0, 105, 3, 0, 0, 0, 0, 0, 0, 0, 0, 65, 1, 14, 1, 0, 0, 1, 1, 3639, 1, 0, 0, 0, 0, 0, 0, 173, 0, 3566, 883, 110, 0, 1241, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2632, 2270, 6341, 3429, 5502, 2898, 3723, 5973] avg=4096.0000 max=6341 max_rank=2 min=2270 min_rank=1 imbalance_ratio=1.548096 max_load_reduction=26427 max_load_reduction_pct=0.806488 moved_to_echo=18096 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 4, 132, 1, 5, 4, 0, 0, 2303, 680, 0, 0, 0, 0, 0, 0, 123, 0, 2789, 94, 0, 0, 1593, 0, 9, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2285, 3, 78, 1, 1735, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1147, 0, 0, 236, 0, 1934, 1626, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 1, 1, 41, 0, 3338, 0, 963, 504, 0, 0, 0, 0, 0, 0, 437, 1, 2991, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 267, 3, 4, 18, 3, 1, 0, 0, 3530, 6, 0, 0, 0, 0, 0, 0, 396, 1, 692, 309, 2131, 4, 323, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3130, 4608, 4111, 4943, 4858, 3430, 3832, 3856] avg=4096.0000 max=4943 max_rank=3 min=3130 min_rank=0 imbalance_ratio=1.206787 max_load_reduction=27825 max_load_reduction_pct=0.849152 moved_to_echo=15976 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[17, 1, 2010, 0, 0, 2, 0, 2, 3377, 704, 0, 0, 0, 0, 0, 0, 3, 2, 3282, 1519, 9, 0, 1041, 14, 5, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2387, 7, 0, 0, 764, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 0, 3, 12, 125, 2334, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 2, 13, 0, 0, 0, 0, 340, 73, 999, 600, 0, 0, 0, 0, 0, 0, 441, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 103, 3523, 3, 0, 0, 0, 0, 0, 0, 594, 1, 1133, 653, 1193, 0, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6113, 5875, 3174, 3663, 2027, 4528, 3635, 3753] avg=4096.0000 max=6113 max_rank=0 min=2027 min_rank=4 imbalance_ratio=1.492432 max_load_reduction=26655 max_load_reduction_pct=0.813446 moved_to_echo=13943 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 8, 404, 0, 0, 2, 0, 0, 3384, 702, 0, 0, 0, 0, 0, 0, 9, 0, 1269, 1138, 151, 0, 243, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2378, 0, 28, 31, 1588, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 3, 1, 0, 3233, 841, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 20, 2576, 3, 995, 832, 0, 0, 0, 0, 0, 0, 455, 0, 4008, 1, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 5, 188, 1, 0, 0, 0, 0, 3630, 3, 0, 0, 0, 0, 0, 0, 596, 2, 1791, 1268, 467, 0, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4501, 2818, 4039, 4085, 4430, 4470, 4290, 4135] avg=4096.0000 max=4501 max_rank=0 min=2818 min_rank=1 imbalance_ratio=1.098877 max_load_reduction=28267 max_load_reduction_pct=0.862640 moved_to_echo=17325 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1242, 0, 0, 5, 0, 1, 2779, 693, 0, 0, 0, 0, 0, 0, 11, 50, 168, 121, 6, 0, 3445, 0, 6, 0, 0, 0, 0, 0, 0, 0, 69, 0, 2347, 5, 0, 67, 824, 16, 0, 0, 0, 0, 0, 0, 0, 0, 1, 76, 0, 0, 14, 0, 2789, 1371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 3457, 2, 982, 718, 0, 0, 0, 0, 0, 0, 455, 9, 3649, 1, 5, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 695, 406, 0, 0, 0, 0, 2, 0, 3634, 2, 0, 0, 0, 0, 0, 0, 490, 14, 1484, 632, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4721, 3807, 3328, 4251, 5165, 4125, 4739, 2632] avg=4096.0000 max=5165 max_rank=4 min=2632 min_rank=7 imbalance_ratio=1.260986 max_load_reduction=27603 max_load_reduction_pct=0.842377 moved_to_echo=16661 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 485, 0, 4, 108, 0, 16, 908, 683, 0, 0, 0, 0, 0, 0, 1, 175, 2979, 277, 3, 0, 2865, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2317, 0, 0, 4, 2644, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 1, 13, 0, 3166, 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 63, 0, 1898, 4, 969, 815, 0, 0, 0, 0, 0, 0, 436, 0, 3848, 0, 1, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3, 5, 0, 1, 0, 0, 3484, 0, 0, 0, 0, 0, 0, 0, 161, 16, 851, 1767, 1061, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2206, 6307, 5025, 3799, 3759, 4289, 3498, 3885] avg=4096.0000 max=6307 max_rank=1 min=2206 min_rank=0 imbalance_ratio=1.539795 max_load_reduction=26461 max_load_reduction_pct=0.807526 moved_to_echo=15431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[97, 0, 1118, 3, 16, 0, 0, 0, 2987, 665, 0, 0, 0, 0, 0, 0, 2, 0, 137, 988, 0, 0, 3361, 0, 6, 0, 0, 0, 0, 0, 0, 0, 1, 3, 2254, 0, 4, 3, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 444, 0, 1, 0, 0, 2561, 1457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 6, 143, 1163, 6, 943, 659, 0, 0, 0, 0, 0, 0, 428, 0, 3963, 0, 6, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 945, 0, 0, 0, 73, 0, 0, 0, 3422, 3, 0, 0, 0, 0, 0, 0, 525, 1, 1105, 2590, 417, 3, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4886, 4494, 2418, 4463, 2931, 4486, 4443, 4647] avg=4096.0000 max=4886 max_rank=0 min=2418 min_rank=2 imbalance_ratio=1.192871 max_load_reduction=27882 max_load_reduction_pct=0.850891 moved_to_echo=16507 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[87, 0, 1732, 1, 0, 0, 0, 0, 2054, 705, 0, 0, 0, 0, 0, 0, 0, 0, 1319, 151, 2, 0, 1107, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2389, 0, 0, 14, 1929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 673, 0, 0, 34, 0, 1935, 1493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2596, 1, 1000, 498, 0, 0, 0, 0, 0, 0, 456, 0, 4078, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 946, 0, 1, 0, 2, 0, 0, 17, 3640, 2, 0, 0, 0, 0, 0, 0, 362, 0, 1773, 1464, 284, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4579, 2589, 4332, 4135, 4096, 4542, 4608, 3887] avg=4096.0000 max=4608 max_rank=6 min=2589 min_rank=1 imbalance_ratio=1.125000 max_load_reduction=28160 max_load_reduction_pct=0.859375 moved_to_echo=17133 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 18816, 128, 896, 256, 12928, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 256, 19456, 128, 7936, 5120, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3328, 128, 12416, 9600, 5760, 128, 1792, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[256, 128, 8064, 128, 128, 256, 0, 128, 18816, 5632, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 3072, 0, 128, 896, 256, 19840, 9088, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[384, 256, 13568, 4864, 256, 0, 13696, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[3456, 512, 256, 128, 128, 128, 128, 128, 28544, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3584, 256, 28800, 128, 128, 0, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[8, 0, 111, 0, 3392, 0, 4071, 0, 5, 0, 0, 4091, 0, 1, 0, 3975, 3942, 8, 166, 0, 0, 0, 0, 0, 0, 0, 0, 873, 0, 136, 0, 23, 1, 18, 0, 0, 26, 0, 5, 0, 0, 0, 0, 0, 25, 1, 3712, 0, 3, 3800, 30, 661, 0, 14, 1029, 0, 2627, 0, 0, 0, 14, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 37, 0, 3906, 0, 4089, 0, 0, 0, 0, 4092, 0, 0, 0, 3939, 4088, 182, 12, 350, 0, 0, 0, 0, 0, 0, 0, 12, 0, 1, 0, 0, 2, 18, 1727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3966, 0, 0, 4082, 0, 5, 37, 19, 16, 0, 2099, 27, 0, 2, 0, 0, 58, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 3908, 0, 3981, 0, 1, 0, 3, 4058, 0, 0, 3, 3177, 4095, 4, 278, 8, 0, 0, 0, 1, 2, 0, 0, 857, 0, 507, 0, 0, 87, 2, 195, 0, 0, 0, 7, 0, 7, 0, 0, 10, 71, 15, 3865, 0, 0, 4068, 1, 1, 91, 54, 3298, 1, 16, 0, 0, 0, 82, 0, 12, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 127, 0, 2880, 0, 4074, 3, 1, 0, 28, 4061, 0, 0, 0, 3006, 4045, 13, 9, 292, 1, 0, 0, 0, 0, 0, 0, 0, 19, 2, 0, 11, 0, 19, 2281, 0, 22, 0, 0, 2, 0, 0, 0, 10, 0, 0, 3433, 0, 46, 4084, 0, 20, 0, 302, 483, 0, 3056, 4, 0, 3, 180, 28, 223, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 44, 0, 3949, 0, 4086, 16, 0, 18, 2, 4090, 0, 4, 0, 3624, 4079, 276, 0, 91, 0, 0, 0, 0, 11, 0, 5, 0, 997, 1037, 0, 0, 17, 372, 552, 5, 0, 0, 55, 2, 0, 0, 1, 15, 7, 143, 3122, 1, 22, 3915, 3, 586, 0, 716, 81, 7, 803, 1, 0, 1, 1, 0, 5, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 12, 0, 0, 3747, 0, 4094, 3, 0, 2, 35, 4087, 0, 8, 0, 4096, 4037, 0, 62, 550, 1, 0, 1, 0, 0, 0, 0, 787, 32, 141, 0, 0, 0, 31, 6, 3, 0, 6, 0, 0, 0, 8, 0, 0, 0, 0, 3827, 0, 167, 3640, 10, 124, 0, 195, 62, 1, 1888, 0, 0, 0, 1072, 0, 26, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3167, 0, 3924, 0, 6, 3, 1, 4090, 0, 0, 0, 4090, 4091, 173, 4, 16, 0, 0, 0, 0, 1, 0, 0, 79, 305, 31, 0, 0, 3, 0, 1184, 0, 6, 0, 0, 1, 0, 1, 2, 0, 9, 0, 4082, 0, 0, 4040, 0, 2, 0, 118, 234, 0, 3029, 0, 3, 1, 50, 1, 13, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1, 11, 0, 0, 3834, 0, 4075, 0, 0, 4, 0, 3942, 0, 0, 2, 3814, 3955, 0, 0, 2023, 0, 0, 0, 1, 0, 1, 0, 1, 0, 2137, 0, 0, 0, 2, 27, 0, 1, 0, 0, 0, 0, 0, 5, 0, 1, 82, 3938, 0, 0, 4047, 0, 100, 0, 0, 240, 0, 257, 2, 0, 0, 246, 0, 19, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 111, 0, 3392, 0, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 368, 0, 1, 0, 3975, 0, 0, 0, 0, 0, 0, 0, 0, 3445, 8, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 873, 0, 136, 0, 23, 2670, 441, 0, 0, 0, 0, 0, 0, 1, 18, 0, 0, 26, 0, 5, 0, 3282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 1, 3712, 0, 293, 0, 0, 0, 0, 0, 0, 0, 3, 2878, 30, 661, 0, 14, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2627, 0, 0, 0, 14, 0, 0, 0, 922, 946, 204, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3966, 4349, 3619, 4143, 3332, 4031, 4615, 4713] avg=4096.0000 max=4713 max_rank=7 min=3332 min_rank=4 imbalance_ratio=1.150635 max_load_reduction=28055 max_load_reduction_pct=0.856171 moved_to_echo=16691 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 127, 0, 2880, 0, 455, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 28, 365, 0, 0, 0, 3006, 0, 0, 0, 0, 0, 0, 0, 0, 3535, 13, 9, 292, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 2, 0, 11, 2672, 438, 0, 0, 0, 0, 0, 0, 0, 19, 2281, 0, 22, 0, 0, 2, 3258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 3433, 0, 301, 0, 0, 0, 0, 0, 0, 0, 46, 3093, 0, 20, 0, 302, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3056, 4, 0, 3, 180, 28, 223, 0, 991, 947, 209, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3465, 3400, 3850, 3142, 5582, 3744, 3944, 5641] avg=4096.0000 max=5641 max_rank=7 min=3142 min_rank=3 imbalance_ratio=1.377197 max_load_reduction=27127 max_load_reduction_pct=0.827850 moved_to_echo=18911 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 37, 0, 3906, 0, 457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 369, 0, 0, 0, 3939, 0, 0, 0, 0, 0, 0, 0, 0, 3572, 182, 12, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 1, 0, 0, 2682, 441, 0, 0, 0, 0, 0, 0, 2, 18, 1727, 0, 0, 0, 0, 0, 3282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3966, 0, 304, 0, 0, 0, 0, 0, 0, 0, 0, 3092, 0, 5, 37, 19, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2099, 27, 0, 2, 0, 0, 58, 0, 990, 950, 212, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4401, 4308, 4116, 3136, 5029, 4271, 3169, 4338] avg=4096.0000 max=5029 max_rank=4 min=3136 min_rank=3 imbalance_ratio=1.227783 max_load_reduction=27739 max_load_reduction_pct=0.846527 moved_to_echo=16807 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 3908, 0, 445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 366, 0, 0, 3, 3177, 0, 0, 0, 0, 0, 0, 0, 0, 3579, 4, 278, 8, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 857, 0, 507, 0, 0, 2611, 437, 0, 0, 0, 0, 0, 0, 87, 2, 195, 0, 0, 0, 7, 0, 3255, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 10, 71, 15, 3865, 0, 304, 0, 0, 0, 0, 0, 0, 0, 0, 3081, 1, 1, 91, 54, 3298, 1, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 82, 0, 12, 1, 987, 925, 212, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4354, 3550, 3870, 4414, 3546, 4272, 6527, 2235] avg=4096.0000 max=6527 max_rank=6 min=2235 min_rank=7 imbalance_ratio=1.593506 max_load_reduction=26241 max_load_reduction_pct=0.800812 moved_to_echo=16580 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 44, 0, 3949, 0, 456, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 2, 368, 0, 4, 0, 3624, 0, 0, 0, 0, 0, 0, 0, 0, 3565, 276, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 5, 0, 997, 1037, 0, 0, 2680, 441, 0, 0, 0, 0, 0, 0, 17, 372, 552, 5, 0, 0, 55, 2, 3281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 15, 7, 143, 3122, 1, 303, 0, 0, 0, 0, 0, 0, 0, 22, 2965, 3, 586, 0, 716, 81, 7, 0, 0, 0, 0, 0, 0, 0, 0, 803, 1, 0, 1, 1, 0, 5, 5, 950, 950, 211, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4466, 4016, 3932, 5171, 4284, 3592, 4380, 2927] avg=4096.0000 max=5171 max_rank=3 min=2927 min_rank=7 imbalance_ratio=1.262451 max_load_reduction=27597 max_load_reduction_pct=0.842194 moved_to_echo=15183 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3167, 0, 438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 1, 368, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 3575, 173, 4, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 79, 305, 31, 0, 0, 2574, 441, 0, 0, 0, 0, 0, 0, 3, 0, 1184, 0, 6, 0, 0, 1, 3281, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 9, 0, 4082, 0, 304, 0, 0, 0, 0, 0, 0, 0, 0, 3060, 0, 2, 0, 118, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3029, 0, 3, 1, 50, 1, 13, 8, 980, 912, 212, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3605, 4468, 3768, 3431, 4475, 4398, 3414, 5209] avg=4096.0000 max=5209 max_rank=7 min=3414 min_rank=6 imbalance_ratio=1.271729 max_load_reduction=27559 max_load_reduction_pct=0.841034 moved_to_echo=17496 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 12, 0, 0, 3747, 0, 457, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 35, 369, 0, 8, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3528, 0, 62, 550, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 787, 32, 141, 0, 0, 2685, 440, 0, 0, 0, 0, 0, 0, 0, 31, 6, 3, 0, 6, 0, 0, 3278, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 3827, 0, 300, 0, 0, 0, 0, 0, 0, 0, 167, 2757, 10, 124, 0, 195, 62, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1888, 0, 0, 0, 1072, 0, 26, 7, 883, 952, 209, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4219, 4510, 4142, 4085, 3324, 4135, 3316, 5037] avg=4096.0000 max=5037 max_rank=7 min=3316 min_rank=6 imbalance_ratio=1.229736 max_load_reduction=27731 max_load_reduction_pct=0.846283 moved_to_echo=15812 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 11, 0, 0, 3834, 0, 446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 347, 0, 0, 2, 3814, 0, 0, 0, 0, 0, 0, 0, 0, 3447, 0, 0, 2023, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 2137, 0, 0, 2678, 429, 0, 0, 0, 0, 0, 0, 0, 2, 27, 0, 1, 0, 0, 0, 3166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 1, 82, 3938, 0, 299, 0, 0, 0, 0, 0, 0, 0, 0, 3063, 0, 100, 0, 0, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 257, 2, 0, 0, 246, 0, 19, 0, 984, 951, 209, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4292, 4167, 5471, 5246, 3196, 4325, 3403, 2668] avg=4096.0000 max=5471 max_rank=2 min=2668 min_rank=7 imbalance_ratio=1.335693 max_load_reduction=27297 max_load_reduction_pct=0.833038 moved_to_echo=13592 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 2688, 1408, 4096, 0, 128, 21376, 3584, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[13824, 128, 128, 128, 1664, 128, 384, 128, 7808, 7552, 1792, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 24064, 128, 1536, 128, 1536, 5504, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 512, 6016, 128, 128, 128, 128, 128, 26112, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 2944, 0, 128, 128, 29824, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[28288, 768, 640, 3456, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 384, 128, 28800, 0, 3712, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 128, 256, 29952, 128, 2432, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 44, 0, 0, 3882, 0, 0, 0, 4090, 176, 0, 0, 3004, 0, 1356, 0, 0, 3, 0, 1, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 0, 34, 182, 0, 3, 1, 0, 0, 0, 0, 4069, 3994, 3, 0, 0, 8, 0, 0, 4, 0, 1, 0, 4094, 0, 0, 0, 0, 3748, 0, 20, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[7, 0, 6, 0, 0, 2250, 0, 0, 0, 4095, 1884, 0, 0, 3440, 0, 178, 0, 0, 432, 9, 0, 0, 4, 0, 0, 0, 9, 0, 1, 0, 0, 4094, 0, 3, 111, 0, 0, 1, 0, 0, 20, 0, 4088, 3901, 2, 0, 0, 101, 5, 3, 7, 0, 10, 0, 4030, 0, 0, 0, 2, 4075, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4090, 0, 0, 1, 4094, 1559, 0, 0, 1379, 0, 186, 0, 0, 732, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1, 4050, 9, 0, 309, 0, 0, 0, 0, 0, 0, 1, 3943, 4086, 0, 3, 1, 0, 8, 101, 14, 0, 128, 3, 4095, 0, 3, 0, 0, 3957, 0, 0, 0, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4075, 0, 0, 0, 3786, 1081, 0, 0, 3568, 0, 1, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 13, 0, 4096, 0, 0, 236, 0, 0, 0, 0, 0, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 3537, 0, 0, 0, 0, 4096, 48, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3810, 0, 0, 0, 4073, 1741, 0, 0, 2410, 0, 230, 0, 1, 170, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4069, 0, 0, 0, 0, 0, 18, 0, 0, 5, 0, 3928, 0, 0, 0, 0, 4081, 25, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 12, 0, 0, 4093, 2, 3, 2, 4091, 2357, 0, 0, 1309, 0, 584, 0, 0, 2, 2, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4082, 0, 0, 5, 0, 0, 0, 0, 12, 10, 0, 4092, 4093, 0, 0, 0, 0, 0, 4, 0, 0, 2, 0, 3913, 0, 0, 1, 0, 4091, 3, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 27, 0, 0, 4093, 0, 0, 0, 4083, 301, 0, 0, 3259, 1, 479, 0, 0, 2, 2, 0, 0, 1, 6, 0, 0, 2, 0, 9, 0, 0, 4077, 0, 0, 329, 0, 0, 0, 0, 0, 1, 7, 4058, 4077, 0, 0, 1, 0, 0, 0, 0, 0, 13, 0, 3831, 0, 0, 6, 0, 4095, 0, 0, 8, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[19, 0, 0, 0, 0, 4090, 0, 0, 8, 4073, 456, 1, 0, 14, 0, 2971, 4, 0, 1177, 0, 11, 0, 0, 1, 0, 0, 3, 7, 3, 67, 0, 4065, 0, 14, 0, 0, 0, 2, 0, 0, 1, 0, 4054, 3534, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 4095, 0, 0, 0, 0, 4093, 3, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 44, 0, 0, 3882, 0, 0, 283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 0, 2814, 0, 1356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 8, 0, 0, 3665, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 6, 0, 0, 0, 0, 0, 0, 0, 0, 34, 182, 0, 3, 1, 0, 0, 3980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 3994, 3, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 4094, 0, 118, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3748, 0, 20, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4209, 4346, 3787, 4049, 4200, 4103, 4240, 3834] avg=4096.0000 max=4346 max_rank=1 min=3787 min_rank=2 imbalance_ratio=1.061035 max_load_reduction=28422 max_load_reduction_pct=0.867371 moved_to_echo=16377 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 6, 0, 0, 2250, 0, 0, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1884, 0, 0, 3223, 0, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 432, 9, 0, 0, 4, 0, 3682, 110, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 1, 0, 0, 4094, 7, 0, 0, 0, 0, 0, 0, 0, 0, 3, 111, 0, 0, 1, 0, 0, 3985, 0, 0, 0, 0, 0, 0, 0, 20, 0, 98, 3901, 2, 0, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3, 7, 0, 10, 0, 4030, 0, 135, 23, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4075, 0, 0, 0, 0, 75, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2548, 5285, 4237, 4111, 4100, 4122, 4213, 4152] avg=4096.0000 max=5285 max_rank=1 min=2548 min_rank=0 imbalance_ratio=1.290283 max_load_reduction=27483 max_load_reduction_pct=0.838715 moved_to_echo=16587 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4090, 0, 0, 274, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1559, 0, 0, 1292, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 732, 1, 0, 0, 0, 0, 3552, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1, 4050, 3, 0, 0, 0, 0, 0, 0, 0, 9, 0, 309, 0, 0, 0, 0, 0, 3983, 0, 0, 0, 0, 0, 0, 0, 0, 1, 95, 4086, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 101, 14, 0, 128, 3, 4095, 0, 54, 22, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3957, 0, 0, 0, 7, 30, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4364, 3038, 4396, 4061, 4301, 4186, 4425, 3997] avg=4096.0000 max=4425 max_rank=6 min=3038 min_rank=1 imbalance_ratio=1.080322 max_load_reduction=28343 max_load_reduction_pct=0.864960 moved_to_echo=16909 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4075, 0, 0, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1081, 0, 0, 3343, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 3689, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 13, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236, 0, 0, 0, 0, 0, 3684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 3537, 0, 140, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 48, 0, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4360, 4425, 3830, 4118, 3920, 4184, 3709, 4222] avg=4096.0000 max=4425 max_rank=1 min=3709 min_rank=6 imbalance_ratio=1.080322 max_load_reduction=28343 max_load_reduction_pct=0.864960 moved_to_echo=16035 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3810, 0, 0, 284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1741, 0, 0, 2258, 0, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 170, 22, 0, 0, 0, 0, 3680, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 3963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 5, 0, 3928, 0, 94, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 25, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4229, 3983, 4101, 3967, 4167, 4068, 4159] avg=4096.0000 max=4229 max_rank=1 min=3967 min_rank=4 imbalance_ratio=1.032471 max_load_reduction=28539 max_load_reduction_pct=0.870941 moved_to_echo=16361 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 12, 0, 0, 4093, 2, 3, 285, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2357, 0, 0, 1228, 0, 584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 1, 0, 0, 3686, 110, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4082, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 12, 3981, 0, 0, 0, 0, 0, 0, 0, 10, 0, 98, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 2, 0, 3913, 0, 51, 23, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4091, 3, 0, 1, 0, 28, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4395, 4171, 3801, 4085, 3998, 4201, 3993, 4124] avg=4096.0000 max=4395 max_rank=0 min=3801 min_rank=2 imbalance_ratio=1.072998 max_load_reduction=28373 max_load_reduction_pct=0.865875 moved_to_echo=16316 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[19, 0, 0, 0, 0, 4090, 0, 0, 282, 0, 0, 0, 0, 0, 0, 0, 8, 0, 456, 1, 0, 14, 0, 2971, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1177, 0, 11, 0, 0, 1, 3652, 109, 0, 0, 0, 0, 0, 0, 0, 0, 3, 7, 3, 67, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 2, 0, 0, 3964, 0, 0, 0, 0, 0, 0, 0, 1, 0, 97, 3534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 4095, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4391, 3450, 4954, 4145, 3980, 3632, 4120, 4096] avg=4096.0000 max=4954 max_rank=2 min=3450 min_rank=1 imbalance_ratio=1.209473 max_load_reduction=27814 max_load_reduction_pct=0.848816 moved_to_echo=15828 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 27, 0, 0, 4093, 0, 0, 287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 301, 0, 0, 3043, 1, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 1, 6, 3659, 110, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 9, 0, 0, 4077, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 329, 0, 0, 0, 0, 0, 3973, 0, 0, 0, 0, 0, 0, 0, 1, 7, 87, 4077, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 3831, 0, 131, 25, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4095, 0, 0, 8, 0, 75, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4407, 3824, 3780, 4098, 4302, 4173, 4000, 4184] avg=4096.0000 max=4407 max_rank=0 min=3780 min_rank=2 imbalance_ratio=1.075928 max_load_reduction=28361 max_load_reduction_pct=0.865509 moved_to_echo=16659 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 0, 0, 30464, 128, 128, 2304, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 9600, 128, 0, 17280, 128, 6016, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 2560, 128, 128, 128, 128, 128, 29312, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 1280, 0, 128, 128, 0, 128, 31616, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 896, 31872, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 0, 128, 128, 128, 128, 128, 32640, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 256, 128, 128, 256, 128, 31616, 0, 768, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 32256, 128, 128, 128, 128, 512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 124, 4089, 0, 0, 1, 11, 1, 0, 0, 4090, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 4, 0, 0, 0, 46, 0, 0, 4, 0, 0, 1, 0, 0, 0, 7, 4096, 4092, 0, 2, 18, 0, 0, 0, 0, 0, 4080, 0, 4062, 0, 1556, 0, 2394, 0, 0, 1, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 316, 5, 7, 34, 4093, 0, 0, 0, 369, 0, 0, 0, 4045, 0, 0, 0, 6, 0, 3963, 10, 0, 0, 0, 0, 0, 0, 1, 152, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4095, 8, 35, 598, 0, 2, 32, 0, 0, 4061, 0, 4047, 0, 226, 0, 2564, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 0, 1, 30, 4096, 0, 3, 0, 141, 0, 0, 0, 3985, 0, 0, 0, 0, 0, 4065, 2, 0, 0, 0, 5, 229, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 4095, 5, 0, 17, 2, 0, 0, 0, 0, 4095, 0, 4056, 0, 567, 0, 3247, 0, 0, 0, 0, 27] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 18, 0, 4, 0, 38, 4092, 0, 0, 0, 43, 0, 0, 0, 3995, 2, 2, 0, 0, 0, 4079, 15, 1, 0, 0, 0, 0, 0, 0, 69, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4085, 4094, 0, 2, 32, 0, 0, 0, 0, 0, 2906, 0, 4013, 0, 2740, 0, 2491, 0, 0, 12, 0, 31] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 5, 0, 29, 0, 8, 0, 344, 4089, 0, 3, 0, 551, 0, 0, 14, 1661, 0, 0, 0, 0, 0, 4084, 20, 6, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 4089, 0, 0, 1753, 0, 0, 0, 0, 0, 4092, 0, 3874, 0, 2988, 0, 886, 0, 0, 0, 0, 189] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 2, 0, 12, 0, 0, 1, 0, 4093, 0, 0, 0, 1, 0, 0, 0, 3162, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 1, 6, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 4087, 4095, 0, 0, 0, 0, 17, 0, 0, 0, 1868, 0, 4085, 0, 3815, 0, 3430, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[7, 0, 0, 0, 1, 15, 0, 0, 4086, 0, 0, 0, 960, 0, 0, 5, 3798, 0, 1, 0, 0, 0, 4093, 0, 12, 0, 3, 0, 0, 0, 0, 138, 0, 0, 56, 2, 0, 0, 0, 0, 0, 0, 3877, 4084, 0, 0, 45, 0, 0, 0, 0, 0, 2445, 1, 4082, 2, 3747, 0, 1223, 0, 0, 0, 0, 85] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 66, 2, 0, 239, 4063, 0, 0, 0, 2630, 0, 0, 0, 1249, 0, 0, 3, 0, 17, 4015, 3, 0, 0, 0, 0, 36, 0, 0, 22, 0, 1, 2, 1, 2, 0, 0, 0, 0, 2, 4085, 4093, 1, 0, 7, 6, 0, 0, 0, 1, 3362, 0, 4092, 2, 1852, 0, 2461, 2, 0, 0, 0, 451] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 124, 2553, 583, 540, 286, 0, 0, 0, 0, 3506, 0, 0, 1, 11, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 46, 3332, 689, 6, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3810, 0, 0, 2, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1556, 0, 1854, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 3519, 4934, 4077, 4091, 3837, 4810, 3413] avg=4096.0000 max=4934 max_rank=2 min=3413 min_rank=7 imbalance_ratio=1.204590 max_load_reduction=27834 max_load_reduction_pct=0.849426 moved_to_echo=16151 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 18, 0, 4, 0, 38, 2549, 583, 562, 285, 0, 0, 0, 0, 3509, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3995, 2, 2, 0, 0, 0, 843, 15, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 69, 3292, 687, 6, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3800, 0, 0, 2, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2906, 0, 721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2740, 0, 1929, 0, 0, 12, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4039, 3552, 4857, 4055, 4092, 3834, 3627, 4712] avg=4096.0000 max=4857 max_rank=2 min=3552 min_rank=1 imbalance_ratio=1.185791 max_load_reduction=27911 max_load_reduction_pct=0.851776 moved_to_echo=16265 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 0, 1, 30, 2540, 584, 732, 286, 0, 0, 0, 0, 3512, 0, 3, 0, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3985, 0, 0, 0, 0, 0, 840, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 229, 0, 0, 1, 3327, 685, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3810, 0, 5, 0, 17, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 567, 0, 2515, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4175, 3656, 4827, 4253, 4090, 3834, 4824, 3109] avg=4096.0000 max=4827 max_rank=2 min=3109 min_rank=7 imbalance_ratio=1.178467 max_load_reduction=27941 max_load_reduction_pct=0.852692 moved_to_echo=15857 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 316, 5, 7, 34, 2476, 584, 578, 286, 0, 0, 0, 0, 3509, 0, 0, 0, 369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 0, 0, 6, 0, 819, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 152, 3320, 668, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3810, 0, 8, 35, 598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 32, 0, 0, 4061, 0, 727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 226, 0, 1986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4288, 3878, 4880, 4147, 4089, 4452, 4822, 2212] avg=4096.0000 max=4880 max_rank=2 min=2212 min_rank=7 imbalance_ratio=1.191406 max_load_reduction=27888 max_load_reduction_pct=0.851074 moved_to_echo=15575 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 0, 29, 0, 8, 0, 344, 2552, 583, 199, 285, 0, 0, 0, 0, 3506, 0, 3, 0, 551, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 1661, 0, 0, 0, 0, 0, 844, 20, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4, 0, 0, 0, 3178, 688, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3794, 0, 0, 0, 1753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2988, 0, 687, 0, 0, 0, 0, 189, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4005, 4074, 2525, 3881, 4084, 5547, 4788, 3864] avg=4096.0000 max=5547 max_rank=5 min=2525 min_rank=2 imbalance_ratio=1.354248 max_load_reduction=27221 max_load_reduction_pct=0.830719 moved_to_echo=18283 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 12, 0, 0, 1, 0, 2556, 584, 774, 285, 0, 0, 0, 0, 3509, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3162, 0, 0, 0, 0, 0, 845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 0, 0, 3351, 689, 5, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 1868, 0, 734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3815, 0, 2656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4214, 3510, 4007, 4052, 4093, 3802, 2619, 6471] avg=4096.0000 max=6471 max_rank=7 min=2619 min_rank=6 imbalance_ratio=1.579834 max_load_reduction=26297 max_load_reduction_pct=0.802521 moved_to_echo=16985 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 66, 2, 0, 239, 2509, 579, 555, 285, 0, 0, 0, 0, 3484, 0, 0, 0, 2630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1249, 0, 0, 3, 0, 17, 829, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 22, 3357, 677, 6, 0, 0, 0, 0, 0, 0, 1, 2, 1, 2, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3800, 0, 1, 0, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3362, 0, 735, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1852, 0, 1906, 2, 0, 0, 0, 451, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4235, 6114, 2101, 4098, 4093, 3816, 4100, 4211] avg=4096.0000 max=6114 max_rank=1 min=2101 min_rank=2 imbalance_ratio=1.492676 max_load_reduction=26654 max_load_reduction_pct=0.813416 moved_to_echo=16220 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 0, 0, 1, 15, 0, 0, 2561, 586, 279, 276, 0, 0, 0, 0, 3500, 0, 0, 0, 960, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 3798, 0, 1, 0, 0, 0, 838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 3, 0, 0, 0, 0, 138, 3352, 694, 6, 0, 0, 0, 0, 0, 0, 0, 56, 2, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3601, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2445, 1, 730, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3747, 0, 944, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3725, 4465, 4637, 4205, 4136, 3646, 3178, 4776] avg=4096.0000 max=4776 max_rank=7 min=3178 min_rank=6 imbalance_ratio=1.166016 max_load_reduction=27992 max_load_reduction_pct=0.854248 moved_to_echo=15736 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 0, 128, 512, 128, 128, 896, 20352, 4736, 4224, 2304, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[17536, 0, 14592, 128, 0, 128, 0, 896, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 0, 128, 128, 384, 0, 128, 512, 26624, 5504, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[26112, 128, 128, 128, 128, 128, 6784, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[28160, 0, 128, 128, 4736, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 30336, 0, 128, 128, 2560, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 27008, 128, 5888, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 128, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4085, 3834, 0, 0, 0, 0, 0, 0, 0, 3902, 0, 43, 1, 0, 0, 0, 0, 0, 0, 0, 181, 3533, 3324, 4083, 0, 0, 0, 73, 0, 0, 9, 0, 0, 0, 10, 0, 4083, 7, 0, 0, 0, 0, 59, 4093, 0, 0, 13, 0, 1421, 0, 0, 0, 0, 0, 2, 10, 2, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4093, 3572, 0, 143, 2, 0, 1, 3, 2, 2162, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 77, 3903, 3709, 3965, 0, 0, 5, 4, 0, 0, 2180, 0, 0, 5, 0, 44, 4085, 0, 0, 2, 0, 0, 6, 3879, 0, 533, 0, 0, 369, 0, 0, 3, 0, 0, 2, 3, 0, 6, 0, 0, 0, 9] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4011, 0, 1, 0, 0, 0, 0, 0, 1036, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2746, 3261, 4095, 4078, 0, 0, 0, 0, 0, 1, 1272, 0, 0, 0, 1, 0, 4085, 0, 0, 0, 0, 0, 3, 4014, 0, 17, 0, 0, 34, 0, 0, 0, 0, 0, 0, 1, 0, 15, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4086, 3987, 11, 431, 0, 0, 0, 0, 0, 1108, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 400, 3861, 3983, 4079, 15, 0, 0, 61, 0, 0, 1607, 0, 0, 0, 0, 0, 4096, 44, 8, 3, 0, 2, 11, 3948, 0, 2, 0, 0, 998, 0, 1, 0, 0, 0, 1, 0, 0, 22, 0, 0, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 2167, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 471, 4080, 4095, 4084, 11, 0, 0, 22, 0, 0, 1032, 0, 0, 0, 17, 0, 4085, 1, 0, 0, 0, 0, 0, 4045, 0, 0, 0, 0, 458, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4092, 4076, 0, 1, 0, 0, 0, 0, 0, 1850, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 392, 4087, 4096, 4094, 2, 0, 0, 3, 0, 0, 60, 0, 0, 0, 0, 0, 3992, 0, 0, 0, 0, 0, 0, 4095, 0, 201, 4, 0, 1682, 0, 0, 0, 0, 0, 0, 0, 1, 7, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 3672, 0, 0, 0, 0, 0, 0, 0, 3891, 0, 2, 0, 0, 0, 183, 0, 19, 0, 0, 283, 4026, 3535, 4094, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 176, 1, 4006, 0, 0, 0, 0, 0, 0, 4096, 0, 32, 0, 0, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[5, 0, 4091, 3705, 1, 0, 0, 2, 0, 0, 0, 3973, 1, 3, 4, 1, 0, 0, 0, 4, 0, 0, 89, 3397, 4088, 4089, 0, 0, 7, 0, 5, 0, 0, 0, 0, 0, 17, 0, 4096, 0, 0, 4, 0, 0, 0, 4087, 0, 0, 5, 0, 996, 0, 0, 0, 0, 0, 97, 1, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 153, 4076, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1850, 0, 0, 0, 0, 766, 469, 306, 47, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 392, 3781, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 207, 2, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 3226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 201, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1682, 0, 0, 0, 0, 0, 3285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 0, 0, 0, 0, 3939, 133, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4230, 3438, 4206, 4308, 3286, 4249, 4971, 4080] avg=4096.0000 max=4971 max_rank=6 min=3286 min_rank=4 imbalance_ratio=1.213623 max_load_reduction=27797 max_load_reduction_pct=0.848297 moved_to_echo=16586 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 152, 3834, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3902, 0, 43, 1, 0, 784, 468, 265, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 3268, 0, 0, 0, 0, 0, 0, 0, 0, 3324, 206, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 10, 0, 3299, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 1421, 0, 0, 0, 0, 0, 3276, 0, 0, 0, 0, 0, 0, 0, 2, 10, 2, 0, 0, 0, 0, 0, 3933, 133, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3986, 5510, 3449, 3603, 3325, 4105, 4710, 4080] avg=4096.0000 max=5510 max_rank=1 min=3325 min_rank=4 imbalance_ratio=1.345215 max_load_reduction=27258 max_load_reduction_pct=0.831848 moved_to_echo=16220 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 153, 4011, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 0, 0, 0, 0, 784, 467, 244, 46, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2746, 3017, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 206, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1272, 0, 0, 0, 1, 0, 3301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3968, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 3272, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 15, 0, 0, 0, 0, 3943, 133, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4165, 2577, 5764, 4302, 4574, 3988, 3306, 4092] avg=4096.0000 max=5764 max_rank=2 min=2577 min_rank=1 imbalance_ratio=1.407227 max_load_reduction=27004 max_load_reduction_pct=0.824097 moved_to_echo=15960 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 153, 3572, 0, 143, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 2, 2162, 0, 0, 0, 0, 784, 454, 292, 44, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 77, 3611, 0, 0, 0, 0, 0, 0, 0, 0, 3709, 201, 0, 0, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2180, 0, 0, 5, 0, 44, 3301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 6, 3835, 0, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 369, 0, 0, 3, 0, 0, 3181, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 6, 0, 0, 0, 9, 3940, 129, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3870, 3742, 3689, 3919, 5530, 4376, 3553, 4089] avg=4096.0000 max=5530 max_rank=4 min=3553 min_rank=6 imbalance_ratio=1.350098 max_load_reduction=27238 max_load_reduction_pct=0.831238 moved_to_echo=17548 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 153, 3672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3891, 0, 2, 0, 0, 769, 469, 302, 47, 0, 0, 0, 0, 0, 183, 0, 19, 0, 0, 283, 3724, 0, 0, 0, 0, 0, 0, 0, 0, 3535, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 176, 1, 3237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 651, 0, 0, 0, 0, 0, 3285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3943, 133, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3825, 5480, 4209, 3742, 3419, 4081, 3936, 4076] avg=4096.0000 max=5480 max_rank=1 min=3419 min_rank=4 imbalance_ratio=1.337891 max_load_reduction=27288 max_load_reduction_pct=0.832764 moved_to_echo=15512 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 153, 3705, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3973, 1, 3, 4, 1, 786, 468, 254, 47, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 89, 3143, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 207, 0, 0, 7, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 3310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 996, 0, 0, 0, 0, 0, 3281, 0, 0, 0, 0, 0, 0, 0, 97, 1, 0, 0, 0, 0, 0, 0, 3938, 133, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3866, 5537, 3236, 4307, 3327, 4044, 4282, 4169] avg=4096.0000 max=5537 max_rank=1 min=3236 min_rank=2 imbalance_ratio=1.351807 max_load_reduction=27231 max_load_reduction_pct=0.831024 moved_to_echo=15822 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 153, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2167, 0, 0, 0, 0, 784, 468, 306, 46, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 471, 3774, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 206, 11, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 0, 0, 0, 17, 0, 3301, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 458, 0, 0, 0, 0, 0, 3277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 3943, 133, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4249, 3772, 4247, 4334, 4351, 3999, 3735, 4081] avg=4096.0000 max=4351 max_rank=4 min=3735 min_rank=6 imbalance_ratio=1.062256 max_load_reduction=28417 max_load_reduction_pct=0.867218 moved_to_echo=16166 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 148, 3987, 11, 431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1108, 0, 0, 0, 0, 791, 471, 294, 48, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 400, 3567, 0, 0, 0, 0, 0, 0, 0, 0, 3983, 194, 15, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1607, 0, 0, 0, 0, 0, 3305, 44, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3, 0, 2, 11, 3900, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 998, 0, 1, 0, 0, 0, 3276, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 22, 0, 0, 0, 2, 3938, 138, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4577, 2712, 3968, 4253, 4956, 3926, 4275, 4101] avg=4096.0000 max=4956 max_rank=4 min=2712 min_rank=1 imbalance_ratio=1.209961 max_load_reduction=27812 max_load_reduction_pct=0.848755 moved_to_echo=17258 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 20096, 128, 128, 128, 128, 6272, 3840, 2304, 384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 6656, 0, 128, 128, 0, 0, 26240, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[6272, 0, 0, 128, 256, 128, 26368, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 1280, 30976, 128, 640, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 256, 128, 128, 0, 0, 4736, 27904, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[30976, 1664, 128, 0, 128, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 128, 32000, 0, 896, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 0, 0, 0, 128, 31616, 1152, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33792 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[3902, 0, 2199, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 98, 0, 14, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3819, 0, 0, 4096, 0, 4088, 0, 73, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 161, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[3796, 57, 3293, 0, 165, 4096, 0, 0, 0, 0, 0, 0, 555, 4, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 18, 11, 0, 0, 0, 4096, 0, 7, 4095, 0, 1, 4095, 0, 4053, 0, 0, 0, 0, 0, 98, 0, 0, 4095, 0, 0, 140, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[4093, 0, 892, 0, 166, 4093, 0, 0, 0, 0, 0, 0, 2977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 36, 0, 0, 1, 1, 0, 0, 4093, 0, 0, 3951, 0, 1, 4096, 0, 4088, 0, 1, 0, 0, 0, 2, 0, 0, 4090, 0, 0, 184, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[3728, 13, 755, 0, 12, 4078, 0, 0, 0, 0, 0, 4, 2640, 0, 1, 0, 16, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 399, 0, 2366, 1, 21, 1, 0, 0, 1, 4072, 6, 16, 3392, 74, 0, 4087, 0, 1644, 0, 0, 0, 0, 5, 90, 0, 10, 4024, 16, 0, 1262, 0, 5, 12, 0, 0, 10, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[2929, 0, 936, 0, 335, 4095, 0, 0, 0, 0, 0, 0, 1855, 72, 0, 0, 0, 0, 0, 0, 0, 15, 263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4084, 0, 229, 3693, 0, 9, 4079, 0, 2840, 0, 0, 0, 0, 0, 4, 0, 0, 4095, 1, 0, 3229, 0, 0, 0, 3, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[3921, 0, 2442, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1843, 6, 7, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4094, 0, 6, 4019, 5, 0, 4074, 0, 3956, 0, 8, 0, 0, 0, 49, 1, 0, 4092, 0, 0, 87, 10, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[4009, 1, 116, 0, 2809, 4066, 0, 0, 0, 0, 0, 0, 3940, 194, 0, 0, 0, 2, 4, 0, 0, 5, 0, 9, 0, 2, 0, 0, 0, 104, 0, 2, 0, 0, 0, 0, 4088, 0, 0, 4096, 1, 1, 4089, 0, 753, 0, 205, 0, 4, 0, 66, 0, 0, 4092, 0, 0, 110, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[4060, 30, 168, 1, 83, 4073, 0, 0, 0, 0, 0, 0, 498, 0, 10, 26, 1, 14, 0, 0, 0, 0, 0, 6, 0, 0, 0, 570, 17, 4, 0, 122, 0, 0, 0, 1, 4085, 0, 0, 3706, 0, 0, 4087, 0, 3234, 3, 69, 2, 3, 0, 6, 0, 0, 4070, 1, 0, 3810, 0, 0, 7, 0, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2346, 0, 2199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2042, 0, 0, 0, 1556, 455, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 98, 0, 14, 0, 0, 3589, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 215, 0, 0, 3819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 961, 0, 4088, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 0, 0, 0, 0, 0, 0, 0, 2680, 292, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4545, 4081, 4058, 3744, 4034, 5122, 4051, 3133] avg=4096.0000 max=5122 max_rank=5 min=3133 min_rank=7 imbalance_ratio=1.250488 max_load_reduction=27646 max_load_reduction_pct=0.843689 moved_to_echo=16340 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2357, 0, 2442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1843, 6, 7, 0, 1564, 453, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3587, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 216, 0, 6, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 955, 0, 3956, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 1, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 10, 0, 0, 0, 0, 0, 0, 2666, 291, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4799, 3901, 4105, 3627, 4244, 4924, 4114, 3054] avg=4096.0000 max=4924 max_rank=5 min=3054 min_rank=7 imbalance_ratio=1.202148 max_load_reduction=27844 max_load_reduction_pct=0.849731 moved_to_echo=16336 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2282, 57, 3293, 0, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 555, 4, 86, 0, 1514, 455, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 18, 3589, 40, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 215, 0, 7, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 960, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 0, 0, 0, 0, 0, 2680, 292, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5797, 2642, 4056, 3654, 4328, 5014, 4165, 3112] avg=4096.0000 max=5797 max_rank=0 min=2642 min_rank=1 imbalance_ratio=1.415283 max_load_reduction=26971 max_load_reduction_pct=0.823090 moved_to_echo=16619 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2461, 0, 892, 0, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2977, 0, 0, 0, 1632, 455, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 36, 0, 0, 3586, 40, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 216, 0, 0, 3951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 961, 0, 4088, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 0, 0, 0, 0, 0, 0, 0, 2680, 291, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3519, 5092, 4053, 3665, 4169, 5051, 4064, 3155] avg=4096.0000 max=5092 max_rank=1 min=3155 min_rank=7 imbalance_ratio=1.243164 max_load_reduction=27676 max_load_reduction_pct=0.844604 moved_to_echo=16439 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1761, 0, 936, 0, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1855, 72, 0, 0, 1168, 454, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 263, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3578, 40, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 215, 0, 229, 3693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 956, 0, 2840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4067, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3229, 0, 0, 0, 3, 1, 0, 0, 2669, 291, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3032, 3577, 4333, 3618, 4138, 3805, 4072, 6193] avg=4096.0000 max=6193 max_rank=7 min=3032 min_rank=0 imbalance_ratio=1.511963 max_load_reduction=26575 max_load_reduction_pct=0.811005 moved_to_echo=18208 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2241, 13, 755, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2640, 0, 1, 0, 1487, 454, 27, 0, 0, 0, 0, 0, 16, 0, 1, 0, 6, 0, 0, 0, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 399, 0, 2366, 1, 21, 3568, 40, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 214, 6, 16, 3392, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 959, 0, 1644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 90, 0, 10, 3997, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1262, 0, 5, 12, 0, 0, 10, 0, 2674, 290, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3021, 4613, 4061, 6395, 3630, 2677, 4118, 4253] avg=4096.0000 max=6395 max_rank=3 min=2677 min_rank=5 imbalance_ratio=1.561279 max_load_reduction=26373 max_load_reduction_pct=0.804840 moved_to_echo=14678 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2410, 1, 116, 0, 2809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3940, 194, 0, 0, 1599, 455, 28, 0, 0, 0, 0, 0, 0, 2, 4, 0, 0, 5, 0, 9, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 104, 0, 2, 3582, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 215, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 958, 0, 753, 0, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 66, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 0, 0, 0, 0, 0, 0, 2676, 291, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5336, 6216, 4047, 3729, 4311, 1918, 4134, 3077] avg=4096.0000 max=6216 max_rank=1 min=1918 min_rank=5 imbalance_ratio=1.517578 max_load_reduction=26552 max_load_reduction_pct=0.810303 moved_to_echo=13440 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2437, 30, 168, 1, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 0, 10, 26, 1623, 459, 30, 0, 0, 0, 0, 0, 1, 14, 0, 0, 0, 0, 0, 6, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 570, 17, 4, 0, 122, 3584, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 207, 0, 0, 3706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 949, 0, 3234, 3, 69, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6, 0, 0, 4040, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3810, 0, 0, 7, 0, 0, 1, 0, 2679, 294, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2719, 2646, 4055, 4336, 3914, 4257, 4050, 6791] avg=4096.0000 max=6791 max_rank=7 min=2646 min_rank=1 imbalance_ratio=1.657959 max_load_reduction=25977 max_load_reduction_pct=0.792755 moved_to_echo=19012 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 16384, 384, 128, 128, 12160, 3712, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[18304, 128, 10880, 128, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 7680, 0, 24704, 128, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[9088, 128, 128, 128, 128, 128, 128, 0, 21504, 2432, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 128, 1152, 128, 2560, 128, 256, 28672, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 1792, 128, 384, 30848, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 384, 128, 128, 32512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 128, 128, 384, 128, 32384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33792 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 7, 1, 4055, 4051, 0, 4072, 143, 0, 0, 40, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 3794, 0, 0, 0, 8, 10, 0, 3197, 0, 4096, 4, 0, 0, 0, 18, 9, 0, 0, 0, 0, 0, 0, 0, 0, 172, 0, 0, 0, 0, 0, 0, 0, 3837, 0, 0, 1129, 0, 0, 21, 6, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4084, 3997, 0, 4048, 279, 0, 0, 5, 5, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 2902, 0, 0, 0, 71, 0, 0, 2696, 0, 4083, 2, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 86, 5, 218, 499, 0, 59, 1, 0, 65, 0, 0, 4040, 0, 0, 1517, 3, 1, 0, 10, 4085] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4092, 0, 4095, 331, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 3, 0, 0, 0, 1, 876, 0, 0, 0, 14, 1, 0, 3665, 0, 4096, 0, 0, 0, 0, 14, 0, 0, 0, 0, 1, 0, 0, 0, 25, 256, 0, 1, 0, 0, 1, 0, 0, 4076, 0, 0, 3018, 0, 0, 0, 2, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[2, 6, 0, 4059, 4096, 0, 4076, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 3954, 0, 0, 0, 0, 0, 0, 2911, 2, 3892, 0, 0, 110, 0, 88, 18, 11, 0, 0, 4, 3, 6, 0, 0, 486, 0, 1, 1, 0, 0, 4, 0, 4009, 0, 0, 881, 4, 1, 0, 65, 4073] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4093, 4095, 0, 3998, 271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1515, 0, 0, 0, 32, 0, 0, 2855, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 701, 0, 0, 0, 0, 4, 0, 2, 4096, 0, 0, 2908, 0, 0, 0, 5, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4061, 4086, 0, 4085, 34, 0, 0, 10, 0, 0, 0, 32, 0, 0, 0, 2, 14, 0, 0, 0, 1960, 0, 0, 0, 11, 0, 0, 4084, 0, 4055, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 458, 0, 224, 1, 1, 0, 0, 28, 4079, 0, 0, 1429, 0, 12, 2, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4086, 4096, 0, 4019, 271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 0, 0, 0, 0, 3, 0, 3039, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 75, 46, 0, 5, 0, 0, 1, 0, 0, 4091, 0, 0, 3829, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4092, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 3430, 0, 0, 0, 10, 0, 0, 3453, 0, 4092, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 44, 1065, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 193, 0, 0, 1, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3915, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 0, 0, 1, 0, 4084, 3, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 2902, 1163, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 2696, 0, 708, 128, 0, 0, 0, 0, 0, 0, 4080, 2, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 5, 218, 499, 3507, 0, 0, 0, 0, 0, 0, 0, 0, 59, 1, 0, 65, 0, 0, 4040, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1517, 3, 1, 0, 10, 2214, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4194, 4098, 4554, 3603, 4086, 4315, 4173, 3745] avg=4096.0000 max=4554 max_rank=2 min=3603 min_rank=3 imbalance_ratio=1.111816 max_load_reduction=28214 max_load_reduction_pct=0.861023 moved_to_echo=16319 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 1, 0, 0, 0, 3939, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 1, 0, 4055, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3794, 1166, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 10, 0, 3197, 0, 710, 128, 0, 0, 0, 0, 0, 0, 4093, 4, 0, 0, 0, 18, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 3556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3837, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1129, 0, 0, 21, 6, 2220, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4098, 5454, 4053, 4124, 3728, 3845, 3376] avg=4096.0000 max=5454 max_rank=2 min=3376 min_rank=7 imbalance_ratio=1.331543 max_load_reduction=27314 max_load_reduction_pct=0.833557 moved_to_echo=15073 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3961, 331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 876, 1166, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 1, 0, 3665, 0, 710, 129, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 25, 256, 3590, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 4076, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3018, 0, 0, 0, 2, 2220, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4292, 4108, 2544, 4519, 4107, 3872, 4086, 5240] avg=4096.0000 max=5240 max_rank=7 min=2544 min_rank=2 imbalance_ratio=1.279297 max_load_reduction=27528 max_load_reduction_pct=0.840088 moved_to_echo=17305 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3887, 271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1166, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3039, 0, 710, 127, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 75, 46, 3593, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 1, 0, 0, 4091, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3829, 0, 0, 0, 0, 2220, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4158, 4088, 2679, 3879, 4093, 3717, 4105, 6049] avg=4096.0000 max=6049 max_rank=7 min=2679 min_rank=2 imbalance_ratio=1.476807 max_load_reduction=26719 max_load_reduction_pct=0.815399 moved_to_echo=17964 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3867, 271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1515, 1166, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 2855, 0, 710, 126, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 701, 3592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2, 4096, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2908, 0, 0, 0, 5, 2220, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4138, 4096, 3181, 3723, 4094, 4293, 4110, 5133] avg=4096.0000 max=5133 max_rank=7 min=3181 min_rank=2 imbalance_ratio=1.253174 max_load_reduction=27635 max_load_reduction_pct=0.843353 moved_to_echo=17630 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2, 6, 0, 0, 0, 0, 3943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 4059, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3954, 1160, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2911, 2, 706, 128, 0, 0, 0, 0, 0, 0, 3890, 0, 0, 110, 0, 88, 18, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 6, 0, 0, 486, 3593, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 4, 0, 4009, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 881, 4, 1, 0, 65, 2207, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3951, 4065, 5616, 3747, 4117, 4092, 4022, 3158] avg=4096.0000 max=5616 max_rank=2 min=3158 min_rank=7 imbalance_ratio=1.371094 max_load_reduction=27152 max_load_reduction_pct=0.828613 moved_to_echo=15389 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3962, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3430, 1166, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 3453, 0, 710, 129, 0, 0, 0, 0, 0, 0, 4089, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 44, 1065, 3592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 193, 0, 0, 1, 0, 2220, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3968, 4109, 5094, 4302, 4092, 4708, 4081, 2414] avg=4096.0000 max=5094 max_rank=2 min=2414 min_rank=7 imbalance_ratio=1.243652 max_load_reduction=27674 max_load_reduction_pct=0.844543 moved_to_echo=15295 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3943, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 32, 0, 4061, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2, 14, 0, 0, 0, 1960, 1172, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 4084, 0, 714, 133, 0, 0, 0, 0, 0, 0, 4051, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 458, 3585, 0, 0, 0, 0, 0, 0, 0, 0, 224, 1, 1, 0, 0, 28, 4079, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1429, 0, 12, 2, 0, 2210, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3977, 4106, 3646, 4942, 4055, 4043, 4346, 3653] avg=4096.0000 max=4942 max_rank=3 min=3646 min_rank=2 imbalance_ratio=1.206543 max_load_reduction=27826 max_load_reduction_pct=0.849182 moved_to_echo=16097 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 0, 128, 128, 19456, 9344, 4096, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 14976, 128, 128, 128, 128, 17792, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 0, 0, 256, 128, 0, 25984, 128, 5760, 1152, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 128, 128, 128, 128, 384, 3712, 28672, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32512, 128, 0, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[0, 384, 128, 128, 128, 128, 128, 32384, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 128, 128, 128, 0, 128, 0, 32640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 0, 0, 0, 31488, 1408, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4091, 0, 0, 36, 0, 0, 28, 0, 0, 4046, 0, 0, 0, 4089, 0, 0, 4088, 0, 0, 0, 1097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 20, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2985, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 3755, 0, 0, 0, 4096, 0, 0, 4081, 0, 0, 0, 1076, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 3409, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4091, 0, 23, 81, 3, 2, 0, 26, 1, 4096, 0, 1, 0, 3878, 0, 0, 3937, 0, 0, 0, 13, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 730, 0, 0, 0, 4090, 0, 11, 1, 10, 0, 0, 0, 0, 4092, 5, 13, 0, 0, 3585, 0, 0, 0, 0, 0, 2, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 11, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4041, 0, 0, 0, 2038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 17, 4095, 0, 0, 0, 2, 0, 0, 0, 0, 4090, 4, 0, 0, 0, 2088, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4062, 1, 0, 301, 21, 0, 0, 0, 0, 4066, 1, 0, 0, 3882, 0, 0, 4026, 2, 73, 0, 145, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 6, 10, 8, 1, 4044, 0, 0, 130, 0, 0, 2, 0, 0, 4018, 0, 29, 11, 0, 3797, 0, 0, 0, 0, 0, 23, 13, 5, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 2, 0, 0, 34, 0, 1, 4096, 0, 0, 0, 3972, 0, 0, 3822, 0, 0, 0, 309, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 9, 0, 0, 10] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4081, 0, 0, 0, 0, 1, 16, 10, 0, 4096, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 6, 4056, 0, 0, 0, 2, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4072, 0, 0, 3, 0, 0, 0, 8, 0, 4096, 0, 0, 0, 4093, 0, 0, 4088, 0, 7, 0, 38, 0, 0, 0, 0, 0, 0, 2, 0, 0, 17, 4095, 0, 2, 0, 1, 4, 4035, 0, 0, 0, 28, 1, 0, 0, 5, 4066, 0, 0, 0, 1, 4087, 9, 0, 8, 0, 1, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3755, 0, 0, 0, 35, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3541, 0, 0, 0, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 3430, 540, 97, 13, 0, 0, 0, 0, 0, 0, 3999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 571, 0, 0, 0, 0, 3409, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 61, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4044, 3791, 4617, 4084, 3999, 4095, 3981, 4157] avg=4096.0000 max=4617 max_rank=2 min=3791 min_rank=1 imbalance_ratio=1.127197 max_load_reduction=28151 max_load_reduction_pct=0.859100 moved_to_echo=16232 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4043, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 4046, 0, 0, 0, 35, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3547, 0, 0, 0, 1097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3459, 541, 97, 13, 0, 0, 0, 0, 0, 0, 3999, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 575, 0, 0, 0, 0, 2985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 62, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4110, 4644, 4110, 4019, 4095, 3560, 4151] avg=4096.0000 max=4644 max_rank=2 min=3560 min_rank=6 imbalance_ratio=1.133789 max_load_reduction=28124 max_load_reduction_pct=0.858276 moved_to_echo=15825 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4048, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 35, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3506, 0, 0, 0, 2038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3454, 535, 97, 13, 0, 0, 0, 0, 0, 0, 3997, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 574, 4, 0, 0, 0, 2088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 62, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4059, 4132, 5544, 4099, 4014, 4096, 2666, 4158] avg=4096.0000 max=5544 max_rank=2 min=2666 min_rank=6 imbalance_ratio=1.353516 max_load_reduction=27224 max_load_reduction_pct=0.830811 moved_to_echo=14934 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4043, 0, 23, 81, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 26, 1, 4096, 0, 1, 0, 35, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3416, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3455, 521, 97, 13, 0, 0, 0, 0, 0, 0, 3979, 0, 730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 11, 1, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 575, 5, 13, 0, 0, 3585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3878, 62, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4150, 4162, 3429, 4087, 4709, 4111, 4178, 3942] avg=4096.0000 max=4709 max_rank=4 min=3429 min_rank=2 imbalance_ratio=1.149658 max_load_reduction=28059 max_load_reduction_pct=0.856293 moved_to_echo=16940 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4046, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 1, 4096, 0, 0, 0, 35, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3316, 0, 0, 0, 309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 3457, 506, 97, 13, 0, 0, 0, 0, 0, 0, 3989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 575, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 10, 3972, 62, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4048, 4167, 3625, 4143, 3989, 4078, 4665, 4053] avg=4096.0000 max=4665 max_rank=6 min=3625 min_rank=2 imbalance_ratio=1.138916 max_load_reduction=28103 max_load_reduction_pct=0.857635 moved_to_echo=16785 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4014, 1, 0, 301, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 1, 0, 0, 35, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3493, 2, 73, 0, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3393, 533, 97, 13, 0, 0, 0, 0, 0, 0, 3991, 0, 6, 10, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 0, 0, 130, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 564, 0, 29, 11, 0, 3797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 13, 5, 0, 3882, 61, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4337, 4103, 3713, 4039, 4016, 4175, 4401, 3984] avg=4096.0000 max=4401 max_rank=6 min=3713 min_rank=2 imbalance_ratio=1.074463 max_load_reduction=28367 max_load_reduction_pct=0.865692 moved_to_echo=16576 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 16, 10, 0, 4096, 0, 0, 0, 35, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3554, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3450, 542, 97, 13, 0, 0, 0, 0, 0, 0, 3998, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 573, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 62, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4033, 4159, 3607, 4102, 4004, 4057, 4649, 4157] avg=4096.0000 max=4649 max_rank=6 min=3607 min_rank=2 imbalance_ratio=1.135010 max_load_reduction=28119 max_load_reduction_pct=0.858124 moved_to_echo=16867 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4015, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 0, 0, 0, 37, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3544, 0, 7, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3437, 544, 101, 20, 0, 0, 0, 0, 0, 17, 3994, 0, 2, 0, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 0, 0, 0, 28, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 566, 0, 0, 0, 1, 4087, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 1, 0, 0, 0, 1, 4093, 63, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4018, 4144, 3589, 4104, 4018, 4061, 4668, 4166] avg=4096.0000 max=4668 max_rank=6 min=3589 min_rank=2 imbalance_ratio=1.139648 max_load_reduction=28100 max_load_reduction_pct=0.857544 moved_to_echo=16913 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 0, 0, 128, 0, 27648, 4352, 896, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 0, 28032, 128, 128, 0, 4864, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 4608, 128, 128, 128, 128, 28160, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 32384, 128, 128, 0, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 0, 32384, 128, 128, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 32000, 0, 768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[32640, 0, 128, 256, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 0, 128, 128, 128, 128, 128, 32256, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 3952, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1, 0, 0, 0, 3501, 0, 0, 0, 0, 0, 0, 0, 732, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 12, 0, 0, 0, 24, 0, 0, 0, 4081, 0, 0, 4096, 0, 0, 4083, 590, 0, 0, 4095, 5, 0, 0, 4095, 0, 0, 2961, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 24, 15, 0, 0, 0, 3483, 0, 0, 0, 0, 0, 0, 45, 1061, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 1, 4096, 0, 1, 4095, 1, 0, 0, 4096, 868, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 873, 0, 7, 0, 0, 0, 0, 0, 2364, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 8, 0, 0, 4078, 0, 0, 4094, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 4086, 0, 0, 4087, 501, 0, 39, 4083, 0, 4, 0, 4079, 0, 0, 3610, 0, 0, 3, 0, 4093, 0, 0, 20, 0, 0, 0, 0, 1, 4, 0, 0, 0, 0, 0, 0, 3832, 0, 2, 0, 0, 0, 0, 24, 192, 0, 0, 0, 0, 0, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3852, 0, 0, 4096, 0, 0, 4096, 62, 0, 0, 4096, 3, 0, 0, 4093, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 434, 0, 0, 0, 0, 0, 0, 1854, 1, 0, 0, 0, 0, 0, 0, 1990, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[7, 1, 0, 0, 0, 8, 0, 0, 52, 0, 4076, 0, 0, 4086, 0, 0, 4049, 208, 5, 0, 4055, 360, 0, 0, 4061, 34, 0, 4086, 8, 0, 0, 0, 4088, 6, 0, 0, 10, 0, 0, 0, 0, 1, 3, 0, 0, 0, 1, 0, 173, 0, 217, 0, 8, 0, 0, 0, 3155, 0, 0, 3, 2, 0, 5, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5, 0, 22, 2, 0, 2, 4020, 0, 0, 4077, 8, 0, 4069, 13, 0, 0, 4096, 184, 0, 0, 4093, 0, 0, 1784, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 4, 1, 0, 0, 0, 2234, 0, 0, 0, 0, 0, 0, 20, 3967, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3751, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 4021, 4, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 2020, 35, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 1552, 345, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4136, 3759, 4600, 4092, 4041, 2062, 5986] avg=4096.0000 max=5986 max_rank=7 min=2062 min_rank=6 imbalance_ratio=1.461426 max_load_reduction=26782 max_load_reduction_pct=0.817322 moved_to_echo=16181 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3751, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 0, 0, 3952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1, 0, 0, 0, 4021, 4, 0, 0, 0, 0, 0, 0, 3501, 0, 0, 0, 0, 0, 0, 0, 2029, 35, 0, 0, 0, 0, 0, 0, 732, 0, 0, 0, 0, 0, 0, 0, 1559, 345, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4135, 3751, 4460, 4095, 4033, 5565, 2637] avg=4096.0000 max=5565 max_rank=6 min=2637 min_rank=7 imbalance_ratio=1.358643 max_load_reduction=27203 max_load_reduction_pct=0.830170 moved_to_echo=16330 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3743, 501, 0, 39, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 0, 0, 3610, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 0, 0, 0, 0, 4011, 4, 0, 0, 0, 0, 0, 0, 3832, 0, 2, 0, 0, 0, 0, 24, 2021, 35, 0, 0, 0, 0, 0, 0, 192, 0, 0, 0, 0, 0, 2, 0, 1552, 344, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4143, 4287, 4119, 4112, 4020, 5914, 2091] avg=4096.0000 max=5914 max_rank=6 min=2091 min_rank=7 imbalance_ratio=1.443848 max_load_reduction=26854 max_load_reduction_pct=0.819519 moved_to_echo=16137 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 12, 0, 0, 0, 24, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3739, 590, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 0, 0, 2961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 24, 15, 0, 0, 0, 4021, 5, 0, 0, 0, 0, 0, 0, 3483, 0, 0, 0, 0, 0, 0, 45, 2029, 35, 0, 0, 0, 0, 0, 0, 1061, 0, 0, 0, 0, 0, 0, 0, 1558, 344, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4121, 4334, 3469, 4095, 4067, 5592, 2964] avg=4096.0000 max=5592 max_rank=6 min=2964 min_rank=7 imbalance_ratio=1.365234 max_load_reduction=27176 max_load_reduction_pct=0.829346 moved_to_echo=16718 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 1, 40, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3750, 1, 0, 0, 0, 868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4021, 4, 0, 0, 0, 0, 0, 0, 873, 0, 7, 0, 0, 0, 0, 0, 2029, 35, 0, 0, 0, 0, 0, 0, 2364, 0, 0, 0, 0, 0, 0, 0, 1559, 345, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4112, 4619, 4604, 4094, 4030, 2944, 4269] avg=4096.0000 max=4619 max_rank=2 min=2944 min_rank=6 imbalance_ratio=1.127686 max_load_reduction=28149 max_load_reduction_pct=0.859039 moved_to_echo=15337 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3852, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3751, 62, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 434, 0, 0, 0, 0, 0, 0, 4021, 4, 0, 0, 0, 0, 0, 0, 1854, 1, 0, 0, 0, 0, 0, 0, 2028, 35, 0, 0, 0, 0, 0, 0, 1990, 0, 0, 0, 0, 0, 0, 0, 1558, 345, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 3892, 3816, 4602, 4095, 4459, 3918, 3894] avg=4096.0000 max=4602 max_rank=3 min=3816 min_rank=2 imbalance_ratio=1.123535 max_load_reduction=28166 max_load_reduction_pct=0.859558 moved_to_echo=16366 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[7, 1, 0, 0, 0, 8, 0, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 52, 0, 4076, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3708, 208, 5, 0, 0, 360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 34, 0, 4086, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 6, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 1, 0, 4011, 4, 0, 0, 0, 0, 0, 0, 173, 0, 217, 0, 8, 0, 0, 0, 2012, 35, 0, 0, 0, 0, 0, 0, 3155, 0, 0, 3, 2, 0, 5, 0, 1545, 341, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4168, 4281, 4632, 4104, 4020, 2445, 5051] avg=4096.0000 max=5051 max_rank=7 min=2445 min_rank=6 imbalance_ratio=1.233154 max_load_reduction=27717 max_load_reduction_pct=0.845856 moved_to_echo=15620 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5, 0, 22, 2, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4020, 0, 0, 31, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3724, 13, 0, 0, 0, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 0, 0, 1784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 4, 1, 0, 0, 0, 4005, 4, 0, 0, 0, 0, 0, 0, 2234, 0, 0, 0, 0, 0, 0, 20, 2033, 41, 0, 0, 0, 0, 0, 0, 3967, 0, 0, 0, 0, 0, 0, 0, 1562, 345, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4061, 3921, 2282, 4081, 4098, 4328, 5876] avg=4096.0000 max=5876 max_rank=7 min=2282 min_rank=3 imbalance_ratio=1.434570 max_load_reduction=26892 max_load_reduction_pct=0.820679 moved_to_echo=18383 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[17664, 0, 0, 128, 128, 0, 128, 0, 12544, 2816, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[16000, 128, 256, 0, 128, 0, 0, 128, 16256, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 128, 0, 28800, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[29952, 1408, 128, 128, 0, 1536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 640, 128, 128, 128, 0, 128, 0, 32256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 32512, 0, 128, 384, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 19, 0, 4, 39, 4076, 0, 0, 0, 0, 0, 0, 1, 606, 4, 49, 0, 4025, 4083, 0, 3850, 0, 945, 0, 0, 0, 3982, 0, 7, 0, 0, 4, 0, 0, 0, 3389, 0, 12, 5, 5, 0, 0, 7, 0, 0, 0, 185, 6, 4, 0, 4083, 0, 3, 0, 19, 25, 9, 1, 0, 3321] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 30, 0, 0, 99, 4096, 17, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 4096, 4095, 0, 4085, 10, 1019, 0, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 3416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3578] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 34, 9, 2, 8, 322, 0, 0, 557, 4087, 444, 0, 0, 0, 6, 0, 0, 326, 0, 0, 0, 4075, 4076, 0, 4038, 98, 1367, 0, 0, 0, 3967, 0, 0, 0, 6, 0, 0, 0, 0, 4080, 1, 0, 0, 0, 118, 3, 0, 0, 0, 0, 166, 0, 0, 0, 4086, 0, 0, 0, 0, 10, 0, 0, 0, 882] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 0, 7, 0, 0, 9, 4083, 20, 0, 0, 0, 0, 0, 0, 3267, 0, 0, 0, 4096, 4096, 0, 4094, 22, 1382, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 3340, 0, 0, 0, 0, 116, 0, 0, 0, 2, 0, 3, 0, 0, 0, 4096, 0, 0, 3, 0, 14, 0, 0, 0, 38] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 53, 0, 0, 384, 4096, 0, 0, 0, 0, 0, 0, 0, 848, 0, 124, 0, 4096, 4096, 0, 4039, 12, 3695, 0, 0, 0, 4083, 2, 0, 0, 0, 0, 0, 0, 0, 3071, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 23, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1323, 0, 0, 505, 4096, 1, 0, 0, 0, 0, 0, 0, 790, 0, 0, 0, 4096, 4089, 0, 4088, 0, 109, 0, 0, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 3204, 0, 0, 0, 0, 871, 0, 0, 0, 161, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1285] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 0, 11, 0, 1267, 0, 0, 523, 4095, 0, 0, 0, 1, 0, 0, 0, 583, 0, 7, 0, 4096, 4088, 0, 3992, 125, 698, 0, 0, 0, 4021, 0, 0, 0, 0, 0, 0, 0, 0, 784, 0, 0, 0, 0, 2651, 0, 0, 0, 0, 0, 68, 0, 0, 0, 4078, 0, 0, 0, 0, 2, 0, 0, 0, 1677] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 31, 0, 0, 8, 82, 0, 0, 124, 4078, 21, 0, 0, 0, 0, 0, 0, 310, 3, 0, 10, 4070, 4082, 4, 4090, 243, 1468, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 3077, 0, 0, 0, 0, 534, 0, 0, 0, 1, 0, 19, 0, 0, 0, 4095, 0, 0, 4, 0, 0, 1, 0, 0, 2319] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 0, 7, 0, 0, 3701, 0, 0, 0, 0, 0, 0, 0, 9, 3748, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3267, 0, 0, 0, 3233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2737, 22, 1382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3340, 640, 335, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 0, 0, 0, 3182, 395, 0, 0, 0, 0, 0, 0, 2, 0, 3, 0, 0, 0, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 14, 0, 0, 0, 38, 1357, 894, 223, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3712, 3777, 6500, 4141, 4389, 3693, 4027, 2529] avg=4096.0000 max=6500 max_rank=2 min=2529 min_rank=7 imbalance_ratio=1.586914 max_load_reduction=26268 max_load_reduction_pct=0.801636 moved_to_echo=14638 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 30, 0, 0, 3701, 0, 0, 0, 0, 0, 0, 0, 99, 3760, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 3233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2731, 10, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3416, 640, 336, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3169, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3578, 1354, 890, 223, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3731, 3876, 3305, 3760, 4466, 3563, 4022, 6045] avg=4096.0000 max=6045 max_rank=7 min=3305 min_rank=2 imbalance_ratio=1.475830 max_load_reduction=26723 max_load_reduction_pct=0.815521 moved_to_echo=18096 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 19, 0, 4, 3689, 0, 0, 0, 0, 0, 0, 0, 39, 3741, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 606, 4, 49, 0, 3177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2574, 0, 945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4, 0, 0, 0, 3389, 629, 335, 74, 0, 0, 0, 0, 0, 0, 12, 5, 5, 0, 0, 7, 0, 3108, 394, 0, 0, 0, 0, 0, 0, 0, 0, 185, 6, 4, 0, 4009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 19, 25, 9, 1, 0, 3321, 1276, 874, 219, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3712, 3780, 3837, 3519, 4438, 3531, 4204, 5747] avg=4096.0000 max=5747 max_rank=7 min=3519 min_rank=3 imbalance_ratio=1.403076 max_load_reduction=27021 max_load_reduction_pct=0.824615 moved_to_echo=17920 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 34, 9, 2, 8, 322, 0, 0, 3684, 0, 0, 0, 0, 0, 0, 0, 557, 3751, 444, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 326, 0, 0, 0, 3216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2699, 98, 1367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4080, 637, 336, 74, 0, 0, 0, 0, 0, 1, 0, 0, 0, 118, 3, 0, 0, 3098, 392, 0, 0, 0, 0, 0, 0, 0, 0, 166, 0, 0, 0, 4012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 882, 1339, 869, 222, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4059, 4758, 3542, 4164, 5133, 3612, 4178, 3322] avg=4096.0000 max=5133 max_rank=4 min=3322 min_rank=7 imbalance_ratio=1.253174 max_load_reduction=27635 max_load_reduction_pct=0.843353 moved_to_echo=16245 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1323, 0, 0, 3694, 0, 0, 0, 0, 0, 0, 0, 505, 3760, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 790, 0, 0, 0, 3233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2733, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3204, 640, 336, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 871, 0, 0, 0, 3165, 395, 0, 0, 0, 0, 0, 0, 161, 0, 0, 0, 0, 0, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1285, 1355, 889, 223, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5017, 4266, 4023, 2842, 4254, 4431, 4183, 3752] avg=4096.0000 max=5017 max_rank=0 min=2842 min_rank=3 imbalance_ratio=1.224854 max_load_reduction=27751 max_load_reduction_pct=0.846893 moved_to_echo=16620 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 11, 0, 1267, 0, 0, 3694, 0, 0, 0, 0, 0, 0, 0, 523, 3759, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 583, 0, 7, 0, 3233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2669, 125, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 784, 640, 336, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2651, 0, 0, 0, 3139, 394, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 4004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1677, 1323, 882, 223, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4973, 4283, 3823, 3492, 1834, 6184, 4072, 4107] avg=4096.0000 max=6184 max_rank=5 min=1834 min_rank=4 imbalance_ratio=1.509766 max_load_reduction=26584 max_load_reduction_pct=0.811279 moved_to_echo=16197 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 53, 0, 0, 3701, 0, 0, 0, 0, 0, 0, 0, 384, 3760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 848, 0, 124, 0, 3233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2700, 12, 3695, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3071, 640, 336, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 3187, 395, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1339, 896, 223, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3754, 4144, 4205, 6409, 4121, 3630, 4045, 2460] avg=4096.0000 max=6409 max_rank=3 min=2460 min_rank=7 imbalance_ratio=1.564697 max_load_reduction=26359 max_load_reduction_pct=0.804413 moved_to_echo=14256 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 31, 0, 0, 8, 82, 0, 0, 3689, 0, 0, 0, 0, 0, 0, 0, 124, 3739, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 310, 3, 0, 10, 3206, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 2730, 243, 1468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3077, 639, 339, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 0, 0, 0, 3197, 393, 0, 0, 0, 0, 0, 0, 1, 0, 19, 0, 0, 0, 4017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1, 0, 0, 2319, 1360, 897, 225, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3810, 3884, 3533, 4441, 4133, 4124, 4037, 4806] avg=4096.0000 max=4806 max_rank=7 min=3533 min_rank=2 imbalance_ratio=1.173340 max_load_reduction=27962 max_load_reduction_pct=0.853333 moved_to_echo=17100 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 128, 0, 13184, 10752, 7168, 1792, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[21632, 512, 10752, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 6912, 128, 256, 128, 25856, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 0, 0, 0, 24448, 5120, 2816, 640, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 4352, 128, 128, 0, 25344, 3200, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 128, 3200, 0, 128, 29568, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2304, 30080, 512, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 0, 512, 128, 128, 0, 32256, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 4080, 0, 4091, 4040, 0, 0, 4096, 0, 2650, 0, 98, 0, 0, 42, 0, 4053, 265, 4087, 0, 0, 0, 1, 5, 0, 0, 0, 1182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4, 0, 2, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 4096, 4048, 0, 0, 4096, 0, 4025, 0, 3, 0, 0, 0, 0, 4096, 82, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 4096, 4095, 0, 0, 4096, 0, 3301, 0, 4, 0, 0, 1, 0, 4095, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 795, 0, 4, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[8, 0, 0, 0, 4088, 0, 4095, 4015, 0, 0, 4096, 0, 2037, 0, 1, 0, 0, 0, 0, 4090, 6, 3868, 0, 0, 0, 0, 1, 0, 0, 0, 2032, 0, 248, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 78, 6, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 4096, 0, 3421, 0, 19, 0, 0, 0, 0, 4096, 264, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 407, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 4078, 8, 4088, 4013, 0, 0, 4080, 2, 3319, 9, 0, 0, 0, 0, 13, 4068, 375, 4056, 0, 0, 0, 0, 335, 0, 0, 7, 0, 22, 119, 5, 0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4077, 11, 23, 8, 7, 16, 0, 8, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 9] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[62, 0, 0, 0, 4096, 5, 4095, 4015, 0, 0, 4088, 0, 3029, 0, 6, 0, 0, 1, 0, 4094, 163, 4017, 0, 0, 0, 2, 718, 0, 0, 0, 212, 31, 10, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4090, 0, 4096, 3972, 0, 0, 4096, 0, 1499, 0, 78, 0, 0, 0, 0, 4096, 1840, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 589, 0, 18, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 50, 0, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1160, 0, 4025, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 3755, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2936, 327, 44, 0, 0, 0, 0, 0, 11, 3, 0, 0, 0, 0, 0, 0, 4002, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 4085, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 5188, 3837, 3307, 4027, 4102, 4113, 4096] avg=4096.0000 max=5188 max_rank=1 min=3307 min_rank=3 imbalance_ratio=1.266602 max_load_reduction=27580 max_load_reduction_pct=0.841675 moved_to_echo=16338 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 49, 0, 0, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1160, 0, 2650, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 265, 3754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 0, 0, 1182, 0, 2936, 327, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3987, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7, 4041, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 3908, 4061, 4495, 3999, 4068, 4050, 4096] avg=4096.0000 max=4495 max_rank=3 min=3908 min_rank=1 imbalance_ratio=1.097412 max_load_reduction=28273 max_load_reduction_pct=0.862823 moved_to_echo=16213 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 50, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1160, 0, 3301, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 795, 0, 2936, 327, 44, 0, 0, 0, 0, 0, 4, 6, 0, 0, 0, 0, 0, 0, 4002, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4083, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4145, 4465, 3746, 4102, 4024, 4106, 4085, 4095] avg=4096.0000 max=4465 max_rank=1 min=3746 min_rank=2 imbalance_ratio=1.090088 max_load_reduction=28303 max_load_reduction_pct=0.863739 moved_to_echo=16310 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 0, 0, 49, 0, 0, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1160, 0, 2037, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2032, 0, 2936, 310, 44, 0, 0, 0, 0, 0, 248, 10, 0, 0, 0, 0, 0, 0, 3995, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 6, 4079, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 3198, 3558, 5323, 4264, 4095, 4164, 4094] avg=4096.0000 max=5323 max_rank=3 min=3198 min_rank=1 imbalance_ratio=1.299561 max_load_reduction=27445 max_load_reduction_pct=0.837555 moved_to_echo=16617 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 50, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1160, 0, 3421, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 264, 3744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 407, 0, 2936, 327, 44, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4002, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4085, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4146, 4600, 4008, 3714, 4020, 4095, 4090, 4095] avg=4096.0000 max=4600 max_rank=1 min=3714 min_rank=3 imbalance_ratio=1.123047 max_load_reduction=28168 max_load_reduction_pct=0.859619 moved_to_echo=16300 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 49, 8, 0, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1155, 2, 3319, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 375, 3725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 335, 0, 0, 7, 0, 22, 2925, 325, 44, 0, 0, 0, 0, 0, 119, 5, 0, 4, 0, 0, 0, 4, 3985, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 11, 23, 6, 0, 0, 0, 0, 0, 0, 0, 8, 7, 16, 0, 8, 0, 0, 0, 4057, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 9, 4087, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4485, 4113, 3658, 4128, 4117, 4097, 4098] avg=4096.0000 max=4485 max_rank=1 min=3658 min_rank=3 imbalance_ratio=1.094971 max_load_reduction=28283 max_load_reduction_pct=0.863129 moved_to_echo=16440 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 49, 0, 0, 3972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1160, 0, 1499, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1840, 3760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 589, 0, 2936, 328, 44, 0, 0, 0, 0, 0, 18, 204, 0, 0, 0, 0, 0, 0, 3997, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4021, 2737, 5600, 3897, 4231, 4102, 4085, 4095] avg=4096.0000 max=5600 max_rank=2 min=2737 min_rank=1 imbalance_ratio=1.367188 max_load_reduction=27168 max_load_reduction_pct=0.829102 moved_to_echo=16513 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[62, 0, 0, 0, 41, 5, 0, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1152, 0, 3029, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 163, 3681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 718, 0, 0, 0, 212, 31, 2936, 325, 48, 0, 0, 0, 0, 0, 10, 46, 0, 0, 0, 0, 0, 0, 4007, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 1, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 2, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4123, 4187, 3845, 4272, 4075, 4083, 4084, 4099] avg=4096.0000 max=4272 max_rank=3 min=3845 min_rank=2 imbalance_ratio=1.042969 max_load_reduction=28496 max_load_reduction_pct=0.869629 moved_to_echo=16341 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 9344, 128, 23296, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 1152, 0, 0, 128, 5248, 128, 23552, 2688, 384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 0, 3072, 29824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 512, 128, 0, 32384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 128, 128, 0, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[512, 384, 0, 128, 0, 0, 0, 128, 32000, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 128, 128, 128, 128, 32640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 999, 0, 4096, 0, 0, 0, 0, 0, 4071, 0, 0, 5, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4092, 0, 2852, 1187, 0, 0, 0, 0, 0, 0, 0, 0, 938, 2996, 0, 0, 0, 3409, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[12, 0, 0, 3988, 0, 0, 0, 0, 0, 334, 0, 0, 0, 0, 0, 0, 0, 2460, 0, 4096, 0, 0, 0, 0, 0, 2600, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 352, 3954, 0, 1, 0, 0, 0, 0, 0, 0, 1400, 2034, 0, 2, 0, 3339, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[7, 0, 0, 3917, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 14, 0, 4096, 0, 0, 0, 0, 0, 3950, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4096, 0, 1183, 3493, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3763, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1, 0, 0, 1593, 0, 0, 0, 0, 0, 499, 0, 0, 0, 1, 0, 0, 0, 1500, 0, 4089, 0, 0, 0, 0, 1, 3738, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 6, 0, 26, 4096, 0, 3024, 1746, 0, 0, 0, 0, 14, 0, 0, 0, 339, 4005, 0, 0, 0, 3994, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 66, 0, 3196, 0, 18, 0, 0, 10, 0, 0, 1, 0, 0, 0, 1, 0, 2001, 13, 4002, 0, 0, 64, 0, 8, 4011, 0, 5, 0, 0, 0, 0, 2, 0, 3, 0, 1, 5, 4056, 0, 13, 0, 0, 0, 0, 3999, 24, 317, 1367, 0, 0, 0, 0, 0, 0, 4, 0, 2751, 2743, 0, 0, 0, 4087, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 3052, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1396, 0, 4096, 0, 0, 0, 0, 0, 3535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4, 0, 4094, 0, 1791, 2826, 0, 0, 0, 0, 0, 0, 0, 0, 1244, 2750, 0, 0, 0, 3884, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4012, 0, 0, 0, 0, 0, 165, 0, 0, 0, 0, 0, 0, 0, 1472, 0, 4096, 0, 0, 0, 0, 0, 3640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 29, 2696, 0, 0, 0, 0, 0, 0, 0, 0, 445, 3971, 0, 0, 0, 4050, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[19, 9, 0, 3074, 0, 0, 16, 0, 0, 104, 0, 0, 0, 0, 0, 0, 0, 1108, 0, 4095, 0, 0, 0, 0, 0, 3648, 0, 0, 0, 0, 7, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 45, 0, 4081, 0, 42, 3457, 0, 0, 0, 0, 0, 0, 0, 2, 3473, 1676, 1, 0, 0, 3813, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 0, 3917, 0, 0, 0, 0, 722, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 4096, 116, 0, 0, 0, 0, 0, 0, 0, 14, 0, 2727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3950, 0, 0, 0, 0, 0, 0, 226, 271, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 4090, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2895, 0, 1183, 0, 0, 0, 0, 0, 0, 0, 0, 3493, 0, 0, 0, 0, 3, 0, 0, 1253, 253, 0, 0, 0, 0, 0, 0, 0, 0, 3487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4646, 4264, 2741, 4447, 4103, 4078, 5002, 3487] avg=4096.0000 max=5002 max_rank=6 min=2741 min_rank=2 imbalance_ratio=1.221191 max_load_reduction=27766 max_load_reduction_pct=0.847351 moved_to_echo=16670 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 0, 3988, 0, 0, 0, 0, 722, 0, 0, 0, 0, 0, 0, 0, 0, 334, 0, 0, 0, 0, 0, 0, 3339, 116, 0, 0, 0, 0, 0, 0, 0, 2460, 0, 2727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2600, 0, 0, 0, 0, 8, 0, 226, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2895, 0, 352, 0, 0, 0, 0, 0, 0, 0, 0, 3954, 0, 1, 0, 0, 0, 0, 0, 1253, 253, 0, 0, 0, 0, 0, 0, 0, 1400, 1885, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4722, 3789, 5187, 2980, 4095, 3247, 5461, 3287] avg=4096.0000 max=5461 max_rank=6 min=2980 min_rank=3 imbalance_ratio=1.333252 max_load_reduction=27307 max_load_reduction_pct=0.833344 moved_to_echo=16090 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 1593, 0, 0, 0, 0, 722, 0, 0, 0, 0, 0, 0, 0, 0, 499, 0, 0, 0, 1, 0, 0, 3994, 116, 0, 0, 0, 0, 0, 0, 0, 1500, 0, 2722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3738, 0, 0, 0, 6, 0, 0, 226, 288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 26, 2895, 0, 3024, 0, 0, 0, 0, 0, 0, 0, 0, 1746, 0, 0, 0, 0, 14, 0, 0, 1251, 253, 0, 0, 0, 0, 0, 0, 0, 339, 3711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2316, 4610, 4222, 4259, 4096, 5951, 3264, 4050] avg=4096.0000 max=5951 max_rank=5 min=2316 min_rank=0 imbalance_ratio=1.452881 max_load_reduction=26817 max_load_reduction_pct=0.818390 moved_to_echo=17361 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4012, 0, 0, 0, 0, 722, 0, 0, 0, 0, 0, 0, 0, 0, 165, 0, 0, 0, 0, 0, 0, 4050, 116, 0, 0, 0, 0, 0, 0, 0, 1472, 0, 2727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3640, 0, 0, 0, 0, 0, 0, 226, 286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2895, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 2696, 0, 0, 0, 0, 0, 0, 0, 1253, 253, 0, 0, 0, 0, 0, 0, 0, 445, 3679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4734, 4331, 4199, 4152, 4102, 2924, 4202, 4124] avg=4096.0000 max=4734 max_rank=0 min=2924 min_rank=5 imbalance_ratio=1.155762 max_load_reduction=28034 max_load_reduction_pct=0.855530 moved_to_echo=15352 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3052, 0, 0, 0, 0, 722, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3884, 116, 0, 0, 0, 0, 0, 0, 0, 1396, 0, 2727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3535, 0, 0, 0, 0, 0, 0, 226, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2893, 0, 1791, 0, 0, 0, 0, 0, 0, 0, 0, 2826, 0, 0, 0, 0, 0, 0, 0, 1253, 253, 0, 0, 0, 0, 0, 0, 0, 1244, 2548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3774, 4001, 4123, 3959, 4099, 4688, 4332, 3792] avg=4096.0000 max=4688 max_rank=5 min=3774 min_rank=0 imbalance_ratio=1.144531 max_load_reduction=28080 max_load_reduction_pct=0.856934 moved_to_echo=16911 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4029, 0, 0, 0, 0, 721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3409, 116, 0, 0, 0, 0, 0, 0, 0, 999, 0, 2727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 5, 0, 0, 0, 226, 216, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4092, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2892, 0, 2852, 0, 0, 0, 0, 0, 0, 0, 0, 1187, 0, 0, 0, 0, 0, 0, 0, 1253, 253, 0, 0, 0, 0, 0, 0, 0, 938, 2776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4750, 3525, 3726, 4518, 4098, 5744, 2693, 3714] avg=4096.0000 max=5744 max_rank=5 min=2693 min_rank=6 imbalance_ratio=1.402344 max_load_reduction=27024 max_load_reduction_pct=0.824707 moved_to_echo=16249 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 66, 0, 3196, 0, 18, 0, 0, 705, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 1, 0, 0, 0, 1, 4087, 113, 0, 0, 0, 0, 0, 0, 0, 2001, 13, 2665, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4011, 0, 5, 0, 0, 0, 0, 221, 197, 0, 0, 0, 0, 0, 0, 2, 0, 3, 0, 1, 5, 4056, 0, 4, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 2826, 24, 317, 0, 0, 0, 0, 0, 0, 0, 0, 1367, 0, 0, 0, 0, 0, 0, 4, 1224, 247, 0, 0, 0, 0, 0, 0, 0, 2751, 2542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3985, 4212, 4743, 4442, 4071, 3180, 2842, 5293] avg=4096.0000 max=5293 max_rank=7 min=2842 min_rank=6 imbalance_ratio=1.292236 max_load_reduction=27475 max_load_reduction_pct=0.838470 moved_to_echo=15386 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[19, 9, 0, 3074, 0, 0, 16, 0, 723, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 3813, 119, 0, 0, 0, 0, 0, 0, 0, 1108, 0, 2719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3648, 0, 0, 0, 0, 7, 0, 231, 125, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 2869, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 3457, 0, 0, 0, 0, 0, 0, 0, 1257, 258, 0, 0, 0, 0, 0, 0, 2, 3473, 1545, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3841, 4036, 3827, 4011, 4104, 2956, 4972, 5021] avg=4096.0000 max=5021 max_rank=7 min=2956 min_rank=5 imbalance_ratio=1.225830 max_load_reduction=27747 max_load_reduction_pct=0.846771 moved_to_echo=17053 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[20736, 0, 128, 0, 0, 128, 0, 128, 10112, 2048, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 11008, 128, 21760, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 128, 23168, 128, 9600, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 10624, 22272, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 26880, 0, 128, 128, 0, 5760, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 29312, 0, 128, 128, 128, 128, 0, 1920, 1792, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 1280, 0, 128, 0, 128, 0, 128, 30720, 1024, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 128, 128, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4072, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 93, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 0, 4091, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4079, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 64, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 4096, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 3974, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4067, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 2, 0, 0, 0, 4096, 0, 0, 0, 4085, 0, 0, 4096, 1, 0, 0, 0, 0, 27, 0, 249, 0, 0, 0, 0, 0, 0, 0, 3827, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4040, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4092, 4, 0, 416, 0, 0, 0, 4086, 0, 0, 0, 4096, 0, 0, 4096, 3, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3738, 0, 0, 4086, 0, 6, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 12, 68, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 4089, 1, 0, 28, 4, 0, 0, 5, 0, 0, 0, 0, 0, 1, 0, 0, 4060, 0, 0, 4011, 1, 0, 0, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3849, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 29, 0, 0, 0, 4096, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 54, 0, 0, 0, 0, 219, 0, 0, 0, 0, 0, 0, 7, 4051, 0, 0, 4081, 0, 0, 0, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3877, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 218, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 4095, 0, 0, 4096, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[4, 3, 0, 1, 0, 16, 4086, 3990, 0, 0, 0, 0, 228, 12, 7, 0, 0, 0, 0, 4, 0, 7, 0, 30, 2, 6, 4052, 0, 0, 0, 15, 8, 14, 4061, 0, 35, 0, 4082, 10, 2, 4082, 2, 0, 0, 0, 0, 41, 0, 0, 0, 0, 1, 0, 0, 0, 11, 3926, 0, 0, 4030, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4072, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 49, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3982, 0, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3909, 111, 20, 2, 0, 0, 0, 0, 4034, 0, 0, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4145, 4076, 4092, 4075, 4094, 4077, 4042, 4167] avg=4096.0000 max=4167 max_rank=7 min=4042 min_rank=6 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16380 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4067, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4023, 49, 4, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3985, 0, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 1, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 249, 0, 0, 0, 0, 0, 0, 0, 3914, 111, 20, 2, 0, 0, 0, 0, 3827, 0, 0, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4140, 4078, 4102, 3994, 4094, 4104, 4296, 3960] avg=4096.0000 max=4296 max_rank=6 min=3960 min_rank=7 imbalance_ratio=1.048828 max_load_reduction=28472 max_load_reduction_pct=0.868896 moved_to_echo=16454 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4079, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 49, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3985, 0, 1, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 3914, 111, 20, 2, 0, 0, 0, 0, 3974, 0, 0, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4152, 4076, 4091, 4050, 4094, 4094, 4104, 4107] avg=4096.0000 max=4152 max_rank=0 min=4050 min_rank=3 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16399 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4040, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4023, 49, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3981, 4, 0, 416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 3, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3904, 111, 20, 1, 0, 0, 0, 0, 3738, 0, 0, 133, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4080, 4092, 4401, 4085, 4084, 4036, 3877] avg=4096.0000 max=4401 max_rank=3 min=3877 min_rank=7 imbalance_ratio=1.074463 max_load_reduction=28367 max_load_reduction_pct=0.865692 moved_to_echo=16082 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3849, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 49, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3985, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 219, 0, 0, 0, 0, 0, 0, 7, 3900, 111, 20, 2, 0, 0, 0, 0, 4051, 0, 0, 132, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3922, 4077, 4086, 4014, 4094, 4130, 4259, 4186] avg=4096.0000 max=4259 max_rank=6 min=3922 min_rank=0 imbalance_ratio=1.039795 max_load_reduction=28509 max_load_reduction_pct=0.870026 moved_to_echo=16669 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3877, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 49, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3985, 0, 0, 218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 3914, 111, 20, 2, 0, 0, 0, 0, 4095, 0, 0, 133, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3950, 4076, 4092, 4203, 4094, 4074, 4050, 4229] avg=4096.0000 max=4229 max_rank=7 min=3950 min_rank=0 imbalance_ratio=1.032471 max_load_reduction=28539 max_load_reduction_pct=0.870941 moved_to_echo=16447 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4, 3, 0, 1, 0, 16, 4086, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 228, 12, 7, 0, 3919, 49, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 7, 0, 30, 4078, 0, 0, 0, 0, 0, 0, 0, 2, 6, 3942, 0, 0, 0, 15, 8, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4060, 0, 35, 0, 0, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 2, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 11, 3851, 110, 20, 1, 0, 0, 0, 0, 3926, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4181, 4219, 4119, 3973, 4121, 4105, 3994, 4056] avg=4096.0000 max=4219 max_rank=1 min=3973 min_rank=3 imbalance_ratio=1.030029 max_load_reduction=28549 max_load_reduction_pct=0.871246 moved_to_echo=16276 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 54, 5, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3978, 0, 12, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 1, 0, 28, 4, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3837, 118, 27, 4, 0, 0, 0, 0, 4060, 0, 0, 120, 1, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4165, 4086, 4094, 4058, 4092, 4100, 3987, 4186] avg=4096.0000 max=4186 max_rank=7 min=3987 min_rank=6 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16365 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[31744, 0, 0, 1152, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 0, 128, 32256, 640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[512, 0, 128, 128, 0, 128, 0, 128, 31232, 896, 256, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 0, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32640, 128, 0, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 256, 128, 128, 0, 32128, 512, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 31872, 128, 128, 896, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32768, 0, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 3157, 4096, 0, 0, 0, 3916, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 854, 4096, 0, 0, 0, 0, 4095, 0, 0, 15, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 675, 0, 0, 0, 0, 0, 0, 3667, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 925, 4096, 0, 1, 0, 3998, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3342, 4096, 0, 0, 0, 1, 4075, 0, 0, 19, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 902, 0, 0, 0, 0, 0, 0, 3120, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 425, 4096, 0, 25, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3670, 4095, 0, 0, 0, 0, 4087, 0, 0, 18, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 0, 0, 0, 513, 4094, 0, 0, 0, 4010, 3, 0, 0, 0, 0, 4096, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2489, 4091, 3, 0, 0, 10, 4092, 0, 0, 0, 0, 0, 261, 0, 4085, 0, 0, 0, 0, 0, 0, 4, 0, 1115, 0, 0, 0, 0, 0, 0, 3875, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 67, 4096, 0, 5, 0, 3463, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3879, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 12, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 825, 0, 0, 0, 0, 0, 0, 4038, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 224, 4096, 0, 1, 0, 3925, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3623, 4096, 0, 0, 0, 0, 4091, 0, 0, 1, 0, 0, 249, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 174, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 5, 12, 22, 15, 0, 0, 7, 256, 4068, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 4085, 0, 9, 0, 0, 0, 0, 0, 0, 0, 10, 1, 1834, 4080, 0, 0, 9, 0, 4086, 0, 0, 1, 0, 11, 0, 0, 4079, 0, 0, 0, 13, 4, 0, 0, 0, 1997, 0, 8, 0, 0, 6, 0, 4079, 2, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 5, 0, 0, 627, 4094, 0, 0, 0, 3635, 0, 0, 12, 0, 0, 4093, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2821, 4086, 0, 0, 0, 0, 4058, 0, 0, 32, 0, 0, 105, 1, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 1417, 0, 0, 0, 0, 0, 0, 3700, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 3157, 0, 0, 0, 0, 3343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 854, 884, 399, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 5, 0, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 675, 0, 3203, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3667, 0, 0, 174, 85, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 6500, 4093, 2137, 4104, 4031, 3883, 3929] avg=4096.0000 max=6500 max_rank=1 min=2137 min_rank=3 imbalance_ratio=1.586914 max_load_reduction=26268 max_load_reduction_pct=0.801636 moved_to_echo=15947 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 925, 0, 0, 1, 0, 3412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3342, 884, 408, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 1, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 902, 0, 3203, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3120, 0, 0, 178, 85, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4338, 4093, 4634, 4085, 4030, 4110, 3386] avg=4096.0000 max=4634 max_rank=3 min=3386 min_rank=7 imbalance_ratio=1.131348 max_load_reduction=28134 max_load_reduction_pct=0.858582 moved_to_echo=15611 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 513, 0, 0, 0, 0, 3423, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2489, 883, 409, 0, 0, 0, 0, 0, 0, 9, 3, 0, 0, 10, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 261, 0, 4000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1115, 0, 3199, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3875, 0, 0, 178, 85, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 3939, 4117, 3781, 4114, 4261, 4323, 4141] avg=4096.0000 max=4323 max_rank=6 min=3781 min_rank=3 imbalance_ratio=1.055420 max_load_reduction=28445 max_load_reduction_pct=0.868073 moved_to_echo=16839 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 425, 0, 0, 25, 0, 3493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3670, 884, 417, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3202, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 182, 85, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 3943, 4093, 4971, 4096, 4029, 3207, 4338] avg=4096.0000 max=4971 max_rank=3 min=3207 min_rank=6 imbalance_ratio=1.213623 max_load_reduction=27797 max_load_reduction_pct=0.848297 moved_to_echo=15670 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 5, 0, 2956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3879, 884, 353, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 825, 0, 3203, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 0, 0, 154, 85, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 3028, 4093, 5116, 4105, 4022, 4033, 4280] avg=4096.0000 max=5116 max_rank=3 min=3028 min_rank=1 imbalance_ratio=1.249023 max_load_reduction=27652 max_load_reduction_pct=0.843872 moved_to_echo=16440 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 224, 0, 0, 1, 0, 3350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3623, 884, 400, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 249, 0, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174, 0, 3203, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 175, 85, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 3575, 4093, 4907, 4100, 4261, 3381, 4359] avg=4096.0000 max=4907 max_rank=3 min=3381 min_rank=6 imbalance_ratio=1.197998 max_load_reduction=27861 max_load_reduction_pct=0.850250 moved_to_echo=16101 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 12, 22, 15, 0, 0, 7, 4063, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0, 3468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1, 1834, 880, 415, 0, 0, 0, 0, 0, 0, 9, 0, 0, 9, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 11, 0, 0, 3994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4, 0, 0, 0, 1997, 0, 3191, 5, 0, 0, 0, 0, 0, 0, 8, 0, 0, 6, 0, 4079, 2, 5, 181, 85, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4124, 3724, 4091, 3140, 4104, 4006, 5210, 4369] avg=4096.0000 max=5210 max_rank=6 min=3140 min_rank=3 imbalance_ratio=1.271973 max_load_reduction=27558 max_load_reduction_pct=0.841003 moved_to_echo=17689 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 5, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 627, 0, 0, 0, 0, 3094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2821, 884, 376, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 105, 1, 3990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1417, 0, 3200, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3700, 0, 0, 165, 91, 10, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 3721, 4095, 4082, 4060, 4128, 4621, 3966] avg=4096.0000 max=4621 max_rank=6 min=3721 min_rank=1 imbalance_ratio=1.128174 max_load_reduction=28147 max_load_reduction_pct=0.858978 moved_to_echo=16775 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[6272, 0, 0, 128, 0, 26624, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 128, 128, 22528, 7168, 3200, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 128, 0, 7168, 0, 25728, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 0, 0, 128, 0, 30720, 128, 128, 1408, 768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 128, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 640, 128, 32128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2930, 0, 0, 7, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1093, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 735, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 2, 3, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 3315, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 64, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 353, 0, 0, 2580, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4052, 0, 0, 0, 1141, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 143, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 42, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 3911, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 4076, 7, 0, 0, 0, 1, 13, 4082, 0, 9, 196, 0, 0, 0, 1, 0, 0, 0, 4069, 0, 7, 1, 0, 3, 0, 8, 0, 0, 0, 0, 0, 10, 0, 0, 0, 14, 0, 26, 0, 0, 0, 380, 10, 8, 4078, 0, 0, 19, 9, 0, 4044, 69, 0, 20, 0, 0, 3869, 0, 6, 0, 3651, 4082, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[11, 4096, 0, 0, 0, 2, 0, 0, 4077, 0, 0, 149, 0, 0, 43, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 3902, 4085, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1166, 0, 0, 2, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 4, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 2948, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 7, 0, 0, 4096, 0, 0, 861, 0, 0, 0, 0, 0, 0, 1, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 1026, 79, 0, 4089, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 4088, 28, 0, 0, 2170, 4086, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3311, 0, 0, 735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 8, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 2751, 785, 520, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 3574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1338, 0, 0, 0, 3315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4046, 4108, 4099, 4101, 3579, 4088, 4653] avg=4096.0000 max=4653 max_rank=7 min=3579 min_rank=5 imbalance_ratio=1.135986 max_load_reduction=28115 max_load_reduction_pct=0.858002 moved_to_echo=16421 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3311, 0, 0, 2930, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 8, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2754, 785, 520, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 3574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1339, 0, 0, 0, 1093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 6248, 4099, 4099, 4059, 3649, 4088, 2432] avg=4096.0000 max=6248 max_rank=1 min=2432 min_rank=7 imbalance_ratio=1.525391 max_load_reduction=26520 max_load_reduction_pct=0.809326 moved_to_echo=14228 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3311, 0, 0, 64, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 8, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 353, 2724, 785, 520, 0, 0, 0, 0, 0, 0, 0, 2580, 0, 0, 3574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1326, 0, 0, 0, 1141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 3376, 4108, 4098, 4382, 6154, 4088, 2467] avg=4096.0000 max=6154 max_rank=5 min=2467 min_rank=7 imbalance_ratio=1.502441 max_load_reduction=26614 max_load_reduction_pct=0.812195 moved_to_echo=17091 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3311, 0, 0, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 8, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 2736, 785, 520, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 3574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1330, 0, 0, 0, 3911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 3454, 4108, 4099, 4068, 3616, 4088, 5241] avg=4096.0000 max=5241 max_rank=7 min=3454 min_rank=1 imbalance_ratio=1.279541 max_load_reduction=27527 max_load_reduction_pct=0.840057 moved_to_echo=17013 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3311, 0, 0, 1166, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 8, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2747, 785, 520, 0, 0, 0, 0, 0, 0, 0, 2, 4, 0, 3574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1336, 0, 0, 0, 2948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4479, 4090, 4099, 4054, 3580, 4088, 4284] avg=4096.0000 max=4479 max_rank=1 min=3580 min_rank=5 imbalance_ratio=1.093506 max_load_reduction=28289 max_load_reduction_pct=0.863312 moved_to_echo=16006 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3311, 0, 0, 861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4030, 0, 0, 0, 0, 8, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 2749, 785, 520, 0, 0, 0, 0, 0, 0, 0, 1026, 79, 0, 3567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1336, 28, 0, 0, 2170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4172, 4043, 4089, 4068, 4672, 4089, 3534] avg=4096.0000 max=4672 max_rank=5 min=3534 min_rank=7 imbalance_ratio=1.140625 max_load_reduction=28096 max_load_reduction_pct=0.857422 moved_to_echo=16363 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4074, 7, 0, 0, 0, 1, 13, 0, 0, 0, 0, 0, 0, 0, 0, 3300, 0, 9, 196, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 7, 1, 0, 8, 2, 2, 0, 0, 0, 0, 0, 3, 0, 8, 0, 0, 0, 0, 0, 4082, 2, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 14, 0, 26, 0, 2601, 782, 518, 0, 0, 0, 0, 0, 0, 0, 380, 10, 8, 3558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 9, 0, 4036, 69, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1266, 0, 6, 0, 3651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 3506, 4089, 4095, 3951, 3956, 4153, 4923] avg=4096.0000 max=4923 max_rank=7 min=3506 min_rank=1 imbalance_ratio=1.201904 max_load_reduction=27845 max_load_reduction_pct=0.849762 moved_to_echo=16983 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[11, 4088, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3295, 0, 0, 149, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 10, 9, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 2757, 782, 526, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1332, 0, 0, 0, 3902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 3487, 4123, 4090, 4085, 3562, 4086, 5234] avg=4096.0000 max=5234 max_rank=7 min=3487 min_rank=1 imbalance_ratio=1.277832 max_load_reduction=27534 max_load_reduction_pct=0.840271 moved_to_echo=16967 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 128, 0, 128, 512, 21888, 6400, 4224, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 10624, 128, 128, 0, 22144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[26496, 0, 128, 6272, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 4096, 256, 128, 28672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 32768, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 0, 128, 32768, 0, 128, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 0, 0, 0, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True + [2026-09-21 08:25:37] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 253.7 | throughput per GPU (TFLOP/s/GPU): 127.0 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 9.125510E+00 | loss scale: 1.0 | grad norm: 15.415 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[286, 268, 715, 2, 3, 809, 726, 157, 0, 1315, 26, 5, 18, 18, 182, 156, 23, 1207, 925, 1969, 23, 258, 241, 15, 225, 9, 74, 275, 1342, 587, 816, 1508, 545, 183, 462, 621, 192, 1497, 217, 524, 1623, 350, 1591, 99, 1021, 825, 9, 138, 920, 460, 608, 64, 739, 1629, 1956, 73, 1470, 25, 184, 4, 19, 21, 486, 30] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[126, 257, 692, 0, 0, 518, 1089, 408, 3, 1032, 19, 0, 6, 19, 406, 77, 15, 878, 1098, 2227, 31, 50, 226, 1, 78, 16, 46, 206, 1083, 1003, 1206, 1647, 893, 392, 332, 745, 201, 1215, 200, 965, 943, 684, 1825, 23, 1020, 680, 11, 35, 726, 969, 589, 120, 648, 902, 1468, 13, 1140, 10, 348, 7, 7, 33, 1151, 10] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[239, 168, 791, 2, 1, 445, 1066, 279, 1, 922, 24, 0, 11, 11, 142, 157, 1, 923, 1203, 1967, 28, 201, 270, 4, 64, 11, 119, 253, 967, 1007, 1036, 1616, 529, 315, 587, 896, 101, 1371, 292, 583, 1387, 661, 2190, 34, 870, 799, 15, 163, 864, 436, 739, 4, 571, 1520, 1758, 28, 1046, 19, 299, 9, 4, 13, 719, 17] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[165, 150, 1061, 2, 0, 699, 1134, 459, 0, 938, 14, 0, 48, 12, 364, 59, 26, 896, 563, 2219, 41, 284, 284, 1, 44, 8, 52, 238, 904, 881, 792, 2116, 532, 298, 402, 1112, 136, 1493, 206, 768, 1615, 487, 1888, 90, 828, 653, 21, 72, 1007, 372, 612, 55, 493, 1585, 1483, 40, 957, 38, 611, 10, 6, 25, 412, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[95, 187, 657, 4, 0, 754, 1211, 184, 0, 1396, 63, 0, 10, 7, 442, 107, 9, 1078, 766, 2368, 29, 54, 295, 1, 46, 23, 54, 207, 1042, 1039, 944, 1733, 376, 270, 393, 658, 166, 1446, 229, 779, 1085, 453, 2446, 35, 892, 970, 9, 31, 592, 487, 787, 63, 618, 1395, 1638, 58, 1183, 20, 348, 13, 10, 19, 482, 12] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[234, 185, 822, 1, 2, 698, 849, 231, 1, 539, 30, 0, 17, 39, 563, 153, 1, 885, 656, 1987, 21, 170, 142, 1, 71, 27, 47, 239, 1084, 704, 1083, 1487, 417, 365, 555, 1361, 80, 1682, 574, 892, 1203, 581, 1864, 40, 1026, 790, 32, 37, 768, 510, 626, 22, 794, 1205, 1927, 23, 1239, 11, 247, 5, 27, 13, 864, 19] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[126, 553, 604, 0, 0, 672, 1201, 354, 0, 847, 29, 2, 37, 42, 165, 120, 2, 895, 777, 1863, 30, 304, 277, 11, 92, 56, 92, 415, 1513, 614, 823, 1504, 505, 436, 404, 1404, 102, 1686, 829, 946, 837, 664, 1920, 69, 1040, 781, 14, 187, 587, 691, 640, 84, 444, 833, 1549, 89, 920, 37, 203, 10, 10, 11, 793, 23] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[187, 127, 536, 5, 1, 691, 892, 138, 4, 732, 24, 2, 13, 44, 301, 104, 6, 1085, 1111, 1936, 16, 304, 271, 2, 76, 3, 21, 211, 1092, 798, 1066, 1112, 786, 388, 600, 1119, 37, 1500, 560, 901, 1089, 360, 1568, 28, 849, 818, 18, 86, 1020, 584, 734, 33, 675, 1534, 2288, 69, 1306, 23, 251, 14, 23, 23, 547, 26] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[126, 257, 692, 0, 0, 518, 1089, 408, 668, 361, 0, 0, 0, 0, 0, 0, 3, 1032, 19, 0, 6, 19, 406, 77, 1466, 823, 0, 0, 0, 0, 0, 0, 15, 878, 1098, 1866, 31, 50, 226, 1, 0, 0, 0, 0, 0, 0, 0, 0, 78, 16, 46, 206, 1083, 1003, 1206, 771, 0, 0, 0, 0, 0, 0, 0, 0, 893, 392, 332, 745, 201, 349, 200, 965, 0, 0, 0, 0, 0, 0, 0, 0, 943, 684, 429, 23, 1020, 680, 11, 35, 0, 0, 0, 0, 0, 0, 0, 0, 726, 969, 589, 120, 648, 902, 2, 13, 0, 0, 0, 0, 0, 0, 0, 0, 1140, 10, 348, 7, 7, 33, 1151, 10, 866, 573, 208, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 3851, 4165, 4409, 4077, 3825, 3969, 4353] avg=4096.0000 max=4409 max_rank=3 min=3825 min_rank=5 imbalance_ratio=1.076416 max_load_reduction=28359 max_load_reduction_pct=0.865448 moved_to_echo=16224 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[286, 268, 715, 2, 3, 809, 726, 157, 612, 319, 0, 0, 0, 0, 0, 0, 0, 1315, 26, 5, 18, 18, 182, 156, 1953, 717, 0, 0, 0, 0, 0, 0, 23, 1207, 925, 1650, 23, 258, 241, 15, 0, 0, 0, 0, 0, 0, 0, 0, 225, 9, 74, 275, 1342, 587, 816, 705, 0, 0, 0, 0, 0, 0, 0, 0, 545, 183, 462, 621, 192, 429, 217, 524, 0, 0, 0, 0, 0, 0, 0, 0, 1623, 350, 375, 99, 1021, 825, 9, 138, 0, 0, 0, 0, 0, 0, 0, 0, 920, 460, 608, 64, 739, 1629, 3, 73, 0, 0, 0, 0, 0, 0, 0, 0, 1470, 25, 184, 4, 19, 21, 486, 30, 1068, 499, 191, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3897, 4390, 4342, 4033, 3173, 4440, 4496, 3997] avg=4096.0000 max=4496 max_rank=6 min=3173 min_rank=4 imbalance_ratio=1.097656 max_load_reduction=28272 max_load_reduction_pct=0.862793 moved_to_echo=16106 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[239, 168, 791, 2, 1, 445, 1066, 279, 656, 319, 0, 0, 0, 0, 0, 0, 1, 922, 24, 0, 11, 11, 142, 157, 1755, 988, 0, 0, 0, 0, 0, 0, 1, 923, 1203, 1648, 28, 201, 270, 4, 0, 0, 0, 0, 0, 0, 0, 0, 64, 11, 119, 253, 967, 1007, 1036, 756, 0, 0, 0, 0, 0, 0, 0, 0, 529, 315, 587, 896, 101, 393, 292, 583, 0, 0, 0, 0, 0, 0, 0, 0, 1387, 661, 515, 34, 870, 799, 15, 163, 0, 0, 0, 0, 0, 0, 0, 0, 864, 436, 739, 4, 571, 1520, 3, 28, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 19, 299, 9, 4, 13, 719, 17, 978, 687, 204, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3966, 4011, 4278, 4213, 3696, 4444, 4165, 3995] avg=4096.0000 max=4444 max_rank=5 min=3696 min_rank=4 imbalance_ratio=1.084961 max_load_reduction=28324 max_load_reduction_pct=0.864380 moved_to_echo=16300 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[95, 187, 657, 4, 0, 754, 1211, 184, 703, 384, 0, 0, 0, 0, 0, 0, 0, 1396, 63, 0, 10, 7, 442, 107, 1636, 1103, 0, 0, 0, 0, 0, 0, 9, 1078, 766, 1984, 29, 54, 295, 1, 0, 0, 0, 0, 0, 0, 0, 0, 46, 23, 54, 207, 1042, 1039, 944, 811, 0, 0, 0, 0, 0, 0, 0, 0, 376, 270, 393, 658, 166, 415, 229, 779, 0, 0, 0, 0, 0, 0, 0, 0, 1085, 453, 575, 35, 892, 970, 9, 31, 0, 0, 0, 0, 0, 0, 0, 0, 592, 487, 787, 63, 618, 1395, 2, 58, 0, 0, 0, 0, 0, 0, 0, 0, 1183, 20, 348, 13, 10, 19, 482, 12, 1031, 768, 219, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4179, 4764, 4216, 4166, 3286, 4050, 4002, 4105] avg=4096.0000 max=4764 max_rank=1 min=3286 min_rank=4 imbalance_ratio=1.163086 max_load_reduction=28004 max_load_reduction_pct=0.854614 moved_to_echo=15443 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[165, 150, 1061, 2, 0, 699, 1134, 459, 859, 360, 0, 0, 0, 0, 0, 0, 0, 938, 14, 0, 48, 12, 364, 59, 1481, 851, 0, 0, 0, 0, 0, 0, 26, 896, 563, 1859, 41, 284, 284, 1, 0, 0, 0, 0, 0, 0, 0, 0, 44, 8, 52, 238, 904, 881, 792, 989, 0, 0, 0, 0, 0, 0, 0, 0, 532, 298, 402, 1112, 136, 428, 206, 768, 0, 0, 0, 0, 0, 0, 0, 0, 1615, 487, 444, 90, 828, 653, 21, 72, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 372, 612, 55, 493, 1585, 2, 40, 0, 0, 0, 0, 0, 0, 0, 0, 957, 38, 611, 10, 6, 25, 412, 7, 1065, 593, 268, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4889, 3767, 3954, 3908, 3882, 4210, 4166, 3992] avg=4096.0000 max=4889 max_rank=0 min=3767 min_rank=1 imbalance_ratio=1.193604 max_load_reduction=27879 max_load_reduction_pct=0.850800 moved_to_echo=16250 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[187, 127, 536, 5, 1, 691, 892, 138, 451, 314, 0, 0, 0, 0, 0, 0, 4, 732, 24, 2, 13, 44, 301, 104, 2286, 707, 0, 0, 0, 0, 0, 0, 6, 1085, 1111, 1622, 16, 304, 271, 2, 0, 0, 0, 0, 0, 0, 0, 0, 76, 3, 21, 211, 1092, 798, 1066, 521, 0, 0, 0, 0, 0, 0, 0, 0, 786, 388, 600, 1119, 37, 430, 560, 901, 0, 0, 0, 0, 0, 0, 0, 0, 1089, 360, 369, 28, 849, 818, 18, 86, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 584, 734, 33, 675, 1534, 2, 69, 0, 0, 0, 0, 0, 0, 0, 0, 1306, 23, 251, 14, 23, 23, 547, 26, 1070, 492, 140, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3342, 4217, 4417, 3788, 4821, 3617, 4651, 3915] avg=4096.0000 max=4821 max_rank=4 min=3342 min_rank=0 imbalance_ratio=1.177002 max_load_reduction=27947 max_load_reduction_pct=0.852875 moved_to_echo=17004 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[234, 185, 822, 1, 2, 698, 849, 231, 603, 322, 0, 0, 0, 0, 0, 0, 1, 539, 30, 0, 17, 39, 563, 153, 1924, 841, 0, 0, 0, 0, 0, 0, 1, 885, 656, 1665, 21, 170, 142, 1, 0, 0, 0, 0, 0, 0, 0, 0, 71, 27, 47, 239, 1084, 704, 1083, 696, 0, 0, 0, 0, 0, 0, 0, 0, 417, 365, 555, 1361, 80, 482, 574, 892, 0, 0, 0, 0, 0, 0, 0, 0, 1203, 581, 438, 40, 1026, 790, 32, 37, 0, 0, 0, 0, 0, 0, 0, 0, 768, 510, 626, 22, 794, 1205, 3, 23, 0, 0, 0, 0, 0, 0, 0, 0, 1239, 11, 247, 5, 27, 13, 864, 19, 1200, 585, 188, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3947, 4107, 3541, 3951, 4726, 4147, 3951, 4398] avg=4096.0000 max=4726 max_rank=4 min=3541 min_rank=2 imbalance_ratio=1.153809 max_load_reduction=28042 max_load_reduction_pct=0.855774 moved_to_echo=17222 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[126, 553, 604, 0, 0, 672, 1201, 354, 615, 304, 0, 0, 0, 0, 0, 0, 0, 847, 29, 2, 37, 42, 165, 120, 1549, 870, 0, 0, 0, 0, 0, 0, 2, 895, 777, 1559, 30, 304, 277, 11, 0, 0, 0, 0, 0, 0, 0, 0, 92, 56, 92, 415, 1513, 614, 823, 695, 0, 0, 0, 0, 0, 0, 0, 0, 505, 436, 404, 1404, 102, 481, 829, 946, 0, 0, 0, 0, 0, 0, 0, 0, 837, 664, 443, 69, 1040, 781, 14, 187, 0, 0, 0, 0, 0, 0, 0, 0, 587, 691, 640, 84, 444, 833, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 920, 37, 203, 10, 10, 11, 793, 23, 1205, 607, 194, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4429, 3661, 3855, 4300, 5107, 4035, 3368, 4013] avg=4096.0000 max=5107 max_rank=4 min=3368 min_rank=6 imbalance_ratio=1.246826 max_load_reduction=27661 max_load_reduction_pct=0.844147 moved_to_echo=16523 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1536, 1920, 5888, 128, 128, 5376, 8192, 2304, 5248, 2688, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[6528, 4608, 5376, 512, 4992, 10624, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[768, 256, 512, 2048, 9088, 6656, 7808, 6016, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[9856, 4352, 3712, 512, 7552, 6400, 256, 768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4608, 2688, 3840, 7936, 1024, 3456, 3200, 6400, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 7936, 7168, 13952, 256, 1664, 2048, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[9344, 256, 2560, 128, 128, 256, 5504, 256, 8576, 4864, 1664, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 7808, 256, 128, 256, 256, 2688, 1024, 14080, 6912, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1, 1, 46, 0, 0, 874, 1, 81, 8, 2, 4078, 0, 9, 0, 6, 1, 1, 0, 4033, 1, 80, 1, 4057, 1, 0, 1833, 0, 0, 48, 0, 3266, 0, 0, 0, 0, 0, 291, 3887, 415, 3, 4085, 8, 91, 1, 0, 0, 0, 0, 0, 0, 35, 0, 134, 0, 0, 1, 20, 0, 3, 2340, 2486, 0, 538, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1930, 0, 2, 0, 10, 4046, 40, 0, 0, 11, 0, 0, 0, 3855, 35, 206, 110, 4002, 0, 0, 1134, 0, 0, 0, 11, 2874, 3, 0, 0, 0, 0, 2, 2773, 1172, 0, 4081, 0, 37, 0, 0, 0, 1, 0, 92, 0, 47, 0, 0, 0, 73, 0, 27, 6, 96, 2781, 636, 0, 2675, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[8, 0, 1, 0, 0, 2106, 0, 8, 1, 2, 3892, 0, 0, 0, 4, 0, 0, 0, 3277, 0, 704, 21, 4078, 0, 0, 12, 0, 1, 3, 27, 3589, 41, 0, 0, 0, 0, 14, 2596, 291, 1, 4020, 0, 56, 0, 0, 0, 0, 0, 113, 1, 136, 2, 0, 0, 0, 0, 6, 0, 896, 3422, 2555, 0, 884, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 0, 1009, 0, 0, 0, 0, 3962, 2, 0, 0, 51, 0, 0, 0, 4054, 16, 0, 82, 3152, 4, 0, 316, 0, 0, 4, 3, 3007, 0, 0, 0, 0, 0, 200, 3271, 401, 0, 4094, 0, 1234, 0, 0, 0, 0, 0, 0, 0, 0, 19, 32, 0, 12, 0, 182, 1, 375, 3992, 2343, 0, 947, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[12, 0, 7, 0, 0, 134, 0, 24, 0, 0, 4019, 5, 0, 0, 152, 0, 0, 0, 3985, 1, 10, 11, 3789, 1, 0, 64, 3, 0, 0, 2, 3809, 0, 0, 0, 0, 0, 471, 2420, 1285, 16, 3849, 0, 181, 0, 0, 0, 2, 0, 14, 0, 1273, 78, 0, 0, 0, 7, 175, 1, 4, 3675, 3144, 0, 145, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[2, 0, 3, 0, 0, 2124, 0, 76, 4, 0, 3945, 1, 4, 0, 0, 0, 0, 0, 4008, 0, 11, 36, 4003, 0, 0, 191, 0, 0, 0, 0, 2564, 1, 0, 0, 0, 0, 2, 2994, 539, 0, 4016, 0, 11, 0, 0, 0, 34, 0, 7, 11, 134, 293, 3, 0, 0, 3, 102, 0, 389, 3345, 3495, 0, 417, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[66, 0, 15, 0, 0, 1862, 0, 76, 0, 0, 3651, 21, 0, 2, 105, 0, 0, 0, 3631, 37, 126, 0, 1019, 0, 0, 22, 0, 0, 1, 1, 3488, 9, 0, 0, 3, 0, 14, 3578, 777, 0, 4009, 0, 114, 0, 0, 0, 0, 18, 2, 6, 41, 2, 6, 1, 14, 0, 12, 0, 828, 3684, 2601, 1, 2925, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[13, 0, 0, 0, 0, 767, 0, 0, 0, 1, 3843, 42, 0, 22, 6, 0, 0, 4, 3839, 34, 91, 88, 2866, 0, 0, 1711, 0, 13, 0, 55, 3337, 116, 1, 0, 0, 0, 38, 1355, 1358, 0, 3702, 0, 125, 5, 8, 0, 1, 0, 42, 8, 571, 1, 4, 0, 0, 1, 145, 5, 1304, 3447, 2405, 0, 1388, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 0, 1009, 0, 0, 2842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3962, 2, 0, 0, 51, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 16, 0, 82, 3152, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 316, 0, 0, 4, 3, 3007, 0, 120, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 3271, 401, 0, 292, 46, 0, 0, 0, 0, 0, 0, 3964, 0, 1234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 32, 0, 12, 0, 3992, 378, 0, 0, 0, 0, 0, 0, 182, 1, 375, 0, 2177, 0, 947, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3854, 4120, 3796, 3475, 4210, 5198, 4433, 3682] avg=4096.0000 max=5198 max_rank=5 min=3475 min_rank=3 imbalance_ratio=1.269043 max_load_reduction=27570 max_load_reduction_pct=0.841370 moved_to_echo=17523 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1930, 0, 2, 2703, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4046, 40, 0, 0, 11, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 35, 206, 110, 4002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1134, 0, 0, 0, 11, 2874, 3, 32, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2773, 1172, 0, 278, 12, 0, 0, 0, 0, 0, 0, 3952, 0, 37, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 47, 0, 0, 0, 73, 0, 2781, 359, 0, 0, 0, 0, 0, 0, 27, 6, 96, 0, 592, 0, 2675, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4635, 4212, 4868, 4078, 4237, 3990, 3352, 3396] avg=4096.0000 max=4868 max_rank=2 min=3352 min_rank=6 imbalance_ratio=1.188477 max_load_reduction=27900 max_load_reduction_pct=0.851440 moved_to_echo=14975 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[13, 0, 0, 0, 0, 767, 0, 0, 2695, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3843, 42, 0, 22, 6, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0, 4, 501, 34, 91, 88, 2866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1711, 0, 13, 0, 55, 3337, 116, 127, 26, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 38, 1355, 1358, 0, 280, 51, 0, 0, 0, 0, 0, 0, 3579, 0, 125, 5, 8, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 8, 571, 1, 4, 0, 0, 1, 3447, 363, 0, 0, 0, 0, 0, 0, 145, 5, 1304, 0, 2227, 0, 1388, 6, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3475, 4011, 3584, 5385, 3083, 3718, 4437, 5075] avg=4096.0000 max=5385 max_rank=3 min=3083 min_rank=4 imbalance_ratio=1.314697 max_load_reduction=27383 max_load_reduction_pct=0.835663 moved_to_echo=16313 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 46, 0, 0, 874, 1, 81, 2828, 0, 0, 0, 0, 0, 0, 0, 8, 2, 4078, 0, 9, 0, 6, 1, 105, 0, 0, 0, 0, 0, 0, 0, 1, 0, 539, 1, 80, 1, 4057, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1833, 0, 0, 48, 0, 3266, 0, 128, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 291, 3887, 415, 3, 290, 48, 0, 0, 0, 0, 0, 0, 3955, 8, 91, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 134, 0, 0, 1, 2340, 376, 0, 0, 0, 0, 0, 0, 20, 0, 3, 0, 2310, 0, 538, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3832, 4209, 4680, 5300, 4934, 4055, 2886, 2872] avg=4096.0000 max=5300 max_rank=3 min=2872 min_rank=7 imbalance_ratio=1.293945 max_load_reduction=27468 max_load_reduction_pct=0.838257 moved_to_echo=14747 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 3, 0, 0, 2124, 0, 76, 2810, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3945, 1, 4, 0, 0, 0, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 0, 11, 36, 4003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 191, 0, 0, 0, 0, 2564, 1, 180, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2994, 539, 0, 289, 68, 0, 0, 0, 0, 0, 0, 3889, 0, 11, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 11, 134, 293, 3, 0, 0, 3, 3345, 373, 0, 0, 0, 0, 0, 0, 102, 0, 389, 0, 3247, 0, 417, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5015, 4057, 4586, 2960, 3892, 3934, 4169, 4155] avg=4096.0000 max=5015 max_rank=0 min=2960 min_rank=3 imbalance_ratio=1.224365 max_load_reduction=27753 max_load_reduction_pct=0.846954 moved_to_echo=16150 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 7, 0, 0, 134, 0, 24, 2794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4019, 5, 0, 0, 152, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 533, 1, 10, 11, 3789, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 3, 0, 0, 2, 3809, 0, 162, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 2420, 1285, 16, 287, 61, 0, 0, 0, 0, 0, 0, 3727, 0, 181, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 1273, 78, 0, 0, 0, 7, 3675, 371, 0, 0, 0, 0, 0, 0, 175, 1, 4, 0, 2921, 0, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2971, 4275, 4345, 4063, 4540, 3910, 5418, 3246] avg=4096.0000 max=5418 max_rank=6 min=2971 min_rank=0 imbalance_ratio=1.322754 max_load_reduction=27350 max_load_reduction_pct=0.834656 moved_to_echo=17114 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[66, 0, 15, 0, 0, 1862, 0, 76, 2546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3651, 21, 0, 2, 105, 0, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 37, 126, 0, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 1, 1, 3488, 9, 134, 24, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 14, 3578, 777, 0, 261, 51, 0, 0, 0, 0, 0, 0, 3882, 0, 114, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6, 41, 2, 6, 1, 14, 0, 3684, 338, 0, 0, 0, 0, 0, 0, 12, 0, 828, 0, 2416, 1, 2925, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4565, 3882, 1668, 3679, 4684, 4014, 4094, 6182] avg=4096.0000 max=6182 max_rank=7 min=1668 min_rank=2 imbalance_ratio=1.509277 max_load_reduction=26586 max_load_reduction_pct=0.811340 moved_to_echo=18974 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 1, 0, 0, 2106, 0, 8, 2298, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3892, 0, 0, 0, 4, 0, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 438, 0, 704, 21, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 1, 3, 27, 3589, 41, 131, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 2596, 291, 1, 236, 50, 0, 0, 0, 0, 0, 0, 3893, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 1, 136, 2, 0, 0, 0, 0, 3422, 305, 0, 0, 0, 0, 0, 0, 6, 0, 896, 0, 2374, 0, 884, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4421, 4002, 5241, 3828, 3188, 3949, 3979, 4160] avg=4096.0000 max=5241 max_rank=2 min=3188 min_rank=4 imbalance_ratio=1.279541 max_load_reduction=27527 max_load_reduction_pct=0.840057 moved_to_echo=15276 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 0, 10880, 128, 384, 21632, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[768, 128, 3968, 0, 18304, 128, 9984, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 4096, 128, 1280, 384, 27008, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[384, 128, 2304, 512, 256, 128, 128, 128, 26752, 2944, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 5376, 128, 128, 128, 128, 25984, 256, 1024, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[30848, 128, 1920, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 31488, 128, 128, 128, 384, 128, 896, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 0, 128, 0, 1152, 22912, 6272, 128, 2304, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4078, 0, 4093, 0, 0, 0, 3, 4082, 0, 4079, 0, 560, 1332, 0, 6, 64, 11, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 2554, 0, 0, 0, 0, 0, 0, 1, 0, 0, 17, 41, 1, 0, 0, 16, 6, 3998, 4, 0, 3094, 546, 0, 21, 0, 0, 0, 4067, 81, 8, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4092, 0, 4085, 0, 273, 0, 1, 4091, 0, 3862, 0, 295, 3488, 1, 65, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 19, 0, 0, 0, 2317, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 217, 225, 4092, 0, 0, 2454, 794, 0, 0, 0, 0, 0, 2373, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 3930, 1, 3433, 0, 150, 0, 0, 4091, 1, 3969, 0, 261, 1293, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 281, 0, 1, 7, 0, 2110, 11, 0, 2, 0, 0, 3, 4, 0, 1, 0, 992, 1, 0, 0, 1, 0, 3993, 0, 0, 4079, 1, 0, 0, 0, 6, 0, 4070, 2, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4086, 0, 4061, 0, 137, 0, 0, 4049, 0, 4079, 0, 2343, 406, 23, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 1185, 39, 0, 0, 14, 2, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 43, 3939, 1, 1, 3414, 794, 0, 0, 3, 0, 0, 3977, 38, 64, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 3552, 0, 4094, 0, 97, 0, 0, 4091, 1, 3888, 1, 2205, 543, 0, 0, 1, 0, 10, 0, 0, 0, 0, 0, 7, 0, 0, 7, 0, 6, 1201, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 22, 0, 0, 1, 0, 6, 3728, 190, 0, 4002, 1101, 0, 0, 0, 0, 0, 4008, 0, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4050, 0, 4095, 0, 6, 0, 5, 4085, 2, 3880, 0, 1284, 2291, 219, 5, 0, 0, 0, 0, 0, 1, 0, 0, 1, 60, 0, 0, 0, 3, 2980, 9, 0, 3, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 3, 4057, 150, 0, 2777, 58, 0, 0, 5, 0, 1, 2661, 0, 47, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4011, 0, 3377, 0, 6, 0, 0, 4057, 0, 4028, 0, 428, 729, 3, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 3966, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4024, 6, 0, 3345, 686, 0, 0, 3, 0, 0, 4053, 1, 18, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[2, 0, 44, 0, 3993, 0, 4000, 0, 1, 0, 0, 4093, 0, 4079, 0, 1900, 3756, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 3323, 4, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 2, 4006, 0, 16, 1588, 59, 0, 2, 8, 0, 1, 1847, 4, 20, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 191, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 273, 0, 1, 0, 0, 2765, 0, 295, 0, 0, 0, 0, 0, 0, 0, 0, 3488, 1, 65, 2, 0, 0, 0, 0, 2369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 19, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 2317, 0, 0, 0, 14, 0, 0, 1199, 446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 3901, 126, 41, 0, 0, 0, 0, 0, 217, 225, 398, 0, 0, 2454, 794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2373, 0, 0, 1, 651, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4277, 3334, 5925, 4070, 3976, 4073, 4088, 3025] avg=4096.0000 max=5925 max_rank=2 min=3025 min_rank=7 imbalance_ratio=1.446533 max_load_reduction=26843 max_load_reduction_pct=0.819183 moved_to_echo=15162 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 183, 1, 3433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 0, 0, 0, 1, 2842, 0, 261, 0, 0, 0, 0, 0, 0, 0, 0, 1293, 0, 72, 0, 0, 0, 0, 0, 2312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 281, 0, 1, 7, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 2110, 11, 0, 2, 0, 0, 3, 1170, 458, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 992, 1, 0, 0, 3747, 123, 40, 0, 0, 0, 0, 0, 1, 0, 388, 0, 0, 4079, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4070, 2, 0, 0, 669, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3619, 3254, 3677, 4340, 3754, 4908, 4469, 4747] avg=4096.0000 max=4908 max_rank=5 min=3254 min_rank=1 imbalance_ratio=1.198242 max_load_reduction=27860 max_load_reduction_pct=0.850220 moved_to_echo=17878 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 190, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 0, 0, 2921, 0, 2343, 0, 0, 0, 0, 0, 0, 0, 0, 406, 23, 1, 0, 3, 0, 0, 0, 2281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 4008, 0, 0, 0, 0, 0, 0, 0, 0, 1185, 39, 0, 0, 14, 2, 0, 1154, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 3896, 121, 41, 0, 0, 0, 0, 0, 0, 43, 383, 1, 1, 3414, 794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3977, 38, 64, 0, 687, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4252, 5401, 2714, 4042, 2865, 4089, 4636, 4769] avg=4096.0000 max=5401 max_rank=1 min=2714 min_rank=2 imbalance_ratio=1.318604 max_load_reduction=27367 max_load_reduction_pct=0.835175 moved_to_echo=16359 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 189, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 5, 0, 2, 2778, 0, 1284, 0, 0, 0, 0, 0, 0, 0, 0, 2291, 219, 5, 0, 0, 0, 0, 0, 2349, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 60, 0, 0, 0, 4045, 0, 0, 0, 0, 0, 0, 0, 3, 2980, 9, 0, 3, 0, 0, 0, 1189, 448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 3861, 125, 40, 0, 0, 0, 0, 0, 0, 3, 394, 150, 0, 2777, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 1, 2661, 0, 47, 0, 654, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4285, 4075, 4864, 4107, 4632, 4055, 3382, 3368] avg=4096.0000 max=4864 max_rank=2 min=3368 min_rank=7 imbalance_ratio=1.187500 max_load_reduction=27904 max_load_reduction_pct=0.851562 moved_to_echo=15437 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 187, 0, 3377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 2884, 0, 428, 0, 0, 0, 0, 0, 0, 0, 0, 729, 3, 1, 0, 2, 0, 0, 0, 2330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 4016, 0, 0, 0, 0, 0, 0, 0, 0, 3966, 1, 0, 0, 0, 0, 0, 1179, 465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3824, 123, 41, 0, 0, 0, 0, 0, 2, 0, 392, 6, 0, 3345, 686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4053, 1, 18, 0, 679, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3564, 3318, 3065, 4037, 5611, 3988, 4431, 4754] avg=4096.0000 max=5611 max_rank=4 min=3065 min_rank=2 imbalance_ratio=1.369873 max_load_reduction=27157 max_load_reduction_pct=0.828766 moved_to_echo=18784 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 44, 0, 186, 0, 4000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2921, 0, 1900, 0, 0, 0, 0, 0, 0, 0, 0, 3756, 0, 0, 0, 2, 0, 0, 0, 2320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 3323, 4, 0, 0, 2, 0, 0, 1174, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3807, 123, 41, 0, 0, 0, 0, 0, 3, 2, 389, 0, 16, 1588, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8, 0, 1, 1847, 4, 20, 0, 687, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4232, 4822, 6078, 4063, 4974, 3973, 2057, 2569] avg=4096.0000 max=6078 max_rank=2 min=2057 min_rank=6 imbalance_ratio=1.483887 max_load_reduction=26690 max_load_reduction_pct=0.814514 moved_to_echo=13573 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 190, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2921, 0, 560, 0, 0, 0, 0, 0, 0, 0, 0, 1332, 0, 6, 64, 11, 0, 0, 0, 2315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 2, 2554, 0, 0, 0, 0, 0, 0, 1172, 471, 0, 0, 0, 0, 0, 0, 1, 0, 0, 17, 41, 1, 0, 0, 3888, 123, 40, 0, 0, 0, 0, 0, 16, 6, 388, 4, 0, 3094, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 4067, 81, 8, 1, 687, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4283, 3484, 3728, 4044, 4199, 4111, 4054, 4865] avg=4096.0000 max=4865 max_rank=7 min=3484 min_rank=1 imbalance_ratio=1.187744 max_load_reduction=27903 max_load_reduction_pct=0.851532 moved_to_echo=17229 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 161, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 0, 1, 2776, 1, 2205, 0, 0, 0, 0, 0, 0, 0, 0, 543, 0, 0, 1, 0, 10, 0, 0, 2163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 7, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 6, 1201, 0, 0, 0, 0, 0, 2, 1096, 452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 1, 3391, 117, 40, 0, 0, 0, 0, 0, 0, 6, 352, 190, 0, 4002, 1101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 0, 3, 0, 660, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4256, 5080, 2717, 4065, 2757, 3571, 5651, 4671] avg=4096.0000 max=5651 max_rank=6 min=2717 min_rank=2 imbalance_ratio=1.379639 max_load_reduction=27117 max_load_reduction_pct=0.827545 moved_to_echo=16650 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 19712, 128, 0, 128, 128, 128, 128, 9344, 3712, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[13952, 256, 256, 128, 128, 128, 0, 0, 18560, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 384, 3200, 384, 128, 24832, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[768, 0, 128, 0, 128, 22912, 128, 9344, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 27136, 128, 256, 128, 5376, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 0, 128, 128, 1152, 128, 128, 128, 30336, 1024, 384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 1536, 128, 31360, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 0, 0, 128, 512, 128, 128, 128, 32384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4088, 0, 0, 0, 272, 43, 0, 0, 4064, 0, 1, 0, 0, 4096, 3763, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 2450, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 5, 0, 1665, 0, 0, 177, 0, 3877, 0, 0, 0, 0, 0, 67, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[2, 0, 4, 1, 0, 4095, 0, 0, 0, 3782, 306, 0, 0, 3809, 0, 0, 0, 0, 4084, 4042, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 175, 0, 0, 2, 0, 0, 0, 0, 4055, 2, 0, 0, 0, 0, 0, 643, 30, 0, 23, 0, 3589, 0, 0, 0, 1, 2, 21, 6, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 4093, 0, 2, 0, 2449, 478, 0, 0, 1040, 0, 9, 0, 0, 4084, 3353, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4, 0, 4072, 0, 0, 1234, 0, 0, 2, 0, 0, 4, 0, 4095, 0, 0, 0, 0, 0, 0, 3885, 0, 0, 78, 0, 3872, 0, 0, 7, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 3633, 285, 0, 0, 3028, 0, 0, 0, 0, 4096, 2119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4089, 0, 0, 3818, 0, 0, 2, 0, 0, 1, 0, 3635, 0, 0, 0, 0, 0, 0, 118, 1, 0, 3, 0, 3830, 0, 0, 0, 0, 3, 5, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 4091, 0, 4, 0, 1773, 4, 0, 0, 1902, 0, 0, 5, 0, 4096, 4071, 0, 0, 0, 2, 0, 0, 0, 0, 0, 6, 0, 4094, 0, 0, 3710, 0, 1, 0, 0, 1, 0, 0, 4092, 26, 0, 0, 0, 2, 2, 23, 0, 0, 844, 3, 3997, 0, 0, 0, 0, 0, 16, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4086, 0, 0, 0, 2118, 3, 0, 0, 2532, 0, 39, 0, 0, 4096, 2645, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 1586, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 3, 0, 3081, 0, 0, 646, 0, 3696, 0, 0, 0, 0, 0, 38, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 1, 0, 1, 0, 4094, 0, 44, 0, 3091, 0, 0, 0, 458, 1, 2, 0, 0, 4055, 2212, 0, 3, 0, 0, 1, 0, 1, 0, 0, 2, 0, 4057, 0, 9, 3576, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 11, 0, 2732, 0, 0, 650, 0, 3675, 0, 0, 0, 0, 1, 4, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4, 4096, 0, 0, 0, 3557, 5, 0, 0, 2107, 0, 38, 0, 0, 3973, 3754, 0, 11, 0, 0, 0, 0, 0, 15, 3, 11, 0, 4074, 0, 0, 2865, 1, 0, 0, 0, 0, 0, 2, 3955, 0, 0, 0, 0, 68, 0, 83, 17, 3, 40, 3, 4065, 1, 5, 0, 9, 0, 0, 3, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2218, 285, 0, 0, 3028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 854, 2119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4089, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3818, 0, 0, 2, 0, 0, 1324, 725, 0, 0, 0, 0, 0, 0, 1, 0, 3635, 0, 0, 0, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 118, 1, 0, 3, 0, 2256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 5, 0, 0, 0, 3242, 849, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 5531, 2973, 4099, 5869, 3727, 2378, 4099] avg=4096.0000 max=5869 max_rank=4 min=2378 min_rank=6 imbalance_ratio=1.432861 max_load_reduction=26899 max_load_reduction_pct=0.820892 moved_to_echo=16073 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 4, 1, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2309, 306, 0, 0, 3809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 851, 4042, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 175, 0, 0, 2, 0, 0, 1378, 680, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 2, 0, 0, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 643, 30, 0, 23, 0, 2114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 21, 6, 0, 0, 3233, 795, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 6424, 4894, 4097, 2235, 4152, 2810, 4058] avg=4096.0000 max=6424 max_rank=1 min=2235 min_rank=4 imbalance_ratio=1.568359 max_load_reduction=26344 max_load_reduction_pct=0.803955 moved_to_echo=13255 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 4089, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1496, 478, 0, 0, 1040, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 851, 3353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4, 0, 4072, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1234, 0, 0, 2, 0, 0, 892, 733, 0, 0, 0, 0, 0, 0, 4, 0, 4095, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 3885, 0, 0, 78, 0, 2281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 3233, 858, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 3023, 4204, 4086, 2861, 4160, 6244, 4098] avg=4096.0000 max=6244 max_rank=6 min=2861 min_rank=4 imbalance_ratio=1.524414 max_load_reduction=26524 max_load_reduction_pct=0.809448 moved_to_echo=17363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 167, 43, 0, 0, 4064, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 854, 3763, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2450, 0, 0, 0, 0, 0, 99, 734, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 5, 6, 0, 0, 0, 0, 0, 0, 0, 0, 1665, 0, 0, 177, 0, 2284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 3242, 859, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4275, 4617, 4101, 3283, 4114, 4126, 4168] avg=4096.0000 max=4617 max_rank=2 min=3283 min_rank=4 imbalance_ratio=1.127197 max_load_reduction=28151 max_load_reduction_pct=0.859100 moved_to_echo=15691 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 4087, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1083, 4, 0, 0, 1902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 854, 4071, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4094, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3710, 0, 1, 0, 0, 1, 646, 757, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 26, 0, 0, 0, 2, 44, 0, 0, 0, 0, 0, 0, 0, 2, 23, 0, 0, 844, 3, 2354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 1, 0, 0, 3242, 886, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 2989, 4932, 4104, 5115, 4164, 3226, 4145] avg=4096.0000 max=5115 max_rank=4 min=2989 min_rank=1 imbalance_ratio=1.248779 max_load_reduction=27653 max_load_reduction_pct=0.843903 moved_to_echo=16650 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 1, 0, 4090, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1888, 0, 0, 0, 458, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 845, 2212, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 2, 0, 4057, 4, 0, 0, 0, 0, 0, 0, 0, 0, 9, 3576, 0, 0, 0, 0, 0, 1126, 696, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 11, 77, 0, 0, 0, 0, 0, 0, 0, 0, 2732, 0, 0, 650, 0, 2165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 0, 3210, 814, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4136, 2349, 3060, 4065, 5407, 4175, 5547, 4029] avg=4096.0000 max=5547 max_rank=6 min=2349 min_rank=1 imbalance_ratio=1.354248 max_load_reduction=27221 max_load_reduction_pct=0.830719 moved_to_echo=19158 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1293, 3, 0, 0, 2532, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 854, 2645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1586, 0, 0, 0, 0, 0, 772, 700, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 3, 53, 0, 0, 0, 0, 0, 0, 0, 0, 3081, 0, 0, 646, 0, 2177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 3242, 819, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 3867, 3499, 4101, 3058, 4158, 5904, 4099] avg=4096.0000 max=5904 max_rank=6 min=3058 min_rank=4 imbalance_ratio=1.441406 max_load_reduction=26864 max_load_reduction_pct=0.819824 moved_to_echo=17219 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2160, 5, 0, 0, 2107, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 824, 3754, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 3, 11, 0, 4074, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2865, 1, 0, 0, 0, 0, 1301, 773, 0, 0, 0, 0, 0, 0, 0, 2, 3955, 0, 0, 0, 0, 68, 93, 0, 0, 0, 0, 0, 0, 0, 0, 83, 17, 3, 40, 3, 2386, 1, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 9, 0, 0, 3, 0, 0, 3149, 906, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4310, 4589, 4115, 4940, 4118, 2533, 4072] avg=4096.0000 max=4940 max_rank=4 min=2533 min_rank=6 imbalance_ratio=1.206055 max_load_reduction=27828 max_load_reduction_pct=0.849243 moved_to_echo=15663 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 19456, 128, 128, 128, 0, 128, 7552, 5888, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 256, 128, 0, 0, 25856, 6912, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 12288, 128, 128, 2560, 128, 18048, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 12672, 1152, 0, 0, 18944, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 6912, 25984, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 0, 128, 128, 128, 128, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 32128, 128, 0, 0, 0, 128, 640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 258, 0, 0, 4067, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4087, 2440, 0, 0, 0, 0, 0, 4058, 0, 4094, 0, 1470, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 3, 0, 4096, 4095, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 4019, 4045, 0, 0, 0, 0, 0, 4093, 11, 3934, 0, 37, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 29, 0, 0, 4044, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4083, 4095, 3, 4096, 3866, 0, 0, 0, 19, 0, 4096, 1, 4062, 0, 195, 0, 0, 0, 0, 0, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 13, 0, 1, 19, 0, 0, 3993, 4093, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 10, 0, 0, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4086, 4073, 0, 3779, 2962, 1, 0, 0, 0, 0, 4072, 10, 4095, 0, 1401, 0, 1, 0, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 9, 0, 0, 3990, 4090, 8, 0, 0, 122, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 7, 0, 2, 1, 0, 1, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 4073, 3112, 0, 1, 0, 0, 0, 4090, 0, 4083, 0, 983, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 152, 0, 0, 4056, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 58, 0, 0, 0, 0, 2, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4095, 4027, 0, 0, 0, 0, 0, 4094, 0, 3907, 0, 83, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1020, 0, 3, 4095, 4091, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 430, 0, 1, 1, 0, 0, 0, 0, 233, 1, 4, 2, 0, 0, 0, 0, 0, 0, 0, 4079, 4094, 0, 4095, 1249, 0, 0, 0, 0, 0, 4090, 52, 3996, 0, 1177, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 8, 2, 340, 6, 0, 4066, 4085, 6, 0, 30, 6, 0, 0, 0, 0, 0, 0, 1, 3, 0, 4, 0, 2, 0, 0, 15, 0, 0, 0, 8, 0, 0, 0, 1, 0, 0, 0, 3, 0, 37, 4076, 4059, 0, 4067, 1548, 0, 2, 0, 1, 0, 3872, 63, 4045, 2, 2408, 1, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 3, 0, 3906, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 227, 3982, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1179, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 11, 3934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 2840, 190, 61, 21, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3915, 4077, 4034, 4270, 4103, 5225, 3995, 3149] avg=4096.0000 max=5225 max_rank=5 min=3149 min_rank=7 imbalance_ratio=1.275635 max_load_reduction=27543 max_load_reduction_pct=0.840546 moved_to_echo=16472 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 13, 0, 1, 19, 0, 0, 3807, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 35, 0, 4005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 104, 3961, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 5, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1108, 2962, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 10, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1401, 0, 1, 0, 0, 0, 0, 1, 2671, 186, 61, 21, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3840, 4072, 4056, 4138, 4091, 4074, 4155, 4342] avg=4096.0000 max=4342 max_rank=7 min=3840 min_rank=0 imbalance_ratio=1.060059 max_load_reduction=28426 max_load_reduction_pct=0.867493 moved_to_echo=16662 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 29, 0, 0, 3856, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 75, 3985, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 5, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 1201, 3866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 50, 1, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 195, 0, 0, 0, 0, 0, 0, 2, 2895, 188, 61, 21, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3887, 4069, 4034, 4125, 4088, 5071, 4132, 3362] avg=4096.0000 max=5071 max_rank=5 min=3362 min_rank=7 imbalance_ratio=1.238037 max_load_reduction=27697 max_load_reduction_pct=0.845245 moved_to_echo=16653 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 258, 0, 0, 3878, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 4028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3948, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1199, 2440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1470, 0, 0, 0, 0, 0, 0, 0, 2888, 189, 61, 21, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4137, 4075, 4033, 4011, 4101, 3639, 4143, 4629] avg=4096.0000 max=4629 max_rank=7 min=3639 min_rank=5 imbalance_ratio=1.130127 max_load_reduction=28139 max_load_reduction_pct=0.858734 moved_to_echo=16512 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1020, 0, 3, 3905, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 430, 4028, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 233, 3979, 61, 0, 0, 0, 0, 0, 0, 1, 4, 2, 0, 0, 0, 0, 0, 4079, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1201, 1249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 52, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1177, 0, 0, 0, 0, 0, 0, 0, 2894, 190, 61, 21, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4928, 4095, 4488, 4275, 4091, 2450, 4098, 4343] avg=4096.0000 max=4928 max_rank=0 min=2450 min_rank=5 imbalance_ratio=1.203125 max_load_reduction=27840 max_load_reduction_pct=0.849609 moved_to_echo=14982 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 9, 0, 0, 3804, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 8, 0, 0, 122, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 1, 0, 0, 0, 3979, 61, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1195, 3112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 50, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 983, 0, 0, 0, 0, 0, 0, 0, 2878, 186, 61, 21, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3813, 4201, 4034, 4044, 4106, 4307, 4134, 4129] avg=4096.0000 max=4307 max_rank=5 min=3813 min_rank=0 imbalance_ratio=1.051514 max_load_reduction=28461 max_load_reduction_pct=0.868561 moved_to_echo=16676 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 152, 0, 0, 3867, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 58, 4028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4, 3983, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1201, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 3907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 2894, 189, 61, 21, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4019, 4075, 4088, 4050, 4103, 5228, 3957, 3248] avg=4096.0000 max=5228 max_rank=5 min=3248 min_rank=7 imbalance_ratio=1.276367 max_load_reduction=27540 max_load_reduction_pct=0.840454 moved_to_echo=16536 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8, 2, 340, 6, 0, 3873, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 6, 0, 30, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 4, 0, 3993, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 15, 0, 0, 0, 8, 3769, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 4076, 5, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 1189, 1548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 41, 63, 4045, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2408, 1, 0, 0, 1, 0, 0, 0, 2878, 193, 62, 23, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4229, 4104, 4001, 3855, 4085, 2774, 4154, 5566] avg=4096.0000 max=5566 max_rank=7 min=2774 min_rank=5 imbalance_ratio=1.358887 max_load_reduction=27202 max_load_reduction_pct=0.830139 moved_to_echo=16579 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 0, 0, 128, 9600, 23296, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[7808, 128, 128, 0, 128, 0, 0, 128, 22912, 1536, 512, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 1920, 128, 128, 30976, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 0, 0, 768, 31616, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[32640, 128, 128, 128, 256, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 0, 0, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 512, 256, 32256, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 128, 0, 128, 512, 32256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 315, 0, 1, 0, 0, 0, 0, 4, 4092, 0, 1, 0, 0, 0, 2920, 317, 24, 0, 0, 16, 0, 3994, 3626, 10, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 1540, 39, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 3631, 1, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 221, 0, 3, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 2415, 1961, 0, 0, 0, 4, 0, 4094, 3924, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 15, 2, 0, 5, 0, 0, 4094, 0, 0, 0, 0, 0, 3744, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 260, 0, 0, 0, 2, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 1241, 183, 0, 0, 0, 449, 0, 3951, 4069, 0, 0, 0, 0, 0, 2, 4085, 0, 1, 0, 0, 2, 4096, 0, 1, 0, 0, 0, 1887, 0, 4, 177, 0, 0, 4094, 0, 2, 0, 0, 0, 4093, 0, 0, 76, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 498, 0, 579, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 615, 688, 1, 0, 0, 1023, 0, 4050, 3371, 0, 0, 0, 1, 0, 0, 4092, 0, 0, 0, 0, 4, 3918, 0, 0, 1, 0, 0, 1191, 18, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3666, 0, 0, 867, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 64, 0, 55, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 124, 2906, 3, 0, 0, 3, 25, 4093, 4049, 0, 1, 0, 0, 0, 0, 4064, 0, 1, 0, 0, 8, 3980, 0, 0, 0, 0, 0, 3680, 4, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 1522, 0, 0, 6, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 845, 0, 12, 0, 0, 0, 0, 0, 4094, 0, 1, 0, 0, 0, 2563, 1552, 0, 0, 0, 788, 0, 4092, 4048, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 89, 3, 0, 10, 0, 0, 4096, 0, 0, 0, 0, 0, 2357, 0, 0, 26, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 22, 332, 0, 54, 0, 9, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 1595, 537, 38, 0, 0, 428, 0, 3085, 4074, 0, 0, 0, 8, 0, 0, 3936, 0, 0, 0, 1, 3, 3958, 0, 0, 0, 1, 0, 3336, 306, 0, 0, 10, 1, 4083, 0, 0, 0, 2, 1, 2862, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 3, 1, 23, 1, 1930, 0, 0, 0, 1, 0, 4087, 6, 35, 0, 0, 0, 1103, 5, 17, 0, 0, 20, 177, 4021, 3951, 0, 0, 2, 0, 0, 0, 4012, 0, 8, 0, 0, 0, 4079, 0, 0, 42, 0, 0, 1937, 0, 0, 12, 41, 2, 4033, 0, 0, 0, 4, 1, 3192, 0, 0, 4, 1, 0, 17, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 498, 0, 579, 0, 0, 3470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 615, 688, 1, 0, 0, 1023, 0, 1165, 0, 0, 0, 0, 0, 0, 0, 212, 3371, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 4, 3918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1191, 18, 0, 0, 1805, 564, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 3666, 0, 0, 867, 0, 0, 0, 0, 864, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4547, 4089, 3492, 3584, 3980, 3579, 4100, 5397] avg=4096.0000 max=5397 max_rank=7 min=3492 min_rank=2 imbalance_ratio=1.317627 max_load_reduction=27371 max_load_reduction_pct=0.835297 moved_to_echo=17056 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 315, 0, 1, 0, 0, 3470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4092, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2920, 317, 24, 0, 0, 16, 0, 1148, 0, 0, 0, 0, 0, 0, 0, 209, 3626, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1540, 39, 0, 0, 1780, 564, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 3631, 1, 0, 0, 0, 0, 0, 0, 853, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3786, 4097, 4425, 3845, 4124, 3923, 4083, 4485] avg=4096.0000 max=4485 max_rank=7 min=3786 min_rank=0 imbalance_ratio=1.094971 max_load_reduction=28283 max_load_reduction_pct=0.863129 moved_to_echo=16615 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 221, 0, 3, 0, 0, 3473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2415, 1961, 0, 0, 0, 4, 0, 1177, 0, 0, 0, 0, 0, 0, 0, 215, 3924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 2, 0, 5, 1824, 565, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 3744, 0, 0, 0, 0, 0, 0, 0, 874, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3697, 4096, 5557, 4139, 4152, 2411, 4098, 4618] avg=4096.0000 max=5557 max_rank=2 min=2411 min_rank=5 imbalance_ratio=1.356689 max_load_reduction=27211 max_load_reduction_pct=0.830414 moved_to_echo=15279 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 260, 0, 0, 0, 2, 3464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1241, 183, 0, 0, 0, 449, 0, 1136, 0, 0, 0, 0, 0, 0, 0, 208, 4069, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 1, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1887, 0, 4, 177, 1760, 563, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 76, 0, 0, 0, 1, 843, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3726, 4092, 3009, 4279, 4157, 4392, 4100, 5013] avg=4096.0000 max=5013 max_rank=7 min=3009 min_rank=2 imbalance_ratio=1.223877 max_load_reduction=27755 max_load_reduction_pct=0.847015 moved_to_echo=17662 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 64, 0, 55, 0, 0, 3446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 2906, 3, 0, 0, 3, 25, 1177, 0, 0, 0, 0, 0, 0, 0, 214, 4049, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 1, 0, 0, 8, 3980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3680, 4, 0, 0, 1824, 560, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1522, 0, 0, 6, 0, 0, 0, 0, 874, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3565, 4083, 4238, 4264, 4047, 6068, 4101, 2402] avg=4096.0000 max=6068 max_rank=5 min=2402 min_rank=7 imbalance_ratio=1.481445 max_load_reduction=26700 max_load_reduction_pct=0.814819 moved_to_echo=16618 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 845, 0, 12, 0, 0, 3473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2563, 1552, 0, 0, 0, 788, 0, 1177, 0, 0, 0, 0, 0, 0, 0, 215, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 3, 0, 10, 1823, 565, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 2357, 0, 0, 26, 0, 0, 0, 0, 873, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4330, 4095, 6080, 4263, 4154, 2490, 4100, 3256] avg=4096.0000 max=6080 max_rank=2 min=2490 min_rank=5 imbalance_ratio=1.484375 max_load_reduction=26688 max_load_reduction_pct=0.814453 moved_to_echo=14000 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 22, 332, 0, 54, 0, 9, 3337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1595, 537, 38, 0, 0, 428, 0, 887, 0, 0, 0, 0, 0, 0, 0, 163, 4074, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 1, 3, 3958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3336, 306, 0, 0, 1374, 543, 0, 0, 0, 0, 0, 0, 10, 1, 4083, 0, 0, 0, 2, 1, 3, 0, 0, 0, 0, 0, 0, 0, 2862, 0, 0, 0, 0, 0, 0, 0, 658, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3754, 4086, 3485, 4245, 4018, 5560, 4100, 3520] avg=4096.0000 max=5560 max_rank=5 min=3485 min_rank=2 imbalance_ratio=1.357422 max_load_reduction=27208 max_load_reduction_pct=0.830322 moved_to_echo=17198 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 1, 23, 1, 1930, 0, 0, 3405, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4087, 6, 35, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1103, 5, 17, 0, 0, 20, 177, 1160, 0, 0, 0, 0, 0, 0, 0, 196, 3951, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 8, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 1937, 0, 0, 12, 1796, 558, 0, 0, 0, 0, 0, 0, 41, 2, 4033, 0, 0, 0, 4, 1, 5, 0, 0, 0, 0, 0, 0, 0, 3192, 0, 0, 4, 1, 0, 17, 0, 863, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5363, 4130, 2482, 4149, 4136, 4345, 4086, 4077] avg=4096.0000 max=5363 max_rank=0 min=2482 min_rank=2 imbalance_ratio=1.309326 max_load_reduction=27405 max_load_reduction_pct=0.836334 moved_to_echo=16644 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 12672, 8192, 128, 0, 0, 2816, 256, 9088, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 13696, 384, 128, 256, 14080, 4608, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[25088, 128, 0, 1024, 128, 0, 128, 128, 6784, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 2560, 128, 2688, 0, 128, 27648, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[1664, 31232, 128, 128, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[512, 0, 128, 0, 128, 128, 32384, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 32768, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 32768, 0, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[3651, 0, 12, 0, 4074, 4092, 0, 0, 0, 0, 0, 0, 70, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 4089, 0, 0, 3, 32, 0, 0, 0, 0, 4096, 0, 848, 0, 0, 0, 2180, 0, 0, 0, 4, 2, 0, 0, 1, 2, 0, 4094, 1479, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[3362, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 4057, 4095, 0, 3, 0, 63, 0, 0, 0, 0, 4096, 0, 2895, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3970, 1719, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[3738, 0, 57, 0, 4089, 4096, 0, 0, 0, 0, 0, 0, 128, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3988, 4096, 0, 0, 0, 333, 0, 1, 0, 0, 4096, 0, 780, 7, 0, 0, 38, 0, 0, 0, 0, 18, 0, 0, 1, 0, 0, 4060, 3225, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[3777, 0, 3, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 4096, 0, 0, 0, 498, 0, 0, 0, 0, 4096, 0, 1231, 0, 0, 0, 100, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 2537, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[79, 2, 0, 1, 4089, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 3505, 4093, 0, 0, 2, 1267, 0, 8, 0, 0, 4091, 1, 3568, 0, 0, 0, 835, 0, 0, 0, 0, 0, 2, 2, 0, 0, 3, 3996, 3112, 0, 0, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[2420, 0, 3, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 519, 0, 0, 0, 0, 4096, 0, 2072, 0, 0, 0, 1048, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 2139, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[3770, 0, 188, 0, 3979, 4096, 0, 0, 0, 0, 0, 0, 1, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4091, 1, 30, 0, 2370, 0, 1, 0, 0, 4094, 0, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 3982, 1015, 0, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[3797, 0, 0, 0, 4078, 4087, 0, 0, 5, 0, 0, 17, 4, 0, 0, 10, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3596, 4041, 0, 0, 0, 604, 31, 0, 0, 0, 4082, 0, 3130, 0, 25, 0, 221, 6, 0, 0, 2, 0, 0, 0, 53, 0, 0, 3735, 1180, 0, 0, 0, 6, 0, 0, 47, 4, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3362, 0, 0, 0, 995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 299, 0, 0, 0, 3101, 916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3489, 0, 0, 3, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2147, 0, 2895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1949, 1029, 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2025, 1719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4357, 4316, 4108, 3555, 5042, 3551, 3744, 4095] avg=4096.0000 max=5042 max_rank=4 min=3551 min_rank=5 imbalance_ratio=1.230957 max_load_reduction=27726 max_load_reduction_pct=0.846130 moved_to_echo=16432 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3651, 0, 12, 0, 989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 2, 0, 0, 3085, 944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3472, 0, 0, 0, 3, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2147, 0, 848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2180, 0, 0, 0, 4, 2, 1949, 1061, 565, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 2089, 1479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4652, 4101, 4092, 3507, 2995, 5761, 3571, 4089] avg=4096.0000 max=5761 max_rank=5 min=2995 min_rank=4 imbalance_ratio=1.406494 max_load_reduction=27007 max_load_reduction_pct=0.824188 moved_to_echo=16416 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3738, 0, 57, 0, 993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 15, 0, 0, 3096, 936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3430, 0, 0, 0, 0, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2147, 0, 780, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 18, 1949, 1052, 558, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2072, 3225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4788, 4175, 4098, 3763, 2935, 3615, 5298, 4096] avg=4096.0000 max=5298 max_rank=6 min=2935 min_rank=4 imbalance_ratio=1.293457 max_load_reduction=27470 max_load_reduction_pct=0.838318 moved_to_echo=15944 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[79, 2, 0, 1, 993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 10, 3096, 922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3014, 0, 0, 0, 2, 1267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 2144, 1, 3568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 835, 0, 0, 0, 0, 0, 1947, 1035, 491, 0, 0, 0, 0, 0, 2, 2, 0, 0, 3, 2039, 3112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4093, 0, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1075, 4030, 4094, 4287, 5721, 4308, 5158, 4095] avg=4096.0000 max=5721 max_rank=4 min=1075 min_rank=0 imbalance_ratio=1.396729 max_load_reduction=27047 max_load_reduction_pct=0.825409 moved_to_echo=19282 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3770, 0, 188, 0, 966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 26, 0, 0, 3013, 918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3515, 0, 1, 30, 0, 2370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2146, 0, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1948, 1032, 572, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2032, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4924, 3958, 4096, 5916, 3181, 3553, 3048, 4092] avg=4096.0000 max=5916 max_rank=3 min=3048 min_rank=6 imbalance_ratio=1.444336 max_load_reduction=26852 max_load_reduction_pct=0.819458 moved_to_echo=13874 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3777, 0, 3, 0, 995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 3101, 934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3510, 0, 0, 0, 0, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2147, 0, 1231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 4, 0, 0, 0, 1949, 1049, 571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2065, 2537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4775, 4140, 4096, 4008, 3378, 3673, 4602, 4096] avg=4096.0000 max=4775 max_rank=0 min=3378 min_rank=4 imbalance_ratio=1.165771 max_load_reduction=27993 max_load_reduction_pct=0.854279 moved_to_echo=15749 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2420, 0, 3, 0, 995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 3101, 938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3520, 0, 0, 0, 0, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2147, 0, 2072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 0, 4, 0, 0, 0, 1949, 1054, 573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2077, 2139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3418, 4056, 4096, 4039, 4219, 4628, 4216, 4096] avg=4096.0000 max=4628 max_rank=5 min=3418 min_rank=0 imbalance_ratio=1.129883 max_load_reduction=28140 max_load_reduction_pct=0.858765 moved_to_echo=17159 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3797, 0, 0, 0, 982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 17, 4, 0, 0, 10, 3091, 865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3089, 0, 0, 0, 0, 604, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 2136, 0, 3130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 221, 6, 0, 0, 2, 0, 1946, 972, 507, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 1898, 1180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 47, 4, 6, 4041, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4779, 3992, 4088, 3693, 5297, 3679, 3131, 4109] avg=4096.0000 max=5297 max_rank=4 min=3131 min_rank=6 imbalance_ratio=1.293213 max_load_reduction=27471 max_load_reduction_pct=0.838348 moved_to_echo=16216 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 0, 4480, 128, 128, 0, 128, 128, 15616, 8320, 4480, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 0, 0, 17280, 128, 15616, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 16384, 16512, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 0, 128, 640, 128, 0, 128, 24704, 7424, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[24704, 128, 384, 128, 7936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 27136, 0, 128, 128, 128, 5760, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 0, 128, 128, 128, 32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 1, 0, 84, 0, 0, 0, 3644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3263, 1353, 0, 0, 0, 0, 0, 0, 0, 1, 0, 15, 3714, 0, 0, 0, 0, 0, 0, 0, 341, 0, 0, 4096, 0, 0, 0, 3969, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 1, 0, 270, 0, 0, 0, 3670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2453, 3485, 103, 0, 0, 0, 0, 1, 0, 0, 0, 9, 1496, 0, 0, 0, 0, 0, 0, 0, 825, 0, 0, 4096, 0, 0, 0, 4070, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 1136, 18, 0, 0, 4018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1656, 1598, 43, 0, 0, 0, 0, 7, 0, 0, 0, 41, 3996, 0, 0, 0, 0, 40, 0, 0, 158, 0, 0, 4096, 0, 0, 0, 3689, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4092, 0, 0, 631, 0, 0, 0, 4080, 0, 0, 0, 75, 0, 0, 0, 0, 0, 7, 1, 0, 4096, 0, 0, 0, 5, 30, 0, 0, 0, 0, 1, 0, 2614, 3010, 118, 0, 0, 0, 0, 0, 0, 0, 0, 276, 973, 0, 0, 0, 0, 0, 0, 0, 1071, 0, 0, 4096, 0, 0, 0, 3507, 4085] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4058, 0, 0, 1780, 0, 0, 0, 2962, 0, 0, 1, 1, 1, 0, 0, 0, 0, 25, 0, 0, 4088, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 2138, 3825, 0, 6, 5, 0, 7, 0, 0, 0, 0, 295, 1250, 0, 0, 0, 0, 0, 0, 0, 544, 0, 1, 4095, 0, 4, 0, 3567, 4077] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 30, 0, 1208, 0, 0, 0, 3889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1782, 2403, 17, 0, 0, 0, 0, 80, 0, 0, 0, 435, 877, 0, 0, 0, 0, 0, 0, 0, 2150, 0, 0, 4096, 0, 0, 0, 3513, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[2, 0, 3, 4042, 41, 2, 1138, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 2, 0, 12, 0, 0, 0, 0, 4094, 1, 1, 0, 0, 0, 0, 37, 0, 0, 0, 2, 2114, 2111, 30, 0, 0, 0, 0, 0, 0, 17, 0, 0, 1681, 0, 4, 0, 2, 0, 0, 0, 1211, 0, 0, 4092, 2, 0, 0, 3975, 4086] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 4, 0, 4065, 13, 0, 506, 0, 0, 3, 101, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 10, 0, 4087, 0, 0, 0, 0, 0, 0, 12, 5, 0, 0, 0, 609, 3050, 0, 27, 0, 0, 0, 1642, 5, 0, 9, 406, 4017, 12, 0, 0, 0, 0, 0, 0, 1944, 1, 4, 4071, 0, 0, 0, 4075, 4086] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3251, 1, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3644, 0, 0, 0, 0, 0, 327, 148, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3263, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 15, 3714, 746, 696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 341, 3049, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3969, 300, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3336, 4121, 4094, 4084, 4289, 5172, 3403, 4269] avg=4096.0000 max=5172 max_rank=5 min=3336 min_rank=0 imbalance_ratio=1.262695 max_load_reduction=27596 max_load_reduction_pct=0.842163 moved_to_echo=17133 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3251, 0, 0, 1136, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4018, 0, 0, 0, 0, 0, 387, 148, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1656, 1211, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 41, 3996, 746, 697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 158, 3049, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3689, 300, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4405, 4555, 4079, 4083, 2910, 5487, 3260, 3989] avg=4096.0000 max=5487 max_rank=5 min=2910 min_rank=4 imbalance_ratio=1.339600 max_load_reduction=27281 max_load_reduction_pct=0.832550 moved_to_echo=15646 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3248, 0, 0, 631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 75, 0, 729, 148, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 30, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2614, 2281, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 276, 973, 744, 696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1071, 3041, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3507, 300, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3879, 5034, 4102, 4118, 5014, 2689, 4125, 3807] avg=4096.0000 max=5034 max_rank=1 min=2689 min_rank=5 imbalance_ratio=1.229004 max_load_reduction=27734 max_load_reduction_pct=0.846375 moved_to_echo=15635 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3251, 1, 0, 270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3670, 0, 0, 0, 0, 0, 844, 148, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2453, 2641, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 9, 1496, 746, 696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 825, 3050, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 300, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3522, 4664, 4094, 4085, 5197, 2948, 3888, 4370] avg=4096.0000 max=5197 max_rank=4 min=2948 min_rank=5 imbalance_ratio=1.268799 max_load_reduction=27571 max_load_reduction_pct=0.841400 moved_to_echo=16403 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3222, 0, 0, 1780, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2962, 0, 0, 1, 1, 1, 926, 146, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2138, 2899, 0, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 295, 1250, 743, 690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 3035, 13, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 0, 3567, 299, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5002, 4039, 4111, 4120, 5048, 2985, 3592, 3871] avg=4096.0000 max=5048 max_rank=4 min=2985 min_rank=5 imbalance_ratio=1.232422 max_load_reduction=27720 max_load_reduction_pct=0.845947 moved_to_echo=15496 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 3, 3209, 41, 2, 1138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 511, 146, 2, 0, 0, 0, 0, 0, 2, 0, 12, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 37, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2114, 1600, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 1681, 744, 687, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2, 0, 0, 0, 1211, 3042, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3975, 300, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4395, 4725, 4106, 4118, 3746, 3129, 4272, 4277] avg=4096.0000 max=4725 max_rank=1 min=3129 min_rank=5 imbalance_ratio=1.153564 max_load_reduction=28043 max_load_reduction_pct=0.855804 moved_to_echo=15424 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3250, 30, 0, 1208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3889, 0, 0, 0, 0, 0, 582, 148, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1782, 1821, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 435, 877, 746, 696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2150, 3050, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3513, 300, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4488, 4621, 4094, 4084, 3620, 2834, 5214, 3813] avg=4096.0000 max=5214 max_rank=6 min=2834 min_rank=5 imbalance_ratio=1.272949 max_load_reduction=27554 max_load_reduction_pct=0.840881 moved_to_echo=15481 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 3218, 13, 0, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 101, 0, 0, 0, 0, 4, 742, 150, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 5, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 609, 2308, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1642, 5, 0, 9, 406, 4017, 748, 697, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 1944, 3046, 12, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 0, 0, 4075, 292, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3741, 1009, 4088, 4076, 2944, 7524, 5014, 4372] avg=4096.0000 max=7524 max_rank=5 min=1009 min_rank=1 imbalance_ratio=1.836914 max_load_reduction=25244 max_load_reduction_pct=0.770386 moved_to_echo=19854 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 1792, 128, 128, 128, 1536, 18048, 6016, 5632, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 25984, 128, 128, 6784, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 16640, 15872, 384, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 128, 0, 0, 8320, 24448, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 26496, 0, 0, 128, 128, 128, 5120, 1280, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 128, 0, 30464, 2432, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 128, 0, 128, 128, 32640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 3713, 0, 0, 4096, 0, 0, 0, 0, 0, 997, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 3787, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 3772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 3967, 0, 2, 4094, 0, 0, 0, 0, 0, 3707, 0, 0, 0, 155, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4090, 0, 0, 0, 0, 0, 2, 0, 12, 0, 0, 0, 0, 352, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 23, 3, 4093, 5, 1, 4092, 0, 7, 0, 0, 1, 906, 0, 0, 0, 0, 2, 0, 4086, 0, 0, 0, 4093, 0, 1, 4092, 0, 0, 4, 0, 1, 0, 0, 4066, 0, 4091, 3, 0, 0, 0, 0, 0, 60, 8, 0, 0, 0, 0, 3126, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4053, 0, 0, 4096, 0, 0, 0, 0, 0, 1086, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0, 0, 2, 0, 0, 1438, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4066, 0, 0, 4075, 7, 33, 37, 0, 0, 4, 0, 0, 3, 15, 0, 0, 4095, 0, 1, 0, 4095, 0, 0, 4066, 2, 6, 0, 0, 0, 0, 8, 4044, 0, 4083, 0, 0, 0, 16, 1, 0, 59, 0, 0, 0, 0, 0, 4039, 0, 0, 4, 0, 0, 1, 0, 8, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4049, 0, 0, 4076, 2, 0, 4, 0, 0, 470, 0, 0, 0, 61, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 4077, 0, 4092, 0, 0, 4, 9, 0, 0, 0, 0, 0, 0, 0, 0, 3632, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 3913, 0, 0, 4080, 0, 0, 0, 0, 6, 35, 0, 14, 0, 45, 2, 0, 4084, 0, 0, 0, 4045, 0, 6, 4088, 0, 48, 13, 0, 4, 0, 0, 4051, 42, 4027, 0, 7, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 4, 0, 0, 0, 1, 9, 0, 0, 0, 14, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3713, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 997, 0, 0, 0, 3008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 3787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 4068, 16, 1, 0, 0, 0, 0, 0, 0, 3772, 0, 0, 0, 0, 0, 0, 974, 28, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3811, 4005, 4068, 4091, 3810, 4109, 4779, 4095] avg=4096.0000 max=4779 max_rank=6 min=3810 min_rank=4 imbalance_ratio=1.166748 max_load_reduction=27989 max_load_reduction_pct=0.854156 moved_to_echo=16793 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3967, 0, 2, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3707, 0, 0, 0, 3006, 0, 0, 0, 0, 0, 0, 0, 155, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 12, 0, 0, 0, 4073, 16, 2, 0, 0, 0, 0, 0, 0, 352, 0, 0, 0, 0, 0, 3, 974, 28, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 6713, 4223, 4091, 4113, 4105, 1362, 4094] avg=4096.0000 max=6713 max_rank=1 min=1362 min_rank=6 imbalance_ratio=1.638916 max_load_reduction=26055 max_load_reduction_pct=0.795135 moved_to_echo=13674 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 23, 3, 4093, 5, 1, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 1, 906, 0, 0, 0, 3005, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4087, 0, 0, 4, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 4091, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 8, 0, 0, 0, 4043, 16, 1, 0, 0, 0, 0, 0, 0, 3126, 0, 0, 0, 1, 0, 0, 973, 28, 5, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4223, 3919, 4067, 4093, 4117, 4128, 4133, 4088] avg=4096.0000 max=4223 max_rank=0 min=3919 min_rank=1 imbalance_ratio=1.031006 max_load_reduction=28545 max_load_reduction_pct=0.871124 moved_to_echo=16466 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1438, 0, 0, 0, 3008, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4072, 16, 1, 0, 0, 0, 0, 0, 0, 2657, 0, 0, 0, 0, 0, 0, 974, 28, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4194, 4448, 4070, 4091, 4115, 4091, 3664, 4095] avg=4096.0000 max=4448 max_rank=1 min=3664 min_rank=6 imbalance_ratio=1.085938 max_load_reduction=28320 max_load_reduction_pct=0.864258 moved_to_echo=15965 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4066, 0, 0, 98, 7, 0, 0, 0, 0, 0, 0, 0, 0, 33, 37, 0, 0, 4, 0, 0, 3, 2992, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 1, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 2, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 23, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 1, 0, 59, 0, 0, 0, 0, 4021, 16, 2, 0, 0, 0, 0, 0, 0, 4039, 0, 0, 4, 0, 0, 1, 969, 28, 5, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4171, 3069, 4083, 4069, 4114, 4115, 5046, 4101] avg=4096.0000 max=5046 max_rank=6 min=3069 min_rank=1 imbalance_ratio=1.231934 max_load_reduction=27722 max_load_reduction_pct=0.846008 moved_to_echo=17376 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4049, 0, 0, 98, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 470, 0, 0, 0, 2993, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 4092, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 4054, 16, 1, 0, 0, 0, 0, 0, 0, 3632, 0, 0, 0, 0, 2, 0, 969, 28, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4149, 3467, 4129, 4094, 4119, 4080, 4636, 4094] avg=4096.0000 max=4636 max_rank=6 min=3467 min_rank=1 imbalance_ratio=1.131836 max_load_reduction=28132 max_load_reduction_pct=0.858521 moved_to_echo=16929 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4053, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1086, 0, 0, 0, 3008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4073, 16, 1, 0, 0, 0, 0, 0, 0, 3054, 0, 0, 0, 0, 0, 0, 974, 28, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4151, 4094, 4068, 4091, 4117, 4091, 4061, 4095] avg=4096.0000 max=4151 max_rank=0 min=4061 min_rank=6 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16364 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3913, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 35, 0, 14, 0, 2998, 0, 0, 0, 0, 0, 0, 0, 45, 2, 0, 0, 0, 0, 0, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4077, 0, 48, 13, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 42, 4027, 0, 7, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4030, 17, 2, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 4, 0, 0, 976, 32, 11, 0, 0, 0, 0, 0, 0, 1, 9, 0, 0, 0, 14, 0, 4082, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4002, 3053, 4060, 4148, 4263, 4049, 5087, 4106] avg=4096.0000 max=5087 max_rank=6 min=3053 min_rank=1 imbalance_ratio=1.241943 max_load_reduction=27681 max_load_reduction_pct=0.844757 moved_to_echo=17505 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 8704, 0, 128, 128, 24064, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 24704, 0, 0, 128, 128, 128, 128, 7808, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 128, 32000, 128, 128, 896, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 0, 0, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 256, 128, 32384, 128, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 256, 128, 0, 0, 0, 32512, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 32768, 128, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[384, 128, 0, 0, 0, 128, 0, 32512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4093, 3, 0, 47, 12, 0, 0, 0, 4096, 0, 0, 479, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 4080, 0, 0, 0, 0, 3587, 0, 7, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 529, 0, 0, 0, 0, 4096, 0, 0, 288, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 4096, 0, 0, 0, 0, 3315, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 106, 0, 0, 0, 0, 4096, 0, 0, 70, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4007, 0, 4090, 0, 0, 0, 0, 4017, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 4, 0, 0, 0, 0, 0, 36, 1, 0, 4093, 0, 0, 4092, 0, 3, 387, 0, 0, 0, 0, 4091, 0, 4, 238, 0, 0, 0, 0, 4053, 6, 0, 0, 0, 1, 0, 3, 0, 3974, 0, 4085, 0, 0, 0, 0, 3620, 0, 0, 1, 2, 0, 0, 2, 4062, 0, 0, 0, 1, 2, 2, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4093, 6, 0, 281, 0, 0, 0, 0, 4096, 0, 0, 237, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4063, 0, 0, 0, 0, 3591, 0, 15, 0, 0, 0, 0, 8, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 125, 0, 0, 0, 1, 4091, 0, 0, 95, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 3976, 0, 4094, 0, 0, 0, 0, 3996, 0, 0, 0, 0, 0, 0, 10, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 4043, 0, 3, 4081, 0, 7, 25, 0, 0, 0, 0, 4069, 51, 4, 4, 3, 0, 0, 0, 3985, 18, 0, 0, 0, 0, 0, 0, 99, 4083, 0, 4082, 0, 9, 0, 0, 4079, 0, 4, 21, 9, 0, 0, 3, 4017, 0, 7, 0, 0, 13, 9, 12] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4082, 0, 0, 83, 21, 0, 0, 0, 4092, 0, 0, 37, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 4071, 1, 0, 0, 0, 3990, 0, 2, 0, 0, 0, 0, 15, 4092, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 4025, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3895, 6, 0, 281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3913, 0, 0, 237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3591, 0, 15, 0, 0, 0, 0, 8, 158, 150, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 7, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4094, 4182, 4150, 4114, 4128, 3922, 4104] avg=4096.0000 max=4182 max_rank=2 min=3922 min_rank=6 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16268 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4058, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3898, 0, 0, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3913, 0, 0, 288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3315, 0, 0, 0, 0, 0, 0, 0, 158, 150, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 7, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4095, 4427, 4201, 4112, 4100, 3623, 4104] avg=4096.0000 max=4427 max_rank=2 min=3623 min_rank=6 imbalance_ratio=1.080811 max_load_reduction=28341 max_load_reduction_pct=0.864899 moved_to_echo=15939 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 4042, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3895, 3, 0, 47, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3913, 0, 0, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3587, 0, 7, 0, 0, 0, 0, 0, 158, 150, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 7, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4095, 3957, 4392, 4105, 4122, 3902, 4104] avg=4096.0000 max=4392 max_rank=3 min=3902 min_rank=6 imbalance_ratio=1.072266 max_load_reduction=28376 max_load_reduction_pct=0.865967 moved_to_echo=16233 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 4, 0, 0, 0, 4047, 48, 0, 0, 0, 0, 0, 0, 0, 0, 36, 1, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3894, 0, 3, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3908, 0, 4, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 6, 0, 0, 0, 1, 0, 3, 18, 0, 0, 0, 0, 0, 0, 0, 0, 3974, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3620, 0, 0, 1, 2, 0, 0, 2, 158, 150, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 1, 2, 2, 3, 7, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4129, 4284, 4150, 4081, 4012, 3933, 4078] avg=4096.0000 max=4284 max_rank=2 min=3933 min_rank=6 imbalance_ratio=1.045898 max_load_reduction=28484 max_load_reduction_pct=0.869263 moved_to_echo=16104 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4052, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3898, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3913, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 4007, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4017, 0, 0, 0, 0, 0, 0, 2, 158, 150, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 7, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4095, 4004, 3983, 4110, 4045, 4327, 4104] avg=4096.0000 max=4327 max_rank=6 min=3983 min_rank=3 imbalance_ratio=1.056396 max_load_reduction=28441 max_load_reduction_pct=0.867950 moved_to_echo=16586 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4056, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3898, 0, 0, 125, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3908, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 3976, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3996, 0, 0, 0, 0, 0, 0, 10, 158, 150, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 7, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4095, 4024, 4003, 4110, 4014, 4314, 4104] avg=4096.0000 max=4314 max_rank=6 min=4003 min_rank=3 imbalance_ratio=1.053223 max_load_reduction=28454 max_load_reduction_pct=0.868347 moved_to_echo=16542 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 5, 0, 0, 4033, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3884, 0, 0, 83, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3909, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 38, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3990, 0, 2, 0, 0, 0, 0, 15, 158, 150, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 7, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4094, 3988, 3946, 4114, 4124, 4315, 4100] avg=4096.0000 max=4315 max_rank=6 min=3946 min_rank=3 imbalance_ratio=1.053467 max_load_reduction=28453 max_load_reduction_pct=0.868317 moved_to_echo=16653 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4050, 55, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 4040, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3870, 0, 7, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3881, 51, 4, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3985, 18, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 99, 4083, 0, 32, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 4, 21, 9, 0, 0, 3, 160, 156, 0, 0, 0, 0, 0, 0, 4017, 0, 7, 0, 0, 13, 9, 12, 9, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4071, 3902, 3943, 4022, 4223, 4432, 4070] avg=4096.0000 max=4432 max_rank=6 min=3902 min_rank=2 imbalance_ratio=1.082031 max_load_reduction=28336 max_load_reduction_pct=0.864746 moved_to_echo=16747 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[31232, 128, 128, 1664, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 0, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[30208, 0, 128, 128, 128, 0, 0, 128, 1280, 1280, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[31360, 128, 128, 1536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 128, 128, 128, 0, 32384, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[32768, 0, 128, 0, 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32384, 0, 384, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32640, 128, 0, 0, 0, 128, 0, 128, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 119, 1, 0, 0, 4069, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 4096, 4000, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 4095, 0, 0, 0, 4095, 0, 0, 0, 0, 12, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 4093, 0, 0, 6, 4085, 0, 0, 4, 9, 0, 4, 3, 0, 0, 0, 10, 0, 26, 0, 4096, 4066, 4090, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 4088, 0, 4084, 0, 0, 0, 4084, 0, 0, 0, 6, 3, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 7, 1, 0, 0, 4061, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 1, 0, 0, 4095, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4078, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 712, 3, 0, 0, 3387, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 29, 0, 0, 4079, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 5, 0, 0, 0, 4081, 0, 4063, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 118, 0, 0, 0, 3839, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 21, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 305, 0, 0, 0, 3789, 0, 0, 0, 4096, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 4080, 4096, 4096, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 4082, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4087, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 41, 0, 2, 0, 4038, 0, 17, 44, 4062, 0, 0, 0, 0, 0, 12, 0, 0, 0, 5, 13, 0, 0, 0, 3937, 4083, 4054, 0, 7, 0, 0, 16, 1, 0, 0, 56, 0, 5, 1, 0, 0, 8, 12, 11, 36, 0, 0, 2, 4045, 0, 4078, 0, 0, 0, 4055, 0, 0, 6, 42, 0, 34, 10, 0, 35] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 4093, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 6, 4080, 0, 0, 4, 9, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 10, 0, 26, 0, 4066, 17, 0, 0, 0, 0, 0, 0, 4096, 0, 22, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 4052, 19, 0, 0, 0, 0, 0, 0, 4088, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 0, 1, 0, 0, 4065, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4103, 4122, 4120, 4061, 4076, 4103, 4081] avg=4096.0000 max=4122 max_rank=2 min=4061 min_rank=4 imbalance_ratio=1.006348 max_load_reduction=28646 max_load_reduction_pct=0.874207 moved_to_echo=16321 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 118, 0, 0, 0, 3839, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4071, 17, 0, 0, 0, 0, 0, 0, 4096, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 0, 0, 0, 4064, 20, 0, 0, 0, 0, 0, 0, 4096, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 4076, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3964, 4091, 4089, 4118, 4091, 4202, 4111, 4102] avg=4096.0000 max=4202 max_rank=5 min=3964 min_rank=0 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16506 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 712, 3, 0, 0, 3387, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 29, 0, 0, 4065, 17, 0, 0, 0, 0, 0, 0, 4079, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4031, 20, 0, 0, 0, 0, 0, 0, 4081, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4091, 4124, 4101, 4110, 4056, 4096, 4081] avg=4096.0000 max=4124 max_rank=2 min=4056 min_rank=5 imbalance_ratio=1.006836 max_load_reduction=28644 max_load_reduction_pct=0.874146 moved_to_echo=16343 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 119, 1, 0, 0, 4069, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 1, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 3976, 17, 0, 0, 0, 0, 0, 0, 4096, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 20, 0, 0, 0, 0, 0, 0, 4083, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 4075, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4196, 4092, 3997, 4118, 4092, 4083, 4098, 4092] avg=4096.0000 max=4196 max_rank=0 min=3997 min_rank=2 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16365 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 305, 0, 0, 0, 3789, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 4091, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 4072, 17, 0, 0, 0, 0, 0, 0, 4080, 0, 22, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 20, 0, 0, 0, 0, 0, 0, 4069, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4076, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4101, 4134, 4104, 4092, 4070, 4084, 4082] avg=4096.0000 max=4134 max_rank=2 min=4070 min_rank=5 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16328 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 1, 0, 0, 4061, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 1, 0, 0, 4072, 17, 0, 0, 0, 0, 0, 0, 4095, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 20, 0, 0, 0, 0, 0, 0, 4096, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4091, 4135, 4117, 4090, 4066, 4111, 4082] avg=4096.0000 max=4135 max_rank=2 min=4066 min_rank=5 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16349 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 4096, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4072, 17, 0, 0, 0, 0, 0, 0, 4096, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 20, 0, 0, 0, 0, 0, 0, 4096, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4091, 4097, 4118, 4091, 4075, 4111, 4081] avg=4096.0000 max=4118 max_rank=3 min=4075 min_rank=5 imbalance_ratio=1.005371 max_load_reduction=28650 max_load_reduction_pct=0.874329 moved_to_echo=16358 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 41, 0, 2, 0, 4038, 0, 17, 10, 8, 0, 0, 0, 0, 0, 0, 44, 4052, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 13, 0, 0, 0, 4034, 18, 0, 0, 0, 0, 0, 0, 3937, 0, 12, 0, 7, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 56, 0, 5, 1, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 8, 12, 11, 36, 0, 0, 2, 4051, 20, 0, 0, 0, 0, 0, 0, 4045, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 42, 0, 34, 10, 0, 35, 4035, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4108, 4070, 3972, 4141, 4140, 4054, 4167] avg=4096.0000 max=4167 max_rank=7 min=3972 min_rank=3 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16502 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32640, 0, 256, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 1408, 128, 128, 0, 31488, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 128, 128, 128, 0, 32512, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 128, 128, 0, 128, 32640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 256, 0, 0, 128, 32512, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 0, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=32896 slack_slots=2816 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3982, 0, 0, 0, 0, 3595, 0, 0, 0, 2, 0, 5, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 22, 0, 0, 4095, 3848, 0, 0, 0, 0, 3989, 0, 0, 0, 1, 0, 294, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 33, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4058, 0, 4091, 4092, 0, 0, 4091, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4093, 3964, 2, 0, 0, 0, 3756, 0, 0, 3, 9, 21, 4, 82, 1, 0, 2, 0, 0, 0, 0, 0, 6, 7, 0, 4091, 2, 2, 0, 0, 3, 0, 2, 1, 0, 271, 1, 0, 0, 0, 4, 0, 107, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 4096, 3569, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4032, 0, 0, 0, 0, 3102, 0, 2, 0, 0, 0, 711, 775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4053, 0, 4096, 4095, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 9, 0, 0, 4084, 3980, 0, 0, 0, 0, 3951, 0, 0, 0, 42, 0, 36, 247, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[78, 0, 0, 0, 4095, 0, 4096, 3872, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4015, 0, 0, 0, 0, 3463, 0, 0, 0, 0, 0, 155, 374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 4037, 0, 3, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0, 300, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3728, 0, 0, 0, 0, 4026, 0, 0, 0, 10, 0, 163, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4044, 0, 4080, 3973, 0, 0, 4079, 16, 3, 11, 0, 0, 0, 127, 8, 0, 4081, 3516, 38, 0, 0, 0, 3497, 0, 0, 9, 0, 0, 1037, 1, 0, 0, 0, 0, 0, 7, 0, 0, 0, 19, 0, 4088, 10, 47, 4, 6, 14, 0, 15, 0, 0, 1, 0, 10, 0, 11, 0, 8, 0, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 102, 0, 0, 3569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3102, 0, 2, 0, 0, 0, 411, 0, 0, 0, 0, 0, 0, 0, 711, 775, 0, 0, 0, 0, 0, 0, 3495, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 0, 3955, 48, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3671, 4095, 4221, 3515, 5020, 4061, 4048, 4137] avg=4096.0000 max=5020 max_rank=4 min=3515 min_rank=3 imbalance_ratio=1.225586 max_load_reduction=27748 max_load_reduction_pct=0.846802 moved_to_echo=17266 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 102, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 189, 3848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3989, 0, 0, 0, 1, 0, 411, 0, 0, 0, 0, 0, 0, 0, 294, 8, 0, 0, 0, 0, 0, 0, 3494, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 3954, 48, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4198, 4096, 4059, 4401, 3835, 4095, 4049, 4035] avg=4096.0000 max=4401 max_rank=3 min=3835 min_rank=4 imbalance_ratio=1.074463 max_load_reduction=28367 max_load_reduction_pct=0.865692 moved_to_echo=16014 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 102, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 3982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3595, 0, 0, 0, 2, 0, 411, 0, 0, 0, 0, 0, 0, 0, 5, 462, 0, 0, 0, 0, 0, 0, 3494, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 0, 3955, 49, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4198, 4095, 4171, 4008, 4000, 4097, 4047, 4152] avg=4096.0000 max=4198 max_rank=0 min=4000 min_rank=4 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16296 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 100, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 3964, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3756, 0, 0, 3, 9, 21, 411, 0, 0, 0, 0, 0, 0, 0, 4, 82, 1, 0, 2, 0, 0, 0, 3492, 39, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 0, 4091, 2, 2, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 1, 0, 271, 4042, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 107, 0, 3919, 49, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4192, 4092, 4155, 4200, 3620, 4110, 4319, 4080] avg=4096.0000 max=4319 max_rank=6 min=3620 min_rank=4 imbalance_ratio=1.054443 max_load_reduction=28449 max_load_reduction_pct=0.868195 moved_to_echo=16129 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 100, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 189, 3980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3951, 0, 0, 0, 42, 0, 410, 0, 0, 0, 0, 0, 0, 0, 36, 247, 0, 0, 0, 0, 0, 0, 3484, 39, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4077, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3914, 48, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4195, 4096, 4178, 4403, 3806, 4080, 4048, 3962] avg=4096.0000 max=4403 max_rank=3 min=3806 min_rank=4 imbalance_ratio=1.074951 max_load_reduction=28365 max_load_reduction_pct=0.865631 moved_to_echo=15896 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[78, 0, 0, 0, 102, 0, 0, 3872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3463, 0, 0, 0, 0, 0, 410, 0, 0, 0, 0, 0, 0, 0, 155, 374, 0, 0, 0, 0, 0, 0, 3485, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 4037, 0, 3, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 0, 3954, 48, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4052, 4095, 4204, 3873, 4053, 4053, 4136, 4302] avg=4096.0000 max=4302 max_rank=7 min=3873 min_rank=3 imbalance_ratio=1.050293 max_load_reduction=28466 max_load_reduction_pct=0.868713 moved_to_echo=16544 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 102, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 3728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 0, 0, 0, 10, 0, 411, 0, 0, 0, 0, 0, 0, 0, 163, 163, 0, 0, 0, 0, 0, 0, 3495, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 3953, 48, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4198, 4095, 3917, 4447, 3860, 4097, 4048, 4106] avg=4096.0000 max=4447 max_rank=3 min=3860 min_rank=4 imbalance_ratio=1.085693 max_load_reduction=28321 max_load_reduction_pct=0.864288 moved_to_echo=16111 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 91, 0, 0, 3973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 16, 3, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 8, 0, 174, 3516, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3497, 0, 0, 9, 0, 0, 415, 0, 0, 0, 0, 0, 0, 0, 1037, 1, 0, 0, 0, 0, 0, 7, 3486, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 4088, 10, 47, 6, 5, 0, 0, 0, 0, 0, 0, 4, 6, 14, 0, 15, 0, 0, 1, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 11, 0, 8, 0, 8, 3910, 47, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4064, 4104, 3863, 3921, 4574, 4175, 4073, 3994] avg=4096.0000 max=4574 max_rank=4 min=3863 min_rank=2 imbalance_ratio=1.116699 max_load_reduction=28194 max_load_reduction_pct=0.860413 moved_to_echo=16816 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[2432, 2176, 128, 0, 128, 0, 0, 128, 28032, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 0, 896, 0, 0, 32000, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 256, 128, 0, 1536, 31104, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 128, 128, 0, 384, 32384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 0, 32640, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 29440, 0, 128, 128, 128, 128, 3328, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 128, 128, 128, 896, 128, 31616, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33792 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 100, 0, 302, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 2, 0, 3740, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 83, 2, 328, 0, 2, 0, 0, 0, 4055, 1, 2, 1, 0, 0, 3, 0, 4080, 0, 4086, 29, 0, 64, 0, 0, 4082, 0, 3, 0, 1, 0, 8, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 0, 20, 0, 4076, 9, 5, 4089, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3966, 0, 0, 0, 3765, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[12, 14, 0, 254, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 3896, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4011, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 164, 0, 36, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 1, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2, 0, 3919, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[16, 60, 0, 664, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 3442, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4076, 0, 0, 4094, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 4025, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 19, 2, 2846, 0, 0, 0, 0, 12, 3942, 0, 0, 0, 0, 0, 13, 0, 4066, 0, 4041, 191, 0, 17, 26, 4, 1610, 0, 9, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 88, 0, 11, 0, 0, 0, 0, 3550, 23, 0, 4069, 0, 0, 0, 0, 0, 0, 19, 0, 0, 4076, 0, 0, 0, 4092, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 51, 0, 987, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 3086, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 8, 0, 4066, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 6, 0, 332, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 3864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4093, 0, 13, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3980, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 83, 2, 328, 0, 2, 0, 0, 3210, 11, 0, 0, 0, 0, 0, 0, 0, 4055, 1, 2, 1, 0, 0, 3, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 29, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 3, 0, 1, 0, 8, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 20, 0, 4076, 9, 5, 54, 11, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 0, 0, 0, 3765, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3636, 4082, 4148, 4654, 4072, 4177, 4092, 3907] avg=4096.0000 max=4654 max_rank=3 min=3636 min_rank=0 imbalance_ratio=1.136230 max_load_reduction=28114 max_load_reduction_pct=0.857971 moved_to_echo=16248 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[12, 14, 0, 254, 0, 0, 0, 0, 3315, 12, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3896, 0, 0, 0, 0, 0, 0, 578, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 56, 11, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 0, 0, 0, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3607, 4117, 4064, 4474, 4089, 4163, 4096, 4158] avg=4096.0000 max=4474 max_rank=3 min=3607 min_rank=0 imbalance_ratio=1.092285 max_load_reduction=28294 max_load_reduction_pct=0.863464 moved_to_echo=16506 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 100, 0, 302, 0, 0, 0, 0, 3312, 11, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 56, 11, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 0, 2, 0, 3740, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3725, 4116, 4064, 4654, 4085, 4139, 4096, 3889] avg=4096.0000 max=4654 max_rank=3 min=3725 min_rank=0 imbalance_ratio=1.136230 max_load_reduction=28114 max_load_reduction_pct=0.857971 moved_to_echo=16209 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 51, 0, 987, 0, 0, 0, 0, 3312, 11, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3086, 0, 0, 4, 0, 0, 0, 578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 56, 11, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 0, 8, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4361, 4117, 4064, 3668, 4085, 4156, 4096, 4221] avg=4096.0000 max=4361 max_rank=0 min=3668 min_rank=3 imbalance_ratio=1.064697 max_load_reduction=28407 max_load_reduction_pct=0.866913 moved_to_echo=16558 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 164, 0, 36, 0, 0, 0, 0, 3314, 11, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 56, 11, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 0, 2, 0, 3919, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3525, 4116, 4065, 4653, 4085, 4160, 4096, 4068] avg=4096.0000 max=4653 max_rank=3 min=3525 min_rank=0 imbalance_ratio=1.135986 max_load_reduction=28115 max_load_reduction_pct=0.858002 moved_to_echo=16409 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 0, 332, 0, 0, 0, 0, 3315, 12, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3864, 0, 0, 0, 0, 0, 0, 578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 56, 11, 0, 0, 0, 0, 0, 0, 4093, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 0, 0, 0, 3980, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3665, 4117, 4064, 4442, 4084, 4163, 4106, 4127] avg=4096.0000 max=4442 max_rank=3 min=3665 min_rank=0 imbalance_ratio=1.084473 max_load_reduction=28326 max_load_reduction_pct=0.864441 moved_to_echo=16480 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[16, 60, 0, 664, 0, 0, 0, 0, 3315, 12, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3442, 0, 0, 1, 0, 0, 0, 578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4076, 0, 0, 56, 11, 0, 0, 0, 0, 0, 0, 4094, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 0, 1, 0, 4025, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4112, 4064, 4021, 4084, 4147, 4100, 4173] avg=4096.0000 max=4173 max_rank=7 min=4021 min_rank=3 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16504 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 19, 2, 2846, 0, 0, 0, 0, 3303, 12, 0, 0, 0, 0, 0, 0, 12, 3942, 0, 0, 0, 0, 0, 13, 22, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4001, 191, 0, 17, 26, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1610, 0, 9, 0, 0, 0, 0, 579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 88, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 3550, 23, 0, 61, 18, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 133, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6182, 3991, 4235, 2202, 4184, 3663, 4086, 4225] avg=4096.0000 max=6182 max_rank=0 min=2202 min_rank=3 imbalance_ratio=1.509277 max_load_reduction=26586 max_load_reduction_pct=0.811340 moved_to_echo=16158 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 512, 128, 5760, 0, 128, 0, 0, 26496, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 28160, 0, 128, 128, 128, 0, 128, 4608, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 0, 128, 0, 32256, 128, 128, 512, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 1280, 0, 128, 0, 31616, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 0, 0, 0, 128, 0, 32640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 32512, 256, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 32640, 128, 128, 128, 0, 0, 128, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 30, 0, 0, 0, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 27, 0, 0, 38, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4070, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 30, 0, 0, 0, 4009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 60, 0, 0, 4096, 0, 0, 92, 0, 0, 4096, 4059, 0, 0, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[3, 6, 0, 2863, 0, 1, 21, 2889, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 29, 0, 4093, 0, 2, 4093, 0, 0, 0, 2, 2, 0, 4034, 0, 1, 0, 94, 3, 3, 9, 0, 5, 4010, 0, 0, 0, 0, 0, 4092, 2421, 0, 0, 0, 1, 4087, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 254, 0, 0, 0, 3964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4089, 101, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 15, 0, 0, 4091, 0, 0, 0, 0, 2, 4096, 3876, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 981, 0, 0, 0, 3231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 34, 0, 0, 25, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 3921, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 676, 0, 0, 0, 3520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 51, 0, 0, 603, 0, 0, 4064, 0, 0, 0, 0, 4, 4096, 3387, 0, 0, 0, 0, 4080, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[14, 2, 0, 310, 0, 0, 41, 3962, 1, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 4, 10, 0, 4077, 4, 1, 4039, 0, 0, 0, 35, 9, 67, 4054, 1, 6, 10, 255, 12, 15, 4, 0, 0, 3357, 0, 4, 231, 0, 0, 4082, 4028, 0, 0, 0, 0, 4055, 0, 0, 0, 47, 0, 0, 0, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 1615, 0, 0, 0, 3398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 20, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1213, 0, 0, 4014, 0, 0, 15, 0, 0, 4091, 2010, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 30, 0, 0, 0, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 3920, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 0, 0, 620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3432, 425, 176, 59, 11, 3, 1, 0] rerouted_tokens_per_ep_rank=[3614, 4095, 4093, 4071, 4037, 4072, 4679, 4107] avg=4096.0000 max=4679 max_rank=6 min=3614 min_rank=0 imbalance_ratio=1.142334 max_load_reduction=28089 max_load_reduction_pct=0.857208 moved_to_echo=16895 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 254, 0, 0, 0, 3544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 3916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3876, 0, 0, 0, 0, 625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3464, 420, 175, 59, 11, 3, 1, 0] rerouted_tokens_per_ep_rank=[3798, 4095, 4093, 4179, 4036, 3931, 4503, 4133] avg=4096.0000 max=4503 max_rank=6 min=3798 min_rank=0 imbalance_ratio=1.099365 max_load_reduction=28265 max_load_reduction_pct=0.862579 moved_to_echo=16603 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3, 6, 0, 2863, 0, 1, 21, 2583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4082, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3976, 0, 1, 0, 94, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 5, 3838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2421, 0, 0, 0, 1, 625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3462, 306, 172, 58, 11, 3, 1, 0] rerouted_tokens_per_ep_rank=[5477, 4095, 4119, 4088, 4077, 3852, 3047, 4013] avg=4096.0000 max=5477 max_rank=0 min=3047 min_rank=6 imbalance_ratio=1.337158 max_load_reduction=27291 max_load_reduction_pct=0.832855 moved_to_echo=14989 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 30, 0, 0, 0, 3599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 3920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3470, 427, 176, 59, 11, 3, 1, 0] rerouted_tokens_per_ep_rank=[3629, 4095, 4094, 4085, 4064, 3958, 4696, 4147] avg=4096.0000 max=4696 max_rank=6 min=3629 min_rank=0 imbalance_ratio=1.146484 max_load_reduction=28072 max_load_reduction_pct=0.856689 moved_to_echo=16865 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 676, 0, 0, 0, 3147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 603, 0, 0, 3890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3387, 0, 0, 0, 0, 624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3456, 373, 174, 59, 11, 3, 1, 0] rerouted_tokens_per_ep_rank=[3823, 4095, 4093, 4083, 4088, 4493, 4015, 4078] avg=4096.0000 max=4493 max_rank=5 min=3823 min_rank=0 imbalance_ratio=1.096924 max_load_reduction=28275 max_load_reduction_pct=0.862885 moved_to_echo=16674 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 981, 0, 0, 0, 2889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 3919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3921, 0, 0, 0, 0, 626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3470, 342, 176, 59, 11, 3, 1, 0] rerouted_tokens_per_ep_rank=[3870, 4095, 4094, 4085, 4071, 3944, 4547, 4062] avg=4096.0000 max=4547 max_rank=6 min=3870 min_rank=0 imbalance_ratio=1.110107 max_load_reduction=28221 max_load_reduction_pct=0.861237 moved_to_echo=16624 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1615, 0, 0, 0, 3038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1213, 0, 0, 3842, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2010, 0, 0, 0, 0, 626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3470, 360, 172, 59, 11, 3, 1, 0] rerouted_tokens_per_ep_rank=[4653, 4090, 4095, 4103, 4037, 5070, 2636, 4084] avg=4096.0000 max=5070 max_rank=5 min=2636 min_rank=6 imbalance_ratio=1.237793 max_load_reduction=27698 max_load_reduction_pct=0.845276 moved_to_echo=15827 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[14, 2, 0, 310, 0, 0, 41, 3537, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 25, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 10, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 4025, 0, 0, 0, 35, 9, 0, 0, 0, 0, 0, 0, 0, 0, 67, 3992, 1, 6, 10, 255, 12, 15, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 3209, 0, 4, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4028, 0, 0, 0, 0, 617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 6, 3438, 425, 148, 62, 14, 4, 0, 0] rerouted_tokens_per_ep_rank=[3904, 4108, 4087, 4074, 4358, 3448, 4645, 4144] avg=4096.0000 max=4645 max_rank=6 min=3448 min_rank=5 imbalance_ratio=1.134033 max_load_reduction=28123 max_load_reduction_pct=0.858246 moved_to_echo=16595 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 128, 27776, 3200, 1408, 512, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 6784, 0, 128, 128, 25984, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 27776, 0, 0, 0, 128, 4992, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 32640, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2048, 0, 128, 30464, 0, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 32256, 128, 128, 128, 512, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 0, 0, 128, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 0, 4050, 0, 0, 0, 113, 0, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 4089, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 4096, 11, 0, 4096, 0, 2929, 0, 0, 0, 1825, 0, 3440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4081, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2, 4078, 2, 0, 4089, 0, 4076, 0, 0, 0, 234, 0, 3873, 0, 3, 0, 0, 0, 2, 0, 0, 0, 3, 0, 17, 4090, 0, 0, 3, 0, 4091, 0, 1, 0, 0, 0, 0, 0, 4093, 4, 4, 0, 0, 4, 0, 0, 0, 4084, 0, 3, 2, 6, 2, 2, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 2807, 0, 0, 0, 1341, 0, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 4096, 0, 3240, 0, 0, 0, 944, 0, 3917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 4096, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 2328, 0, 0, 0, 2095, 0, 3784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 16, 0, 19, 13, 3987, 0, 1, 4075, 0, 1671, 0, 0, 0, 2802, 0, 3692, 7, 4, 2, 0, 0, 0, 0, 0, 0, 14, 0, 66, 4077, 0, 0, 24, 50, 4082, 0, 0, 0, 0, 1, 0, 0, 4080, 1, 12, 0, 0, 0, 0, 0, 0, 4010, 0, 15, 21, 0, 9, 0, 0, 16, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2, 0, 0, 4095, 0, 0, 4096, 0, 3739, 0, 1, 0, 2200, 0, 2167, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 4087, 1, 0, 0, 52, 4096, 0, 0, 0, 0, 0, 42, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 993, 1, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 0, 2954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3103, 1080, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 5044, 3067, 4193, 4081, 4090, 4109, 4091] avg=4096.0000 max=5044 max_rank=1 min=3067 min_rank=2 imbalance_ratio=1.231445 max_load_reduction=27724 max_load_reduction_pct=0.846069 moved_to_echo=16371 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 993, 11, 0, 0, 0, 2929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1825, 0, 2519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3103, 921, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 3933, 4344, 4035, 4072, 4091, 4109, 4090] avg=4096.0000 max=4344 max_rank=2 min=3933 min_rank=1 imbalance_ratio=1.060547 max_load_reduction=28424 max_load_reduction_pct=0.867432 moved_to_echo=16362 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2, 4085, 0, 0, 0, 0, 0, 0, 0, 989, 2, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 234, 0, 2836, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 0, 17, 3089, 1037, 8, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4081, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4084, 0, 7, 5, 0, 0, 0, 0, 0, 0, 3, 2, 6, 2, 2, 0, 0, 0, 4082, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 5067, 3073, 4159, 4085, 4096, 4100, 4101] avg=4096.0000 max=5067 max_rank=1 min=3073 min_rank=2 imbalance_ratio=1.237061 max_load_reduction=27701 max_load_reduction_pct=0.845367 moved_to_echo=16382 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 993, 3, 0, 0, 0, 3240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 944, 0, 2868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 3103, 1049, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4236, 3812, 4252, 4089, 4093, 4103, 4090] avg=4096.0000 max=4252 max_rank=3 min=3812 min_rank=2 imbalance_ratio=1.038086 max_load_reduction=28516 max_load_reduction_pct=0.870239 moved_to_echo=16375 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 993, 0, 0, 0, 0, 2328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2095, 0, 2771, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3101, 1013, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 3321, 4866, 4124, 4074, 4090, 4109, 4091] avg=4096.0000 max=4866 max_rank=2 min=3321 min_rank=1 imbalance_ratio=1.187988 max_load_reduction=27902 max_load_reduction_pct=0.851501 moved_to_echo=16364 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 993, 0, 0, 0, 0, 3739, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2200, 0, 1587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 3102, 580, 9, 3, 0, 0, 0, 0, 0, 1, 0, 0, 52, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4733, 3787, 3698, 4139, 4131, 4104, 4081] avg=4096.0000 max=4733 max_rank=1 min=3698 min_rank=3 imbalance_ratio=1.155518 max_load_reduction=28035 max_load_reduction_pct=0.855560 moved_to_echo=16455 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 993, 0, 0, 0, 0, 2807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1341, 0, 2962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3102, 1083, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 3800, 4303, 4196, 4086, 4090, 4109, 4092] avg=4096.0000 max=4303 max_rank=2 min=3800 min_rank=1 imbalance_ratio=1.050537 max_load_reduction=28465 max_load_reduction_pct=0.868683 moved_to_echo=16377 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 16, 0, 19, 13, 4072, 0, 0, 0, 0, 0, 0, 0, 962, 0, 1, 0, 0, 1671, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2802, 0, 2701, 7, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 66, 3025, 991, 9, 6, 0, 0, 0, 0, 0, 0, 0, 24, 50, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4073, 1, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4010, 0, 8, 7, 0, 0, 0, 0, 0, 0, 15, 21, 0, 9, 0, 0, 16, 0, 4068, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 2634, 5516, 4111, 4142, 4087, 4025, 4132] avg=4096.0000 max=5516 max_rank=2 min=2634 min_rank=1 imbalance_ratio=1.346680 max_load_reduction=27252 max_load_reduction_pct=0.831665 moved_to_echo=16386 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 0, 128, 0, 256, 24832, 7808, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[7936, 128, 128, 0, 0, 24960, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 11648, 0, 21248, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 0, 0, 32768, 0, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 0, 128, 128, 32640, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 0, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 3676, 0, 0, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0, 1, 0, 3981, 4088, 0, 273, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3450, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 626, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 16, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2848, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1362, 0, 0, 0, 0, 0, 0, 3981, 4096, 0, 0, 0, 0, 0, 0, 13, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3731, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 372, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 1, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 4092, 16, 0, 0, 0, 0, 0, 4091, 0, 7, 0, 0, 0, 4087, 0, 0, 0, 0, 4005, 7, 0, 4079, 0, 0, 1, 2, 0, 0, 4, 0, 0, 11, 0, 0, 44, 6, 3, 4, 0, 0, 0, 4093, 4079, 3, 0, 0, 4, 0, 0, 39, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4090, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 4074, 49, 0, 65, 0, 5, 16, 4071, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 2101, 2, 4, 4000, 0, 1, 12, 14, 0, 0, 3, 0, 0, 4, 17, 0, 1902, 1, 0, 24, 0, 0, 0, 3991, 4035, 12, 47, 0, 17, 0, 0, 0, 5, 54, 0, 0, 7, 0, 92, 0, 0, 0, 0, 4076, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4054, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3450, 0, 0, 603, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 626, 0, 0, 0, 0, 3485, 17, 15, 8, 4, 0, 0, 0, 0, 0, 4092, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4100, 4053, 4095, 4155, 4108, 4088, 4088] avg=4096.0000 max=4155 max_rank=4 min=4053 min_rank=2 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16439 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4081, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3676, 0, 0, 594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0, 3432, 17, 15, 8, 4, 0, 0, 0, 1, 0, 3977, 0, 0, 273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4100, 4271, 4095, 3732, 4251, 4148, 4087] avg=4096.0000 max=4271 max_rank=2 min=3732 min_rank=4 imbalance_ratio=1.042725 max_load_reduction=28497 max_load_reduction_pct=0.869659 moved_to_echo=16218 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2848, 0, 0, 598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1362, 0, 0, 0, 0, 3455, 17, 15, 8, 4, 0, 0, 0, 0, 0, 3977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 31, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4100, 3446, 4095, 4861, 3977, 4120, 4088] avg=4096.0000 max=4861 max_rank=4 min=3446 min_rank=2 imbalance_ratio=1.186768 max_load_reduction=27907 max_load_reduction_pct=0.851654 moved_to_echo=17046 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4077, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 7, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4005, 7, 0, 602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 4, 0, 0, 4087, 2, 0, 0, 0, 0, 0, 0, 11, 0, 0, 44, 6, 3, 4, 0, 3477, 16, 15, 8, 4, 0, 0, 0, 0, 0, 4089, 0, 3, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4098, 4614, 4096, 3588, 4096, 4100, 4083] avg=4096.0000 max=4614 max_rank=2 min=3588 min_rank=4 imbalance_ratio=1.126465 max_load_reduction=28154 max_load_reduction_pct=0.859192 moved_to_echo=15867 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3731, 0, 0, 603, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 372, 0, 0, 0, 0, 3484, 17, 15, 8, 4, 0, 0, 0, 0, 0, 4092, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4100, 4334, 4095, 3900, 4093, 4078, 4088] avg=4096.0000 max=4334 max_rank=2 min=3900 min_rank=4 imbalance_ratio=1.058105 max_load_reduction=28434 max_load_reduction_pct=0.867737 moved_to_echo=16159 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3492, 17, 15, 8, 4, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4100, 4699, 4095, 3537, 4092, 4076, 4088] avg=4096.0000 max=4699 max_rank=2 min=3537 min_rank=4 imbalance_ratio=1.147217 max_load_reduction=28069 max_load_reduction_pct=0.856598 moved_to_echo=15793 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 3492, 17, 15, 8, 4, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4100, 4658, 4095, 3578, 4091, 4077, 4088] avg=4096.0000 max=4658 max_rank=2 min=3578 min_rank=4 imbalance_ratio=1.137207 max_load_reduction=28110 max_load_reduction_pct=0.857849 moved_to_echo=15834 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4052, 49, 0, 65, 0, 5, 16, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2101, 2, 4, 586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 12, 14, 0, 0, 3, 0, 0, 4068, 4, 0, 0, 0, 0, 0, 0, 4, 17, 0, 1902, 1, 0, 24, 0, 3414, 16, 22, 10, 7, 0, 0, 0, 0, 0, 3984, 0, 12, 47, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 54, 0, 0, 7, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4187, 4070, 2693, 4102, 5417, 4060, 4081, 4158] avg=4096.0000 max=5417 max_rank=4 min=2693 min_rank=2 imbalance_ratio=1.322510 max_load_reduction=27351 max_load_reduction_pct=0.834686 moved_to_echo=17716 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 0, 4608, 128, 128, 128, 0, 27776, 256, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 0, 28032, 128, 128, 4864, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 32640, 128, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 0, 128, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 0, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 32512, 0, 128, 384, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 128, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 128, 128, 128, 0, 0, 128, 32640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=32896 slack_slots=2816 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True + [2026-09-21 08:25:37] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 254.4 | throughput per GPU (TFLOP/s/GPU): 126.6 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.836311E+00 | loss scale: 1.0 | grad norm: 7.078 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[129, 277, 862, 7, 0, 637, 879, 301, 14, 552, 106, 3, 23, 14, 485, 109, 1, 913, 1195, 1943, 46, 155, 386, 25, 75, 12, 191, 293, 1025, 687, 843, 1357, 495, 373, 468, 1110, 92, 1178, 556, 1002, 1656, 887, 1530, 89, 988, 616, 8, 39, 800, 638, 778, 26, 599, 1082, 1719, 22, 1509, 28, 219, 23, 17, 29, 621, 26] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[217, 453, 656, 1, 0, 426, 927, 211, 13, 409, 16, 3, 31, 2, 423, 65, 18, 1345, 681, 1560, 79, 318, 516, 6, 116, 14, 71, 316, 974, 696, 992, 1447, 838, 567, 296, 837, 113, 1093, 595, 1114, 1373, 963, 1910, 403, 620, 336, 44, 107, 1075, 631, 609, 85, 847, 1774, 914, 61, 1466, 60, 317, 11, 27, 63, 589, 28] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[94, 318, 760, 7, 0, 449, 844, 319, 1, 648, 33, 0, 5, 38, 431, 202, 1, 1365, 1038, 1645, 24, 100, 286, 4, 110, 3, 63, 353, 1092, 378, 1350, 1574, 626, 485, 548, 1459, 175, 1258, 886, 726, 1076, 331, 1500, 45, 984, 732, 3, 47, 797, 723, 778, 7, 455, 1402, 1706, 22, 1268, 30, 225, 2, 17, 10, 899, 11] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[181, 326, 958, 2, 0, 768, 918, 234, 1, 635, 30, 0, 8, 35, 241, 103, 9, 1509, 674, 2141, 56, 186, 225, 2, 133, 15, 51, 342, 908, 693, 743, 1486, 933, 482, 437, 1259, 139, 1136, 690, 1033, 1463, 569, 1621, 37, 941, 843, 4, 36, 1081, 591, 512, 27, 584, 1309, 1650, 68, 1107, 8, 208, 10, 7, 15, 329, 26] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[174, 255, 1090, 0, 0, 684, 1167, 395, 4, 621, 12, 5, 8, 53, 350, 145, 8, 1025, 895, 1804, 31, 154, 234, 7, 68, 10, 95, 300, 1286, 808, 1009, 1229, 684, 354, 413, 1026, 116, 1160, 479, 807, 1667, 430, 1621, 51, 936, 634, 23, 53, 944, 689, 642, 43, 457, 1474, 1810, 37, 1488, 35, 248, 11, 55, 11, 433, 11] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[145, 258, 1270, 0, 0, 560, 869, 227, 3, 866, 40, 0, 7, 3, 294, 89, 15, 800, 1011, 1816, 18, 288, 355, 25, 90, 16, 83, 465, 1093, 780, 980, 1969, 788, 348, 519, 980, 100, 1114, 888, 893, 1001, 503, 1847, 109, 727, 424, 17, 26, 1054, 580, 569, 4, 522, 1765, 1511, 28, 1093, 20, 207, 3, 9, 6, 667, 11] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[237, 244, 704, 4, 3, 432, 880, 273, 3, 592, 29, 0, 22, 24, 543, 51, 7, 662, 546, 1930, 23, 289, 272, 13, 123, 10, 144, 323, 1146, 610, 863, 1466, 545, 599, 552, 1391, 211, 1864, 676, 1162, 1206, 497, 1958, 47, 931, 685, 3, 70, 944, 856, 422, 42, 601, 733, 1628, 34, 1582, 5, 298, 5, 93, 4, 642, 14] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[265, 270, 712, 10, 2, 542, 762, 368, 6, 916, 17, 2, 23, 47, 484, 116, 1, 1011, 975, 1952, 79, 135, 275, 0, 255, 24, 22, 356, 957, 532, 956, 1555, 1075, 296, 322, 1131, 134, 1007, 669, 996, 987, 691, 1546, 49, 734, 625, 5, 67, 718, 1049, 685, 81, 561, 889, 1726, 76, 1471, 24, 278, 2, 11, 38, 1177, 21] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[129, 277, 862, 7, 0, 637, 879, 301, 449, 307, 115, 70, 0, 0, 0, 0, 14, 552, 106, 3, 23, 14, 485, 109, 1719, 1178, 27, 0, 0, 0, 0, 0, 1, 913, 1195, 1636, 46, 155, 386, 25, 0, 0, 0, 0, 0, 0, 0, 0, 75, 12, 191, 293, 1025, 687, 843, 768, 0, 0, 0, 0, 0, 0, 0, 0, 495, 373, 468, 661, 92, 0, 556, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1656, 887, 490, 89, 988, 616, 8, 39, 0, 0, 0, 0, 0, 0, 0, 0, 800, 638, 778, 26, 599, 967, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 1509, 28, 219, 23, 17, 29, 621, 26, 1013, 519, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4033, 4230, 4357, 3894, 3647, 4773, 3830, 4004] avg=4096.0000 max=4773 max_rank=5 min=3647 min_rank=4 imbalance_ratio=1.165283 max_load_reduction=27995 max_load_reduction_pct=0.854340 moved_to_echo=16254 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[94, 318, 760, 7, 0, 449, 844, 319, 590, 260, 149, 82, 0, 0, 0, 0, 1, 648, 33, 0, 5, 38, 431, 202, 1706, 1258, 26, 0, 0, 0, 0, 0, 1, 1365, 1038, 1385, 24, 100, 286, 4, 0, 0, 0, 0, 0, 0, 0, 0, 110, 3, 63, 353, 1092, 378, 1350, 890, 0, 0, 0, 0, 0, 0, 0, 0, 626, 485, 548, 869, 175, 0, 886, 726, 0, 0, 0, 0, 0, 0, 0, 0, 1076, 331, 481, 45, 984, 732, 3, 47, 0, 0, 0, 0, 0, 0, 0, 0, 797, 723, 778, 7, 455, 1253, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 1268, 30, 225, 2, 17, 10, 899, 11, 993, 602, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3872, 4348, 4203, 4239, 4315, 3699, 4035, 4057] avg=4096.0000 max=4348 max_rank=1 min=3699 min_rank=5 imbalance_ratio=1.061523 max_load_reduction=28420 max_load_reduction_pct=0.867310 moved_to_echo=16106 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[217, 453, 656, 1, 0, 426, 927, 211, 338, 246, 189, 75, 0, 0, 0, 0, 13, 409, 16, 3, 31, 2, 423, 65, 914, 1093, 33, 0, 0, 0, 0, 0, 18, 1345, 681, 1314, 79, 318, 516, 6, 0, 0, 0, 0, 0, 0, 0, 0, 116, 14, 71, 316, 974, 696, 992, 818, 0, 0, 0, 0, 0, 0, 0, 0, 838, 567, 296, 499, 113, 0, 595, 1114, 0, 0, 0, 0, 0, 0, 0, 0, 1373, 963, 612, 403, 620, 336, 44, 107, 0, 0, 0, 0, 0, 0, 0, 0, 1075, 631, 609, 85, 847, 1585, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 1466, 60, 317, 11, 27, 63, 589, 28, 1265, 554, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3739, 3002, 4277, 3997, 4022, 4458, 4893, 4380] avg=4096.0000 max=4893 max_rank=6 min=3002 min_rank=1 imbalance_ratio=1.194580 max_load_reduction=27875 max_load_reduction_pct=0.850677 moved_to_echo=17753 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[181, 326, 958, 2, 0, 768, 918, 234, 509, 338, 139, 77, 0, 0, 0, 0, 1, 635, 30, 0, 8, 35, 241, 103, 1650, 1136, 28, 0, 0, 0, 0, 0, 9, 1509, 674, 1803, 56, 186, 225, 2, 0, 0, 0, 0, 0, 0, 0, 0, 133, 15, 51, 342, 908, 693, 743, 840, 0, 0, 0, 0, 0, 0, 0, 0, 933, 482, 437, 750, 139, 0, 690, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1463, 569, 520, 37, 941, 843, 4, 36, 0, 0, 0, 0, 0, 0, 0, 0, 1081, 591, 512, 27, 584, 1170, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 1107, 8, 208, 10, 7, 15, 329, 26, 1073, 569, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4450, 3867, 4464, 3725, 4464, 4413, 4033, 3352] avg=4096.0000 max=4464 max_rank=2 min=3352 min_rank=7 imbalance_ratio=1.089844 max_load_reduction=28304 max_load_reduction_pct=0.863770 moved_to_echo=16262 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[174, 255, 1090, 0, 0, 684, 1167, 395, 415, 285, 157, 64, 0, 0, 0, 0, 4, 621, 12, 5, 8, 53, 350, 145, 1810, 1160, 28, 0, 0, 0, 0, 0, 8, 1025, 895, 1519, 31, 154, 234, 7, 0, 0, 0, 0, 0, 0, 0, 0, 68, 10, 95, 300, 1286, 808, 1009, 695, 0, 0, 0, 0, 0, 0, 0, 0, 684, 354, 413, 611, 116, 0, 479, 807, 0, 0, 0, 0, 0, 0, 0, 0, 1667, 430, 520, 51, 936, 634, 23, 53, 0, 0, 0, 0, 0, 0, 0, 0, 944, 689, 642, 43, 457, 1317, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 1488, 35, 248, 11, 55, 11, 433, 11, 1073, 470, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4686, 4196, 3873, 4271, 3464, 4314, 4129, 3835] avg=4096.0000 max=4686 max_rank=0 min=3464 min_rank=4 imbalance_ratio=1.144043 max_load_reduction=28082 max_load_reduction_pct=0.856995 moved_to_echo=15742 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[145, 258, 1270, 0, 0, 560, 869, 227, 396, 287, 188, 102, 0, 0, 0, 0, 3, 866, 40, 0, 7, 3, 294, 89, 1511, 1114, 32, 0, 0, 0, 0, 0, 15, 800, 1011, 1529, 18, 288, 355, 25, 0, 0, 0, 0, 0, 0, 0, 0, 90, 16, 83, 465, 1093, 780, 980, 1114, 0, 0, 0, 0, 0, 0, 0, 0, 788, 348, 519, 584, 100, 0, 888, 893, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 503, 592, 109, 727, 424, 17, 26, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 580, 569, 4, 522, 1577, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 1093, 20, 207, 3, 9, 6, 667, 11, 1223, 753, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4302, 3959, 4041, 4621, 4120, 3399, 4334, 3992] avg=4096.0000 max=4621 max_rank=3 min=3399 min_rank=5 imbalance_ratio=1.128174 max_load_reduction=28147 max_load_reduction_pct=0.858978 moved_to_echo=15845 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[265, 270, 712, 10, 2, 542, 762, 368, 457, 308, 94, 81, 0, 0, 0, 0, 6, 916, 17, 2, 23, 47, 484, 116, 1726, 1007, 27, 0, 0, 0, 0, 0, 1, 1011, 975, 1644, 79, 135, 275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 24, 22, 356, 957, 532, 956, 879, 0, 0, 0, 0, 0, 0, 0, 0, 1075, 296, 322, 674, 134, 0, 669, 996, 0, 0, 0, 0, 0, 0, 0, 0, 987, 691, 496, 49, 734, 625, 5, 67, 0, 0, 0, 0, 0, 0, 0, 0, 718, 1049, 685, 81, 561, 795, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 1471, 24, 278, 2, 11, 38, 1177, 21, 1023, 595, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3871, 4371, 4120, 3981, 4166, 3654, 3965, 4640] avg=4096.0000 max=4640 max_rank=7 min=3654 min_rank=5 imbalance_ratio=1.132812 max_load_reduction=28128 max_load_reduction_pct=0.858398 moved_to_echo=16425 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[237, 244, 704, 4, 3, 432, 880, 273, 566, 309, 83, 80, 0, 0, 0, 0, 3, 592, 29, 0, 22, 24, 543, 51, 1628, 1864, 39, 0, 0, 0, 0, 0, 7, 662, 546, 1621, 23, 289, 272, 13, 0, 0, 0, 0, 0, 0, 0, 0, 123, 10, 144, 323, 1146, 610, 863, 821, 0, 0, 0, 0, 0, 0, 0, 0, 545, 599, 552, 825, 211, 0, 676, 1162, 0, 0, 0, 0, 0, 0, 0, 0, 1206, 497, 619, 47, 931, 685, 3, 70, 0, 0, 0, 0, 0, 0, 0, 0, 944, 856, 422, 42, 601, 650, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 1582, 5, 298, 5, 93, 4, 642, 14, 1300, 565, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3815, 4795, 3433, 4040, 4570, 4058, 3549, 4508] avg=4096.0000 max=4795 max_rank=1 min=3433 min_rank=2 imbalance_ratio=1.170654 max_load_reduction=27973 max_load_reduction_pct=0.853668 moved_to_echo=16685 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[1536, 2432, 7040, 128, 128, 4608, 7296, 2432, 3840, 2432, 1152, 640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[6016, 3584, 3584, 5504, 1152, 0, 5504, 7808, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[10496, 4992, 4352, 896, 6912, 4992, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 5248, 384, 128, 128, 256, 3328, 896, 12672, 9856, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1024, 128, 768, 2816, 8576, 5248, 7808, 6912, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[7424, 5760, 5120, 384, 4736, 9344, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[11008, 256, 2048, 128, 256, 256, 5376, 256, 9088, 4736, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 8704, 7040, 12544, 384, 1664, 2560, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[624, 0, 0, 0, 0, 981, 0, 98, 2, 0, 3927, 479, 0, 1, 362, 0, 0, 0, 53, 0, 845, 75, 34, 0, 1, 1626, 0, 0, 2, 523, 2262, 0, 0, 0, 1, 0, 2444, 4085, 3742, 1, 193, 3, 185, 0, 0, 0, 24, 0, 1, 0, 806, 415, 396, 0, 0, 0, 151, 1, 176, 2962, 1989, 0, 3298, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[618, 0, 0, 1, 0, 1453, 0, 88, 0, 11, 3656, 1125, 0, 0, 140, 18, 0, 3, 282, 148, 5, 41, 390, 0, 2, 2235, 1, 0, 280, 281, 973, 0, 0, 3, 0, 0, 1927, 4033, 1609, 3, 1148, 0, 608, 3, 0, 0, 152, 3, 0, 6, 18, 1467, 1201, 0, 0, 0, 1349, 0, 30, 2878, 2095, 0, 2484, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[288, 12, 2, 0, 0, 90, 0, 287, 0, 37, 3808, 501, 0, 1, 125, 2, 1, 0, 914, 1, 1079, 1, 18, 1, 0, 1627, 1, 1, 0, 117, 190, 2, 0, 0, 0, 0, 3178, 4017, 1860, 10, 1785, 0, 1405, 0, 3, 1, 20, 0, 0, 0, 122, 654, 807, 0, 0, 0, 438, 1, 19, 2911, 2672, 0, 3758, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[919, 0, 0, 1, 0, 767, 3, 11, 14, 2, 3866, 47, 0, 0, 830, 1, 0, 0, 815, 76, 80, 28, 41, 5, 0, 1298, 0, 6, 14, 2002, 2356, 15, 3, 0, 0, 0, 1925, 3148, 3517, 0, 1742, 0, 285, 0, 0, 0, 170, 0, 2, 14, 225, 50, 554, 4, 1, 3, 40, 0, 728, 2997, 3333, 0, 830, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[781, 0, 1, 0, 0, 1480, 0, 0, 0, 73, 2632, 364, 4, 0, 1728, 0, 0, 0, 112, 19, 395, 1, 75, 15, 0, 2293, 0, 0, 0, 1499, 1871, 26, 0, 0, 1, 0, 2285, 3818, 1444, 0, 1445, 0, 235, 0, 0, 0, 0, 0, 4, 0, 27, 1812, 214, 0, 111, 7, 338, 9, 728, 1988, 1353, 0, 3580, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[2095, 0, 480, 0, 0, 1524, 0, 157, 21, 202, 885, 600, 0, 0, 7, 0, 0, 0, 3050, 32, 3226, 1, 24, 0, 0, 233, 0, 0, 2, 157, 1900, 24, 0, 0, 0, 0, 677, 3819, 1208, 6, 94, 0, 680, 0, 0, 0, 166, 0, 1, 1, 652, 105, 712, 0, 2, 0, 298, 0, 1535, 944, 3673, 0, 3575, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[107, 0, 5, 0, 2, 829, 0, 81, 14, 128, 3780, 969, 0, 0, 155, 0, 0, 0, 823, 1, 380, 3, 29, 94, 0, 808, 0, 57, 1, 22, 2299, 142, 5, 0, 2, 0, 3451, 2485, 1553, 0, 1626, 0, 131, 0, 7, 0, 6, 4, 0, 0, 3348, 140, 1214, 6, 0, 20, 1656, 1, 128, 75, 2564, 0, 3617, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[28, 0, 24, 0, 0, 1042, 0, 59, 0, 0, 3940, 581, 8, 0, 263, 21, 0, 0, 885, 0, 1622, 13, 909, 0, 1, 2659, 1, 0, 173, 84, 215, 21, 1, 11, 0, 1, 1562, 3694, 1259, 0, 283, 9, 577, 1, 1, 0, 1, 0, 0, 0, 363, 374, 729, 1, 0, 1, 704, 0, 2190, 3007, 2879, 0, 2569, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[624, 0, 0, 0, 0, 981, 0, 98, 1309, 1182, 0, 0, 0, 0, 0, 0, 2, 0, 3549, 479, 0, 1, 362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 845, 75, 34, 0, 1461, 150, 0, 0, 0, 0, 0, 0, 1, 1626, 0, 0, 2, 523, 2262, 0, 228, 129, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2315, 0, 3742, 1, 0, 0, 0, 0, 0, 0, 0, 0, 193, 3, 185, 0, 0, 0, 24, 0, 2776, 0, 0, 0, 0, 0, 0, 0, 1, 0, 806, 415, 396, 0, 0, 0, 2116, 93, 0, 0, 0, 0, 0, 0, 151, 1, 176, 2962, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4194, 4393, 2618, 4771, 6059, 3181, 3827, 3725] avg=4096.0000 max=6059 max_rank=4 min=2618 min_rank=2 imbalance_ratio=1.479248 max_load_reduction=26709 max_load_reduction_pct=0.815094 moved_to_echo=16792 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[618, 0, 0, 1, 0, 1453, 0, 88, 1293, 891, 0, 0, 0, 0, 0, 0, 0, 11, 3303, 1125, 0, 0, 140, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 282, 148, 5, 41, 390, 0, 1539, 140, 0, 0, 0, 0, 0, 0, 2, 2235, 1, 0, 280, 281, 973, 0, 213, 101, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1826, 0, 1609, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1148, 0, 608, 3, 0, 0, 152, 3, 2740, 0, 0, 0, 0, 0, 0, 0, 0, 6, 18, 1467, 1201, 0, 0, 0, 1593, 98, 0, 0, 0, 0, 0, 0, 1349, 0, 30, 2878, 458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4344, 4597, 2548, 4086, 3441, 4654, 4383, 4715] avg=4096.0000 max=4715 max_rank=7 min=2548 min_rank=2 imbalance_ratio=1.151123 max_load_reduction=28053 max_load_reduction_pct=0.856110 moved_to_echo=17193 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[288, 12, 2, 0, 0, 90, 0, 287, 1287, 1348, 0, 0, 0, 0, 0, 0, 0, 37, 3441, 501, 0, 1, 125, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 914, 1, 1079, 1, 18, 1, 1963, 146, 0, 0, 0, 0, 0, 0, 0, 1627, 1, 1, 0, 117, 190, 2, 221, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3011, 0, 1860, 10, 0, 0, 0, 0, 0, 0, 0, 0, 1785, 0, 1405, 0, 3, 1, 20, 0, 2730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 654, 807, 0, 0, 0, 2410, 126, 0, 0, 0, 0, 0, 0, 438, 1, 19, 2911, 583, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3314, 4107, 4124, 2326, 4881, 5944, 4119, 3953] avg=4096.0000 max=5944 max_rank=5 min=2326 min_rank=3 imbalance_ratio=1.451172 max_load_reduction=26824 max_load_reduction_pct=0.818604 moved_to_echo=18897 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[919, 0, 0, 1, 0, 767, 3, 11, 1009, 297, 0, 0, 0, 0, 0, 0, 14, 2, 3493, 47, 0, 0, 830, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 815, 76, 80, 28, 41, 5, 2448, 148, 0, 0, 0, 0, 0, 0, 0, 1298, 0, 6, 14, 2002, 2356, 15, 225, 101, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1824, 0, 3517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1742, 0, 285, 0, 0, 0, 170, 0, 2139, 0, 0, 0, 0, 0, 0, 0, 2, 14, 225, 50, 554, 4, 1, 3, 533, 157, 0, 0, 0, 0, 0, 0, 40, 0, 728, 2997, 728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3007, 4387, 3641, 6017, 5344, 4336, 1543, 4493] avg=4096.0000 max=6017 max_rank=3 min=1543 min_rank=6 imbalance_ratio=1.468994 max_load_reduction=26751 max_load_reduction_pct=0.816376 moved_to_echo=15716 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[107, 0, 5, 0, 2, 829, 0, 81, 796, 1297, 0, 0, 0, 0, 0, 0, 14, 128, 3415, 969, 0, 0, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 823, 1, 380, 3, 29, 94, 1883, 145, 0, 0, 0, 0, 0, 0, 0, 808, 0, 57, 1, 22, 2299, 142, 220, 182, 0, 0, 0, 0, 0, 0, 5, 0, 2, 0, 3269, 0, 1553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1626, 0, 131, 0, 7, 0, 6, 4, 1689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3348, 140, 1214, 6, 0, 20, 2320, 120, 0, 0, 0, 0, 0, 0, 1656, 1, 128, 75, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3117, 4681, 3358, 3731, 4829, 3463, 7168, 2421] avg=4096.0000 max=7168 max_rank=6 min=2421 min_rank=7 imbalance_ratio=1.750000 max_load_reduction=25600 max_load_reduction_pct=0.781250 moved_to_echo=17881 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[28, 0, 24, 0, 0, 1042, 0, 59, 1185, 922, 0, 0, 0, 0, 0, 0, 0, 0, 3553, 581, 8, 0, 263, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 885, 0, 1622, 13, 909, 0, 2119, 154, 0, 0, 0, 0, 0, 0, 1, 2659, 1, 0, 173, 84, 215, 21, 233, 86, 0, 0, 0, 0, 0, 0, 1, 11, 0, 1, 1476, 0, 1259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 283, 9, 577, 1, 1, 0, 1, 0, 2509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 363, 374, 729, 1, 0, 1, 1647, 140, 0, 0, 0, 0, 0, 0, 704, 0, 2190, 3007, 620, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3260, 4426, 5702, 3473, 2748, 3381, 3255, 6523] avg=4096.0000 max=6523 max_rank=7 min=2748 min_rank=4 imbalance_ratio=1.592529 max_load_reduction=26245 max_load_reduction_pct=0.800934 moved_to_echo=15907 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[781, 0, 1, 0, 0, 1480, 0, 0, 1224, 1284, 0, 0, 0, 0, 0, 0, 0, 73, 2378, 364, 4, 0, 1728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 19, 395, 1, 75, 15, 994, 101, 0, 0, 0, 0, 0, 0, 0, 2293, 0, 0, 0, 1499, 1871, 26, 153, 120, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2165, 0, 1444, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1445, 0, 235, 0, 0, 0, 0, 0, 2594, 0, 0, 0, 0, 0, 0, 0, 4, 0, 27, 1812, 214, 0, 111, 7, 2296, 63, 0, 0, 0, 0, 0, 0, 338, 9, 728, 1988, 296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4770, 4547, 1712, 5962, 3610, 4274, 4534, 3359] avg=4096.0000 max=5962 max_rank=3 min=1712 min_rank=2 imbalance_ratio=1.455566 max_load_reduction=26806 max_load_reduction_pct=0.818054 moved_to_echo=15777 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2095, 0, 480, 0, 0, 1524, 0, 157, 1224, 1282, 0, 0, 0, 0, 0, 0, 21, 202, 800, 600, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3050, 32, 3226, 1, 24, 0, 2698, 34, 0, 0, 0, 0, 0, 0, 0, 233, 0, 0, 2, 157, 1900, 24, 51, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 642, 0, 1208, 6, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 680, 0, 0, 0, 166, 0, 2595, 0, 0, 0, 0, 0, 0, 0, 1, 1, 652, 105, 712, 0, 2, 0, 2293, 173, 0, 0, 0, 0, 0, 0, 298, 0, 1535, 944, 802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6762, 1630, 9065, 2402, 1856, 3535, 3939, 3579] avg=4096.0000 max=9065 max_rank=2 min=1630 min_rank=1 imbalance_ratio=2.213135 max_load_reduction=23703 max_load_reduction_pct=0.723358 moved_to_echo=12909 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[5504, 128, 512, 128, 128, 8192, 128, 896, 9344, 8576, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 12800, 128, 128, 512, 4736, 12160, 256, 1664, 1024, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 7040, 384, 7680, 256, 1536, 128, 15232, 1024, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4992, 128, 5632, 17792, 4608, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 5632, 5120, 5888, 128, 128, 128, 15232, 1024, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[8320, 128, 4224, 128, 128, 128, 640, 128, 19840, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 16640, 0, 16256, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 512, 23936, 4736, 128, 128, 3712, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3161, 0, 1481, 0, 5, 0, 10, 280, 1, 3277, 1, 1584, 3480, 2, 3973, 26, 11, 0, 0, 0, 0, 0, 1, 210, 1, 0, 0, 0, 0, 370, 0, 0, 1, 1, 8, 1, 0, 1, 0, 7, 3087, 0, 3, 0, 8, 0, 3977, 1225, 0, 1701, 3980, 0, 0, 0, 0, 0, 837, 56, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 4047, 0, 3752, 6, 21, 0, 1, 1688, 0, 1145, 0, 2582, 2095, 0, 3770, 5, 0, 10, 0, 0, 0, 0, 1, 110, 0, 0, 0, 0, 49, 26, 0, 0, 14, 3, 70, 0, 0, 0, 0, 0, 3423, 22, 0, 0, 13, 0, 4092, 19, 0, 1696, 3574, 0, 0, 0, 0, 0, 368, 4, 160, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 53, 0, 4085, 1, 2669, 0, 646, 2, 0, 210, 0, 3266, 0, 2087, 2683, 164, 2598, 1, 41, 0, 0, 0, 2, 1, 0, 5, 0, 0, 0, 1, 1, 280, 0, 5, 0, 0, 7, 0, 1, 0, 0, 1, 3792, 0, 0, 0, 0, 0, 4060, 22, 0, 2758, 3233, 0, 1, 42, 0, 2, 46, 2, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 4085, 2, 305, 0, 2, 0, 0, 101, 0, 4079, 0, 493, 2018, 13, 4084, 0, 1, 2, 0, 3, 0, 0, 0, 0, 0, 2, 0, 0, 0, 738, 0, 0, 115, 3, 0, 0, 0, 1, 2, 6, 4007, 0, 0, 0, 2, 39, 4036, 532, 5, 3370, 3851, 0, 0, 13, 0, 0, 815, 0, 40, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[4, 3, 37, 0, 4023, 0, 495, 0, 56, 0, 0, 1924, 1, 4091, 0, 1872, 3329, 0, 3725, 0, 0, 1, 0, 0, 0, 0, 1, 1, 123, 13, 0, 0, 6, 614, 23, 0, 0, 227, 0, 0, 1, 0, 2, 1, 1870, 0, 0, 0, 39, 2, 4069, 17, 1, 2763, 3166, 0, 1, 0, 0, 0, 257, 0, 10, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3155, 0, 2319, 0, 1907, 0, 0, 1086, 0, 4091, 0, 932, 108, 3, 3700, 2, 8, 0, 0, 0, 0, 0, 0, 21, 8, 0, 0, 0, 0, 2464, 89, 0, 0, 0, 37, 0, 0, 0, 0, 0, 756, 0, 0, 0, 0, 2, 4064, 211, 2, 2999, 4065, 0, 0, 35, 0, 0, 688, 3, 13, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2100, 0, 3900, 0, 102, 0, 0, 239, 0, 4079, 1, 130, 3774, 13, 1275, 12, 21, 0, 0, 0, 2, 0, 0, 18, 0, 0, 0, 0, 0, 758, 0, 3, 0, 192, 16, 0, 0, 0, 0, 0, 2885, 1, 0, 0, 0, 0, 4054, 2378, 0, 996, 3962, 0, 2, 0, 0, 2, 1562, 6, 285, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3089, 0, 1953, 0, 145, 0, 0, 1268, 0, 4069, 0, 82, 2133, 0, 3119, 25, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 5, 2, 1586, 0, 0, 2, 93, 14, 0, 0, 0, 0, 6, 3233, 0, 0, 0, 5, 0, 4094, 10, 0, 3187, 4087, 0, 15, 0, 0, 0, 516, 0, 7, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 2303, 0, 3752, 6, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 1, 1688, 0, 543, 0, 2582, 0, 0, 0, 0, 0, 0, 0, 0, 2095, 0, 1837, 5, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 110, 0, 0, 0, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 49, 26, 0, 0, 14, 3, 70, 0, 1933, 374, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3423, 22, 0, 0, 1409, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 19, 0, 1696, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 368, 4, 160, 0, 2526, 228, 29, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6063, 4835, 3947, 4174, 2804, 4854, 2776, 3315] avg=4096.0000 max=6063 max_rank=0 min=2776 min_rank=6 imbalance_ratio=1.480225 max_load_reduction=26705 max_load_reduction_pct=0.814972 moved_to_echo=13749 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1799, 0, 1481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 10, 280, 1, 1553, 1, 1584, 0, 0, 0, 0, 0, 0, 0, 0, 3480, 2, 1936, 26, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 210, 1, 0, 0, 0, 3949, 0, 0, 0, 0, 0, 0, 0, 0, 370, 0, 0, 1, 1, 8, 1, 2037, 1071, 262, 0, 0, 0, 0, 0, 0, 1, 0, 7, 3087, 0, 3, 0, 1100, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 1225, 0, 1701, 1167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 837, 56, 1, 0, 2813, 653, 28, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3280, 3434, 5455, 4161, 3751, 4198, 4101, 4388] avg=4096.0000 max=5455 max_rank=2 min=3280 min_rank=0 imbalance_ratio=1.331787 max_load_reduction=27313 max_load_reduction_pct=0.833527 moved_to_echo=16438 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 2325, 2, 305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 101, 0, 1933, 0, 493, 0, 0, 0, 0, 0, 0, 0, 0, 2018, 13, 1990, 0, 1, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4008, 0, 0, 0, 0, 0, 0, 0, 0, 738, 0, 0, 115, 3, 0, 0, 2094, 1334, 338, 0, 0, 0, 0, 0, 0, 1, 2, 6, 4007, 0, 0, 0, 1422, 0, 0, 0, 0, 0, 0, 0, 2, 39, 0, 532, 5, 3370, 1129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 815, 0, 40, 1, 2722, 812, 28, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2634, 2529, 4027, 4010, 4622, 5438, 5077, 4431] avg=4096.0000 max=5438 max_rank=5 min=2529 min_rank=1 imbalance_ratio=1.327637 max_load_reduction=27330 max_load_reduction_pct=0.834045 moved_to_echo=19568 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1796, 0, 2319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1907, 0, 0, 1086, 0, 1939, 0, 932, 0, 0, 0, 0, 0, 0, 0, 0, 108, 3, 1803, 2, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 8, 0, 0, 0, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 2464, 89, 0, 0, 0, 37, 0, 1897, 1337, 261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 756, 0, 0, 0, 1098, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 211, 2, 2999, 1192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 688, 3, 13, 0, 2873, 815, 29, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 5864, 1924, 4064, 6085, 1854, 4406, 4456] avg=4096.0000 max=6085 max_rank=4 min=1854 min_rank=5 imbalance_ratio=1.485596 max_load_reduction=26683 max_load_reduction_pct=0.814301 moved_to_echo=16801 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4, 3, 37, 0, 2290, 0, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 1924, 1, 1939, 0, 1872, 0, 0, 0, 0, 0, 0, 0, 0, 3329, 0, 1815, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 123, 13, 0, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 6, 614, 23, 0, 0, 227, 0, 0, 1910, 1337, 333, 0, 0, 0, 0, 0, 1, 0, 2, 1, 1870, 0, 0, 0, 1400, 0, 0, 0, 0, 0, 0, 0, 39, 2, 0, 17, 1, 2763, 928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 257, 0, 10, 0, 2238, 815, 28, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2829, 5792, 5145, 4179, 4450, 3274, 3750, 3349] avg=4096.0000 max=5792 max_rank=1 min=2829 min_rank=0 imbalance_ratio=1.414062 max_load_reduction=26976 max_load_reduction_pct=0.823242 moved_to_echo=14823 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1758, 0, 1953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 0, 1268, 0, 1929, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 2133, 0, 1520, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 5, 4066, 0, 0, 0, 0, 0, 0, 0, 2, 1586, 0, 0, 2, 93, 14, 0, 1599, 1330, 256, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3233, 0, 0, 0, 1075, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 10, 0, 3187, 1198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 516, 0, 7, 0, 2889, 810, 28, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3711, 3424, 3678, 4094, 4882, 4314, 4400, 4265] avg=4096.0000 max=4882 max_rank=4 min=3424 min_rank=1 imbalance_ratio=1.191895 max_load_reduction=27886 max_load_reduction_pct=0.851013 moved_to_echo=17861 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1195, 0, 3900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 0, 239, 0, 1933, 1, 130, 0, 0, 0, 0, 0, 0, 0, 0, 3774, 13, 622, 12, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 18, 0, 0, 0, 0, 4025, 0, 0, 0, 0, 0, 0, 0, 0, 758, 0, 3, 0, 192, 16, 0, 653, 1334, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2885, 1, 0, 0, 731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2378, 0, 996, 1161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 1562, 6, 285, 0, 2801, 812, 29, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5095, 2405, 4442, 4045, 3130, 3617, 4535, 5499] avg=4096.0000 max=5499 max_rank=7 min=2405 min_rank=1 imbalance_ratio=1.342529 max_load_reduction=27269 max_load_reduction_pct=0.832184 moved_to_echo=16781 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 53, 0, 2318, 1, 2669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 646, 2, 0, 210, 0, 1540, 0, 2087, 0, 0, 0, 0, 0, 0, 0, 0, 2683, 164, 1261, 1, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 5, 0, 0, 0, 1, 4032, 0, 0, 0, 0, 0, 0, 0, 1, 280, 0, 5, 0, 0, 7, 0, 1337, 1072, 342, 0, 0, 0, 0, 0, 1, 0, 0, 1, 3792, 0, 0, 0, 1425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 2758, 943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 42, 0, 2, 46, 2, 0, 0, 2290, 654, 28, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5041, 4485, 4150, 4041, 3044, 5219, 3723, 3065] avg=4096.0000 max=5219 max_rank=5 min=3044 min_rank=4 imbalance_ratio=1.274170 max_load_reduction=27549 max_load_reduction_pct=0.840729 moved_to_echo=15051 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 6912, 128, 128, 256, 640, 256, 128, 13568, 9216, 2304, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 5120, 128, 640, 128, 21248, 5632, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 15872, 128, 16896, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[2944, 128, 128, 6912, 128, 13312, 128, 9856, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[19712, 256, 12800, 128, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 4480, 128, 19584, 8832, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 23168, 128, 128, 0, 9728, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 512, 256, 128, 0, 128, 32256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3571, 0, 0, 0, 2527, 773, 0, 0, 2471, 0, 12, 0, 0, 4096, 2140, 0, 0, 0, 0, 0, 0, 0, 0, 1, 351, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4084, 43, 0, 0, 0, 1, 0, 406, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4057, 0, 1, 0, 711, 162, 0, 0, 2023, 0, 0, 0, 0, 4048, 4086, 3, 0, 0, 0, 0, 0, 1, 0, 1, 8, 0, 4095, 0, 0, 3849, 0, 8, 0, 0, 0, 0, 0, 3801, 4, 0, 0, 0, 0, 0, 2270, 0, 0, 3575, 0, 20, 0, 0, 0, 0, 0, 35, 10, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4075, 1, 0, 0, 967, 448, 0, 0, 2576, 0, 42, 0, 5, 3562, 3897, 0, 0, 3, 0, 2, 0, 0, 0, 0, 0, 0, 4090, 1, 0, 4027, 0, 0, 0, 0, 0, 4, 0, 4094, 227, 1, 0, 0, 0, 0, 598, 0, 0, 3913, 5, 2, 0, 0, 1, 0, 0, 225, 0, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 1, 4093, 0, 0, 1, 0, 1055, 0, 0, 378, 0, 17, 0, 0, 4089, 4070, 0, 0, 0, 0, 0, 0, 0, 7, 0, 166, 0, 4088, 1, 2, 3993, 0, 0, 0, 0, 0, 0, 0, 4028, 0, 0, 0, 0, 2, 0, 2515, 0, 0, 4079, 0, 138, 0, 0, 0, 0, 0, 40, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4023, 0, 0, 0, 1116, 2348, 0, 0, 472, 0, 1, 0, 0, 3972, 4017, 0, 0, 20, 0, 0, 0, 0, 2, 0, 412, 0, 4096, 0, 0, 4034, 0, 0, 0, 0, 0, 0, 0, 3754, 0, 0, 0, 0, 0, 0, 320, 0, 0, 4075, 0, 5, 0, 0, 0, 0, 14, 87, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1, 0, 0, 2, 0, 4092, 1, 0, 1, 1054, 988, 0, 3, 8, 0, 0, 0, 0, 3940, 4082, 0, 20, 0, 0, 0, 0, 0, 17, 1, 510, 0, 4019, 0, 0, 4031, 0, 4, 2, 0, 6, 1, 0, 3599, 0, 0, 0, 0, 4, 0, 2322, 3, 0, 3972, 0, 53, 0, 0, 0, 0, 0, 30, 2, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1, 0, 0, 2, 1, 4089, 0, 0, 0, 43, 3138, 0, 0, 418, 0, 4, 2, 0, 3962, 3966, 0, 0, 1, 0, 0, 2, 0, 0, 2, 301, 0, 4084, 0, 8, 4041, 0, 1, 0, 0, 0, 9, 0, 3780, 2, 10, 0, 0, 0, 0, 598, 0, 0, 4089, 1, 212, 0, 0, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[2, 0, 15, 0, 2, 4065, 0, 0, 1, 586, 1853, 2, 0, 1557, 0, 90, 0, 0, 3991, 4093, 0, 0, 0, 0, 0, 0, 0, 1, 5, 19, 6, 3800, 0, 0, 3275, 4, 0, 4, 2, 0, 55, 0, 4091, 8, 0, 0, 0, 0, 0, 1563, 0, 1, 3420, 0, 244, 1, 4, 0, 3, 0, 2, 0, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 2, 0, 4092, 1, 0, 83, 0, 0, 0, 0, 0, 0, 0, 1, 1054, 988, 0, 3, 8, 0, 0, 626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 295, 4082, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 1, 510, 0, 3844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4031, 0, 4, 2, 0, 6, 175, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3599, 0, 0, 0, 0, 4, 92, 53, 0, 0, 0, 0, 0, 0, 0, 2322, 3, 0, 2736, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 2, 0, 0, 3645, 382, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4179, 2680, 4397, 4372, 4218, 3749, 5114, 4059] avg=4096.0000 max=5114 max_rank=6 min=2680 min_rank=1 imbalance_ratio=1.248535 max_load_reduction=27654 max_load_reduction_pct=0.843933 moved_to_echo=17140 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4075, 1, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 967, 448, 0, 0, 2576, 0, 42, 616, 0, 0, 0, 0, 0, 0, 0, 0, 5, 266, 3897, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3913, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4027, 0, 0, 0, 0, 0, 172, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4094, 227, 1, 0, 0, 0, 93, 53, 0, 0, 0, 0, 0, 0, 0, 598, 0, 0, 2696, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 225, 0, 0, 2, 3296, 376, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4160, 4649, 4171, 3915, 4200, 4472, 3301, 3900] avg=4096.0000 max=4649 max_rank=1 min=3301 min_rank=6 imbalance_ratio=1.135010 max_load_reduction=28119 max_load_reduction_pct=0.858124 moved_to_echo=15873 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3571, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 2527, 773, 0, 0, 2471, 0, 12, 645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 306, 2140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 351, 0, 3916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 43, 0, 0, 0, 1, 93, 55, 0, 0, 0, 0, 0, 0, 0, 406, 0, 0, 2822, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 3790, 394, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3655, 6428, 2446, 4268, 4276, 4276, 3228, 4191] avg=4096.0000 max=6428 max_rank=1 min=2446 min_rank=2 imbalance_ratio=1.569336 max_load_reduction=26340 max_load_reduction_pct=0.803833 moved_to_echo=15971 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4057, 0, 1, 84, 0, 0, 0, 0, 0, 0, 0, 0, 711, 162, 0, 0, 2023, 0, 0, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 303, 4086, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 8, 0, 3917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3849, 0, 8, 0, 0, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3801, 4, 0, 0, 0, 0, 94, 48, 0, 0, 0, 0, 0, 0, 0, 2270, 0, 0, 2464, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 10, 0, 0, 3745, 343, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4142, 3459, 4392, 3927, 4014, 3947, 4754, 4133] avg=4096.0000 max=4754 max_rank=6 min=3459 min_rank=1 imbalance_ratio=1.160645 max_load_reduction=28014 max_load_reduction_pct=0.854919 moved_to_echo=16848 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4023, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 1116, 2348, 0, 0, 472, 0, 1, 642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 297, 4017, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 412, 0, 3918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 0, 0, 0, 0, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3754, 0, 0, 0, 0, 0, 94, 55, 0, 0, 0, 0, 0, 0, 0, 320, 0, 0, 2806, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 87, 0, 0, 0, 3675, 392, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4579, 4334, 4332, 4214, 3903, 3131, 4168] avg=4096.0000 max=4579 max_rank=1 min=3131 min_rank=6 imbalance_ratio=1.117920 max_load_reduction=28189 max_load_reduction_pct=0.860260 moved_to_echo=15416 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 1, 4093, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1055, 0, 0, 378, 0, 17, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 306, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 166, 0, 3911, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3993, 0, 0, 0, 0, 0, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4028, 0, 0, 0, 0, 2, 93, 55, 0, 0, 0, 0, 0, 0, 0, 2515, 0, 0, 2809, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 1, 3783, 392, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4182, 2094, 4376, 4084, 4176, 4178, 5462, 4216] avg=4096.0000 max=5462 max_rank=6 min=2094 min_rank=1 imbalance_ratio=1.333496 max_load_reduction=27306 max_load_reduction_pct=0.833313 moved_to_echo=18032 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 2, 1, 4089, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 43, 3138, 0, 0, 418, 0, 4, 644, 0, 0, 0, 0, 0, 0, 0, 2, 0, 296, 3966, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 301, 0, 3907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4041, 0, 1, 0, 0, 0, 180, 0, 0, 0, 0, 0, 0, 0, 9, 0, 3780, 2, 10, 0, 0, 0, 93, 55, 0, 0, 0, 0, 0, 0, 0, 598, 0, 0, 2817, 1, 212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3666, 393, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4177, 4247, 4265, 4212, 4230, 3949, 3628, 4060] avg=4096.0000 max=4265 max_rank=2 min=3628 min_rank=6 imbalance_ratio=1.041260 max_load_reduction=28503 max_load_reduction_pct=0.869843 moved_to_echo=15867 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 15, 0, 2, 4065, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 1, 586, 1853, 2, 0, 1557, 0, 90, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 294, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 19, 6, 3627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3275, 4, 0, 4, 2, 0, 155, 0, 0, 0, 0, 0, 0, 0, 55, 0, 4091, 8, 0, 0, 0, 0, 91, 49, 0, 0, 0, 0, 0, 0, 0, 1563, 0, 1, 2341, 0, 244, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3, 0, 2, 0, 3, 0, 3697, 332, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4166, 4632, 4387, 3658, 3440, 4294, 4150, 4041] avg=4096.0000 max=4632 max_rank=1 min=3440 min_rank=4 imbalance_ratio=1.130859 max_load_reduction=28136 max_load_reduction_pct=0.858643 moved_to_echo=15925 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 7040, 10880, 128, 128, 9984, 0, 256, 4992, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 10624, 128, 128, 21504, 128, 768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 512, 128, 128, 128, 29312, 3072, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 2432, 30464, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 1792, 128, 30976, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 31360, 128, 128, 128, 128, 128, 1408, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 31232, 384, 128, 0, 0, 128, 768, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 0, 128, 128, 128, 32128, 128, 128, 768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33792 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1, 5, 0, 0, 2284, 3, 0, 3485, 4091, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 3, 0, 162, 4013, 4096, 0, 4096, 612, 0, 0, 0, 0, 13, 1307, 0, 2928, 0, 3756, 0, 3, 0, 0, 0, 0, 1879] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 11, 326, 0, 2, 4002, 4094, 0, 0, 1, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 26, 563, 0, 0, 0, 0, 163, 0, 0, 2679, 0, 0, 19, 0, 1, 0, 0, 5, 0, 2, 4002, 4090, 0, 4058, 331, 0, 18, 0, 1, 1, 210, 0, 2764, 2, 4068, 1, 0, 0, 0, 0, 1, 1305] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 2067, 0, 0, 4020, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 39, 0, 0, 2414, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4089, 4064, 1, 4089, 2447, 0, 0, 0, 0, 0, 1075, 21, 1077, 0, 63, 0, 0, 0, 0, 0, 0, 3159] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 3270, 0, 0, 3779, 4096, 7, 4, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 33, 0, 0, 0, 4, 0, 0, 0, 1904, 2, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4094, 4092, 0, 3939, 7, 0, 0, 0, 3, 0, 88, 18, 3536, 0, 1279, 0, 0, 0, 1, 0, 0, 2602] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[3, 0, 0, 0, 3759, 0, 0, 3442, 4073, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 3292, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 3958, 4085, 0, 4007, 458, 0, 0, 0, 1, 0, 648, 47, 3003, 0, 1224, 0, 0, 0, 0, 0, 0, 696] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 3318, 4, 0, 4072, 4094, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 70, 1, 0, 0, 0, 0, 0, 0, 556, 0, 0, 2, 0, 0, 0, 13, 0, 0, 0, 4091, 4094, 0, 4062, 511, 0, 1, 0, 1, 0, 292, 0, 1523, 0, 3199, 0, 0, 0, 0, 2, 1, 2857] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4040, 0, 0, 3852, 4096, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 281, 0, 0, 0, 0, 12, 0, 0, 2652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4095, 3783, 0, 0, 0, 0, 0, 1063, 0, 388, 0, 150, 0, 0, 0, 0, 0, 0, 157] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3752, 0, 0, 3698, 4084, 12, 0, 0, 82, 0, 3, 0, 83, 0, 0, 1, 0, 0, 0, 2, 2, 0, 2, 0, 18, 0, 0, 1792, 1, 0, 0, 4, 7, 0, 0, 0, 0, 0, 4065, 4082, 0, 4086, 533, 0, 0, 12, 12, 1, 172, 0, 2155, 1, 3385, 2, 4, 0, 4, 0, 1, 710] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 11, 326, 0, 2, 1309, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 26, 563, 3952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 163, 0, 0, 2679, 2157, 106, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 1, 0, 0, 5, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2955, 0, 0, 0, 331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 1, 1, 210, 0, 2764, 2, 780, 536, 0, 0, 0, 0, 0, 0, 4068, 1, 0, 0, 0, 0, 1, 1305, 267, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1648, 4084, 4563, 5105, 4115, 3288, 4312, 5653] avg=4096.0000 max=5653 max_rank=7 min=1648 min_rank=0 imbalance_ratio=1.380127 max_load_reduction=27115 max_load_reduction_pct=0.827484 moved_to_echo=17368 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 5, 0, 0, 2284, 3, 0, 1139, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 26, 1879, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 162, 2963, 0, 0, 0, 612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1307, 0, 2928, 0, 782, 467, 0, 0, 0, 0, 0, 0, 3756, 0, 3, 0, 0, 0, 0, 1879, 268, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3432, 4081, 3991, 2014, 4099, 3737, 5497, 5917] avg=4096.0000 max=5917 max_rank=7 min=2014 min_rank=3 imbalance_ratio=1.444580 max_load_reduction=26851 max_load_reduction_pct=0.819427 moved_to_echo=19250 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 2067, 0, 0, 1315, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 3983, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 2414, 2167, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3019, 0, 1, 0, 2447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1075, 21, 1077, 0, 797, 538, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 3159, 273, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3385, 4086, 4022, 4726, 4068, 5467, 3508, 3506] avg=4096.0000 max=5467 max_rank=5 min=3385 min_rank=0 imbalance_ratio=1.334717 max_load_reduction=27301 max_load_reduction_pct=0.833160 moved_to_echo=16549 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 3270, 0, 0, 1236, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 7, 4, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 33, 3836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1904, 2037, 103, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 6, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3022, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 88, 18, 3536, 0, 798, 506, 0, 0, 0, 0, 0, 0, 1279, 0, 0, 0, 1, 0, 0, 2602, 274, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4507, 4098, 3870, 4048, 4100, 3029, 4949, 4167] avg=4096.0000 max=4949 max_rank=6 min=3029 min_rank=5 imbalance_ratio=1.208252 max_load_reduction=27819 max_load_reduction_pct=0.848969 moved_to_echo=16245 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4040, 0, 0, 1260, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 281, 3988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 2652, 2076, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3024, 0, 0, 0, 3783, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 0, 388, 0, 798, 516, 0, 0, 0, 0, 0, 0, 150, 0, 0, 0, 0, 0, 0, 157, 274, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5301, 4091, 4269, 4847, 4096, 6807, 2765, 592] avg=4096.0000 max=6807 max_rank=5 min=592 min_rank=7 imbalance_ratio=1.661865 max_load_reduction=25961 max_load_reduction_pct=0.792267 moved_to_echo=14260 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 3318, 4, 0, 1332, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 70, 3956, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 556, 2195, 106, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 13, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3021, 0, 0, 0, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 292, 0, 1523, 0, 797, 545, 0, 0, 0, 0, 0, 0, 3199, 0, 0, 0, 0, 2, 1, 2857, 273, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4655, 4085, 4027, 2858, 4109, 3532, 3159, 6343] avg=4096.0000 max=6343 max_rank=7 min=2858 min_rank=3 imbalance_ratio=1.548584 max_load_reduction=26425 max_load_reduction_pct=0.806427 moved_to_echo=17143 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 0, 3759, 0, 0, 1126, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 3902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3292, 1855, 105, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2922, 0, 0, 0, 458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 648, 47, 3003, 0, 771, 461, 0, 0, 0, 0, 0, 0, 1224, 0, 0, 0, 0, 0, 0, 696, 265, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4888, 4074, 3959, 5252, 4088, 3380, 4931, 2196] avg=4096.0000 max=5252 max_rank=3 min=2196 min_rank=7 imbalance_ratio=1.282227 max_load_reduction=27516 max_load_reduction_pct=0.839722 moved_to_echo=14595 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3752, 0, 0, 1200, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 12, 0, 0, 82, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 1, 0, 0, 0, 2, 3980, 1, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 18, 0, 0, 1792, 1998, 106, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 7, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2995, 0, 0, 0, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 12, 1, 172, 0, 2155, 1, 794, 500, 0, 0, 0, 0, 0, 0, 3385, 2, 4, 0, 4, 0, 1, 710, 276, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4952, 4169, 4067, 3918, 4093, 3528, 3647, 4394] avg=4096.0000 max=4952 max_rank=0 min=3528 min_rank=5 imbalance_ratio=1.208984 max_load_reduction=27816 max_load_reduction_pct=0.848877 moved_to_echo=15662 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 256, 0, 0, 15360, 16384, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 22912, 128, 128, 9984, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[17152, 128, 128, 0, 128, 128, 128, 13440, 2176, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 4864, 128, 17408, 128, 6400, 4096, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 256, 23936, 0, 128, 0, 8704, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 0, 0, 128, 1152, 31616, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[32640, 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 1442, 0, 867, 0, 0, 0, 0, 0, 4090, 0, 5, 0, 0, 0, 2576, 3342, 0, 0, 0, 0, 0, 43, 13, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1684, 0, 0, 0, 0, 0, 3537, 366, 0, 1968, 0, 0, 4096, 0, 0, 0, 0, 0, 3873, 0, 0, 770, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 1062, 0, 46, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 3974, 3985, 2, 0, 0, 3, 0, 7, 26, 0, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 3, 1648, 0, 0, 3, 0, 0, 3983, 207, 0, 3630, 0, 0, 4067, 0, 0, 0, 0, 0, 1161, 0, 0, 774, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 16, 0, 67, 2, 0, 0, 0, 0, 4094, 0, 7, 0, 0, 1, 189, 3796, 0, 0, 9, 0, 0, 2, 345, 0, 0, 0, 330, 0, 0, 4089, 0, 0, 0, 0, 1, 3512, 1, 1, 10, 1, 0, 3853, 72, 1, 3332, 0, 0, 4084, 4, 0, 0, 0, 0, 3978, 0, 0, 970, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 1, 382, 0, 260, 0, 1, 2, 2, 1, 4094, 0, 24, 0, 0, 2, 4014, 3610, 58, 0, 0, 2, 0, 0, 1, 3, 0, 0, 668, 0, 0, 4072, 0, 0, 0, 0, 0, 411, 0, 0, 0, 0, 0, 3276, 467, 2, 452, 0, 0, 4026, 0, 0, 0, 0, 0, 4090, 3, 0, 2842, 0, 0, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 2, 2881, 0, 71, 0, 0, 0, 0, 1, 4067, 0, 0, 1, 0, 0, 716, 4083, 2, 0, 0, 56, 0, 1153, 43, 0, 0, 0, 8, 0, 0, 4095, 0, 0, 0, 0, 0, 1331, 1, 1, 0, 0, 0, 3278, 5, 0, 1473, 0, 0, 4093, 0, 0, 0, 0, 0, 4089, 0, 0, 1317, 0, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 1014, 0, 565, 0, 0, 0, 0, 0, 4091, 0, 0, 2, 0, 0, 1919, 4090, 89, 0, 0, 0, 0, 48, 27, 0, 0, 0, 66, 0, 0, 3914, 0, 0, 0, 0, 0, 1053, 0, 0, 13, 0, 0, 4095, 12, 0, 1496, 0, 0, 4088, 0, 0, 0, 0, 0, 3424, 0, 0, 2762, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 56, 0, 1354, 0, 0, 0, 0, 0, 4096, 0, 42, 0, 0, 0, 3288, 3842, 2, 0, 0, 912, 0, 116, 28, 0, 0, 0, 326, 0, 0, 4096, 0, 0, 0, 0, 6, 217, 0, 0, 0, 0, 0, 3931, 943, 0, 1926, 0, 0, 4096, 0, 0, 0, 0, 0, 3359, 0, 0, 132, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 5, 0, 514, 0, 6, 0, 0, 7, 2, 5, 4085, 0, 0, 0, 0, 2, 1033, 3723, 0, 2, 0, 160, 0, 302, 29, 3, 0, 2, 45, 5, 0, 4067, 0, 1, 0, 0, 8, 3531, 12, 0, 0, 0, 0, 4033, 0, 4, 2451, 0, 1, 4091, 2, 0, 0, 8, 5, 4016, 0, 0, 603, 1, 4, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 382, 0, 260, 0, 1, 1649, 780, 397, 0, 0, 0, 0, 0, 2, 2, 1, 4089, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4014, 1630, 58, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 668, 0, 0, 1980, 1356, 0, 0, 0, 0, 0, 0, 2423, 0, 0, 0, 0, 0, 411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1523, 467, 2, 452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 0, 0, 0, 0, 0, 9, 5, 0, 0, 0, 0, 0, 0, 3301, 3, 0, 2842, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3470, 4118, 5706, 4008, 2834, 2444, 4040, 6148] avg=4096.0000 max=6148 max_rank=7 min=2444 min_rank=5 imbalance_ratio=1.500977 max_load_reduction=26620 max_load_reduction_pct=0.812378 moved_to_echo=15466 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1442, 0, 867, 0, 0, 1659, 738, 429, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2576, 1509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 13, 0, 0, 0, 0, 0, 0, 1833, 1464, 0, 0, 0, 0, 0, 0, 2437, 0, 0, 0, 0, 0, 1684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1644, 366, 0, 1968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 9, 5, 0, 0, 0, 0, 0, 0, 3126, 0, 0, 770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5135, 4090, 4085, 3353, 4121, 3978, 4110, 3896] avg=4096.0000 max=5135 max_rank=0 min=3353 min_rank=3 imbalance_ratio=1.253662 max_load_reduction=27633 max_load_reduction_pct=0.843292 moved_to_echo=16105 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 16, 0, 67, 2, 0, 1656, 758, 467, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 189, 1714, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 345, 0, 0, 0, 330, 0, 0, 2082, 1595, 0, 0, 0, 0, 0, 0, 2433, 0, 0, 0, 0, 1, 3512, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 10, 1, 0, 1791, 72, 1, 3332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4, 0, 0, 0, 0, 9, 5, 0, 0, 0, 0, 0, 0, 3211, 0, 0, 970, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2966, 4096, 1913, 4354, 5947, 5208, 4102, 4182] avg=4096.0000 max=5947 max_rank=4 min=1913 min_rank=2 imbalance_ratio=1.451904 max_load_reduction=26821 max_load_reduction_pct=0.818512 moved_to_echo=19439 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1062, 0, 46, 0, 0, 1658, 221, 483, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3974, 1800, 2, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 26, 0, 1, 0, 0, 0, 0, 2185, 1648, 0, 0, 0, 0, 0, 0, 2437, 0, 0, 0, 0, 3, 1648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1852, 207, 0, 3630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 2, 5, 0, 0, 0, 0, 0, 0, 938, 0, 0, 774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3470, 4086, 5779, 3867, 4088, 5692, 4074, 1712] avg=4096.0000 max=5779 max_rank=2 min=1712 min_rank=7 imbalance_ratio=1.410889 max_load_reduction=26989 max_load_reduction_pct=0.823639 moved_to_echo=15566 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 2881, 0, 71, 0, 0, 1658, 779, 397, 0, 0, 0, 0, 0, 0, 0, 1, 4062, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 716, 1844, 2, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1153, 43, 0, 0, 0, 8, 0, 0, 2239, 1357, 0, 0, 0, 0, 0, 0, 2437, 0, 0, 0, 0, 0, 1331, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1524, 5, 0, 1473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 9, 5, 0, 0, 0, 0, 0, 0, 3301, 0, 0, 1317, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5788, 4064, 2618, 4800, 3769, 3003, 4107, 4619] avg=4096.0000 max=5788 max_rank=0 min=2618 min_rank=2 imbalance_ratio=1.413086 max_load_reduction=26980 max_load_reduction_pct=0.823364 moved_to_echo=15498 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 56, 0, 1354, 0, 0, 1659, 640, 477, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3288, 1735, 2, 0, 0, 912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 28, 0, 0, 0, 326, 0, 0, 2107, 1627, 0, 0, 0, 0, 0, 0, 2437, 0, 0, 0, 0, 6, 217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1827, 943, 0, 1926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 7, 5, 0, 0, 0, 0, 0, 0, 2712, 0, 0, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4186, 4133, 5937, 4204, 2660, 4696, 4108, 2844] avg=4096.0000 max=5937 max_rank=2 min=2660 min_rank=4 imbalance_ratio=1.449463 max_load_reduction=26831 max_load_reduction_pct=0.818817 moved_to_echo=14308 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1014, 0, 565, 0, 0, 1585, 652, 497, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1919, 1847, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 27, 0, 0, 0, 66, 0, 0, 2243, 1695, 0, 0, 0, 0, 0, 0, 2329, 0, 0, 0, 0, 0, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 1903, 12, 0, 1496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 8, 5, 0, 0, 0, 0, 0, 0, 2764, 0, 0, 2762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4313, 4088, 3855, 4079, 3382, 3424, 4101, 5526] avg=4096.0000 max=5526 max_rank=7 min=3382 min_rank=4 imbalance_ratio=1.349121 max_load_reduction=27242 max_load_reduction_pct=0.831360 moved_to_echo=16433 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 0, 514, 0, 6, 0, 0, 1652, 770, 493, 0, 0, 0, 0, 0, 7, 2, 5, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1033, 1678, 0, 2, 0, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 302, 29, 3, 0, 2, 45, 5, 0, 2045, 1672, 0, 0, 0, 0, 0, 0, 2415, 0, 1, 0, 0, 8, 3531, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1868, 0, 4, 2451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4091, 2, 0, 0, 8, 5, 13, 6, 0, 0, 0, 0, 0, 0, 3233, 0, 0, 603, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3440, 4093, 2875, 4103, 5967, 4323, 4126, 3841] avg=4096.0000 max=5967 max_rank=4 min=2875 min_rank=2 imbalance_ratio=1.456787 max_load_reduction=26801 max_load_reduction_pct=0.817902 moved_to_echo=18257 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 128, 7424, 0, 3328, 128, 128, 13184, 5376, 3712, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[1792, 512, 128, 128, 128, 1536, 128, 0, 16768, 12416, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 17792, 13824, 256, 128, 128, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[19456, 0, 128, 0, 0, 128, 13440, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[22656, 128, 0, 10240, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 13952, 2176, 128, 16768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 32768, 128, 0, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 32768, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[2, 0, 161, 0, 3131, 3938, 0, 0, 0, 0, 0, 0, 3981, 769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 392, 0, 0, 1815, 4096, 0, 1, 0, 4095, 0, 0, 0, 0, 4096, 0, 2620, 0, 0, 0, 75, 0, 7, 0, 17, 0, 0, 0, 0, 0, 0, 8, 3564, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[528, 0, 1, 0, 4056, 3908, 0, 0, 0, 0, 0, 0, 339, 29, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 4096, 0, 0, 1, 3773, 51, 0, 0, 0, 4094, 0, 3468, 0, 0, 0, 37, 0, 0, 0, 136, 0, 0, 0, 2, 0, 0, 107, 4076, 0, 0, 0, 0, 2, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4042, 4014, 0, 1, 0, 0, 5, 0, 315, 221, 17, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2, 0, 4087, 4094, 0, 0, 0, 3993, 22, 0, 0, 0, 4095, 0, 3979, 0, 0, 1, 0, 4, 0, 0, 21, 0, 18, 0, 12, 0, 0, 85, 3712, 1, 0, 17, 0, 0, 0, 3, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 91, 0, 4011, 2347, 0, 0, 0, 0, 0, 0, 1737, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4096, 4096, 0, 793, 0, 3956, 0, 0, 0, 0, 4096, 0, 3382, 0, 0, 0, 0, 0, 0, 0, 85, 7, 0, 0, 0, 0, 0, 53, 4002, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[9, 0, 4, 0, 3758, 4087, 0, 0, 0, 0, 0, 0, 625, 371, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4077, 4094, 0, 0, 1, 4068, 0, 0, 0, 0, 4092, 0, 3649, 0, 0, 0, 0, 0, 48, 1, 17, 17, 5, 0, 0, 3, 2, 16, 3823, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[13, 0, 2, 6, 3535, 3974, 0, 4, 15, 0, 0, 0, 108, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4091, 4088, 0, 359, 0, 3902, 72, 0, 8, 0, 4081, 0, 3191, 0, 0, 0, 88, 6, 0, 7, 0, 7, 0, 0, 0, 2, 0, 1107, 4068, 2, 14, 0, 5, 0, 0, 5, 7, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[105, 0, 0, 0, 4038, 4072, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 2, 0, 0, 1, 3, 0, 319, 0, 2, 4045, 4082, 0, 0, 0, 4095, 0, 2, 0, 0, 4078, 12, 4003, 0, 0, 0, 19, 0, 0, 0, 0, 40, 0, 0, 19, 0, 0, 8, 3815, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[10, 0, 5, 0, 4093, 4086, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4067, 4096, 0, 63, 1, 4066, 0, 0, 0, 0, 4096, 0, 3907, 1, 0, 0, 134, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 161, 0, 144, 3938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3981, 769, 0, 0, 3030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 392, 3940, 128, 3, 0, 0, 0, 0, 0, 0, 0, 1815, 0, 0, 1, 0, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 0, 2620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 7, 0, 17, 0, 2859, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3564, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4245, 7780, 4463, 1971, 3173, 3456, 3587, 4093] avg=4096.0000 max=7780 max_rank=1 min=1971 min_rank=3 imbalance_ratio=1.899414 max_load_reduction=24988 max_load_reduction_pct=0.762573 moved_to_echo=14309 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 186, 4014, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 315, 221, 17, 0, 3029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3842, 165, 3, 0, 0, 0, 0, 0, 2, 0, 4087, 0, 0, 0, 0, 151, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 553, 0, 3979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 21, 0, 3691, 498, 0, 0, 0, 0, 0, 0, 18, 0, 12, 0, 0, 85, 3712, 1, 15, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 3, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4201, 3587, 4017, 4240, 4554, 4215, 3843, 4111] avg=4096.0000 max=4554 max_rank=4 min=3587 min_rank=1 imbalance_ratio=1.111816 max_load_reduction=28214 max_load_reduction_pct=0.861023 moved_to_echo=16723 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 91, 0, 184, 2347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1737, 6, 0, 0, 3030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 3806, 164, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 793, 0, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 0, 3382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 7, 3663, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 4002, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2623, 4773, 3983, 5039, 3935, 4253, 4070, 4092] avg=4096.0000 max=5039 max_rank=3 min=2623 min_rank=0 imbalance_ratio=1.230225 max_load_reduction=27729 max_load_reduction_pct=0.846222 moved_to_echo=16350 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[9, 0, 4, 0, 172, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 625, 371, 0, 0, 3027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3914, 154, 4, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 1, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 0, 3649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 1, 17, 17, 3432, 498, 0, 0, 0, 0, 0, 0, 5, 0, 0, 3, 2, 16, 3823, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4272, 4023, 4073, 4232, 4201, 4013, 3864, 4090] avg=4096.0000 max=4272 max_rank=0 min=3864 min_rank=6 imbalance_ratio=1.042969 max_load_reduction=28496 max_load_reduction_pct=0.869629 moved_to_echo=16168 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[528, 0, 1, 0, 186, 3908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 339, 29, 0, 0, 3028, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3630, 166, 3, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 1, 143, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 553, 0, 3468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 136, 0, 3704, 498, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 107, 4076, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4623, 3396, 3801, 4206, 4072, 4375, 4200, 4095] avg=4096.0000 max=4623 max_rank=0 min=3396 min_rank=1 imbalance_ratio=1.128662 max_load_reduction=28145 max_load_reduction_pct=0.858917 moved_to_echo=16742 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[105, 0, 0, 0, 186, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 3017, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 3, 0, 319, 3940, 165, 3, 0, 0, 0, 0, 0, 0, 2, 4045, 0, 0, 0, 0, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 550, 12, 4003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 40, 3687, 496, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 8, 3815, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4363, 3025, 4433, 4202, 4567, 4242, 3857, 4079] avg=4096.0000 max=4567 max_rank=4 min=3025 min_rank=1 imbalance_ratio=1.114990 max_load_reduction=28201 max_load_reduction_pct=0.860626 moved_to_echo=16745 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 5, 0, 187, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 3030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3912, 168, 3, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 63, 1, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 0, 3907, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 1, 0, 0, 0, 0, 3738, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 4011, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4288, 3039, 4085, 4285, 4461, 4371, 4146, 4093] avg=4096.0000 max=4461 max_rank=4 min=3039 min_rank=1 imbalance_ratio=1.089111 max_load_reduction=28307 max_load_reduction_pct=0.863861 moved_to_echo=17071 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[13, 0, 2, 6, 154, 3974, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 108, 0, 0, 0, 3022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3759, 149, 4, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 359, 0, 143, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 8, 0, 534, 0, 3191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 6, 0, 7, 0, 7, 3232, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1107, 4068, 2, 22, 0, 0, 0, 0, 0, 0, 0, 14, 0, 5, 0, 0, 5, 7, 0, 4084, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4153, 3145, 3913, 4593, 3805, 3843, 5201, 4115] avg=4096.0000 max=5201 max_rank=6 min=3145 min_rank=1 imbalance_ratio=1.269775 max_load_reduction=27567 max_load_reduction_pct=0.841278 moved_to_echo=16964 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 0, 7168, 1408, 128, 0, 24320, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 384, 128, 128, 128, 384, 128, 28032, 4096, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[256, 128, 128, 0, 4480, 128, 28288, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 128, 128, 128, 128, 768, 30848, 1280, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[768, 0, 384, 128, 1408, 30464, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 30464, 0, 0, 1280, 128, 1280, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 128, 1536, 31104, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 0, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33792 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1, 2, 1, 3131, 0, 0, 553, 1, 0, 2, 0, 0, 0, 0, 44, 0, 0, 0, 5, 0, 0, 0, 3, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4069, 4081, 3450, 0, 0, 0, 0, 2844, 0, 6, 0, 2083, 3474, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4093, 0, 0, 0, 451, 376] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 561, 3, 0, 0, 0, 0, 0, 8, 0, 0, 0, 3, 0, 0, 0, 0, 0, 7, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2988, 4096, 4094, 7, 0, 0, 0, 401, 0, 0, 0, 4090, 473, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 3789, 4053] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 1802, 0, 0, 6, 34, 0, 0, 1, 0, 0, 0, 380, 0, 0, 0, 0, 0, 12, 0, 0, 4049, 1, 0, 0, 4, 48, 0, 0, 0, 0, 0, 0, 3843, 4091, 2749, 0, 0, 0, 0, 2634, 0, 0, 0, 1899, 2127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 2457, 2545] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[2, 1, 7, 1857, 2, 0, 7, 0, 0, 4, 128, 0, 7, 2, 152, 18, 0, 12, 0, 0, 0, 0, 0, 4088, 0, 8, 0, 0, 0, 1, 8, 12, 0, 0, 0, 3948, 4068, 3928, 2, 0, 3, 0, 3961, 20, 41, 0, 3034, 2330, 0, 0, 0, 7, 0, 2, 2, 9, 0, 0, 4079, 0, 0, 0, 581, 437] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 402, 8, 0, 105, 0, 0, 0, 7, 0, 0, 0, 1184, 0, 0, 0, 6, 0, 0, 0, 0, 4093, 0, 0, 0, 16, 0, 1, 0, 0, 0, 3, 0, 3209, 4088, 4082, 0, 0, 0, 0, 3305, 0, 0, 0, 4078, 150, 1, 0, 0, 0, 10, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 1015, 2916] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 1, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4, 0, 3973, 4091, 580, 20, 0, 0, 0, 4028, 0, 6, 0, 3952, 1228, 0, 8, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 2912, 3692] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1, 0, 0, 1377, 0, 0, 93, 2, 0, 0, 202, 0, 0, 2, 595, 0, 0, 0, 0, 0, 1, 6, 0, 4095, 4, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 2374, 4094, 1799, 0, 0, 0, 0, 2788, 0, 0, 0, 4021, 2461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 0, 0, 0, 2083, 2746] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 410, 0, 0, 117, 0, 0, 0, 121, 0, 0, 0, 47, 0, 0, 0, 0, 0, 6, 0, 1, 4076, 0, 0, 0, 7, 23, 0, 0, 0, 0, 0, 0, 3926, 4096, 3146, 1, 0, 0, 0, 1584, 0, 0, 0, 3913, 2076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 1606, 3526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 2, 1, 3131, 0, 0, 553, 1, 2394, 418, 44, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 44, 0, 2083, 396, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 3, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1279, 0, 3450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2426, 0, 6, 0, 0, 3474, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4081, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 451, 332, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6545, 2525, 4102, 4091, 4730, 5906, 4084, 785] avg=4096.0000 max=6545 max_rank=0 min=785 min_rank=7 imbalance_ratio=1.597900 max_load_reduction=26223 max_load_reduction_pct=0.800262 moved_to_echo=15505 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 561, 3, 0, 0, 0, 1758, 59, 478, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 3, 0, 4090, 290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 940, 0, 4094, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 342, 0, 0, 0, 0, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3789, 3571, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2859, 4391, 4107, 4094, 5041, 815, 4101, 7360] avg=4096.0000 max=7360 max_rank=7 min=815 min_rank=5 imbalance_ratio=1.796875 max_load_reduction=25408 max_load_reduction_pct=0.775391 moved_to_echo=17317 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 402, 8, 0, 105, 0, 1888, 486, 344, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 1184, 0, 4078, 312, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 1, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1009, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2819, 0, 0, 0, 0, 150, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 10, 0, 0, 0, 4088, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 2569, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3233, 5581, 4102, 4104, 5094, 2969, 4101, 3584] avg=4096.0000 max=5581 max_rank=1 min=2969 min_rank=5 imbalance_ratio=1.362549 max_load_reduction=27187 max_load_reduction_pct=0.829681 moved_to_echo=15748 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1802, 0, 0, 6, 34, 2261, 387, 300, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 380, 0, 1899, 374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4049, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 48, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1208, 0, 2749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2247, 0, 0, 0, 0, 2127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2457, 2243, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4790, 2654, 4063, 4136, 3957, 4374, 4094, 4700] avg=4096.0000 max=4790 max_rank=0 min=2654 min_rank=1 imbalance_ratio=1.169434 max_load_reduction=27978 max_load_reduction_pct=0.853821 moved_to_echo=17125 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 1377, 0, 0, 93, 2, 1396, 410, 324, 0, 0, 0, 0, 0, 0, 0, 202, 0, 0, 2, 595, 0, 4021, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 4095, 2, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 747, 0, 1799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2378, 0, 0, 0, 0, 2461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2083, 2420, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3603, 5051, 4104, 4024, 2547, 4839, 4097, 4503] avg=4096.0000 max=5051 max_rank=1 min=2547 min_rank=4 imbalance_ratio=1.233154 max_load_reduction=27717 max_load_reduction_pct=0.845856 moved_to_echo=15986 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 42, 0, 0, 0, 0, 2337, 593, 436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 1, 3952, 386, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1250, 0, 580, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3435, 0, 6, 0, 0, 1228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 1, 0, 0, 0, 4091, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2912, 3253, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3408, 4370, 4100, 4099, 1854, 4669, 4103, 6165] avg=4096.0000 max=6165 max_rank=7 min=1854 min_rank=4 imbalance_ratio=1.505127 max_load_reduction=26603 max_load_reduction_pct=0.811859 moved_to_echo=16791 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 410, 0, 0, 117, 0, 2310, 233, 416, 0, 0, 0, 0, 0, 0, 0, 121, 0, 0, 0, 47, 0, 3913, 382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1, 4076, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 23, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1234, 0, 3146, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1351, 0, 0, 0, 0, 2076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1606, 3107, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3486, 4463, 4086, 4114, 4381, 3427, 4098, 4713] avg=4096.0000 max=4713 max_rank=7 min=3427 min_rank=5 imbalance_ratio=1.150635 max_load_reduction=28055 max_load_reduction_pct=0.856171 moved_to_echo=16619 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2, 1, 7, 1857, 2, 0, 7, 0, 2326, 586, 56, 0, 0, 0, 0, 0, 0, 4, 128, 0, 7, 2, 152, 18, 3034, 388, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 4088, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 1, 8, 12, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1234, 0, 3928, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3375, 20, 41, 0, 0, 2330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 2, 2, 9, 4068, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 581, 377, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4844, 3733, 4104, 4106, 5164, 5769, 4090, 958] avg=4096.0000 max=5769 max_rank=5 min=958 min_rank=7 imbalance_ratio=1.408447 max_load_reduction=26999 max_load_reduction_pct=0.823944 moved_to_echo=15981 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 9600, 128, 0, 896, 128, 16768, 3200, 2432, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 8960, 0, 23936, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 0, 0, 14976, 17920, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 18432, 128, 128, 0, 0, 14336, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 512, 0, 128, 128, 2560, 128, 27136, 2816, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 128, 128, 128, 128, 32640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 128, 128, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33792 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4065, 0, 8, 4096, 14, 14, 152, 0, 0, 4080, 0, 0, 0, 36, 0, 0, 4092, 0, 0, 0, 4094, 0, 1, 4096, 0, 2, 0, 0, 0, 0, 0, 130, 15, 4018, 0, 0, 1, 0, 0, 0, 3391, 58, 0, 0, 0, 0, 400, 0, 0, 0, 0, 2, 1, 0, 2, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 5, 0, 4065, 0, 1, 4095, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 4095, 0, 1, 0, 4095, 1, 0, 4096, 0, 8, 0, 0, 0, 0, 0, 1265, 158, 4079, 0, 0, 0, 1, 0, 0, 2699, 5, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 1, 4083, 1, 0, 4094, 0, 0, 1, 0, 0, 3994, 0, 0, 0, 0, 1, 0, 4086, 0, 0, 0, 4094, 0, 0, 3920, 1, 16, 0, 0, 1, 0, 0, 199, 319, 3742, 0, 0, 17, 0, 4, 0, 3797, 61, 0, 1, 0, 0, 326, 0, 0, 4, 1, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 11, 3946, 0, 12, 4077, 0, 0, 0, 0, 0, 4055, 0, 13, 0, 101, 6, 0, 4084, 0, 0, 0, 4075, 0, 0, 3773, 0, 375, 11, 0, 9, 2, 0, 2459, 397, 3476, 9, 2, 0, 15, 0, 0, 1738, 17, 0, 4, 0, 0, 2, 0, 2, 35, 7, 0, 36, 0, 0, 3, 0, 7, 0, 9, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 3664, 1, 0, 4076, 0, 64, 463, 0, 0, 4096, 0, 2, 0, 28, 0, 0, 4094, 0, 0, 0, 4091, 1, 0, 4092, 6, 798, 0, 0, 0, 0, 0, 2214, 1, 2381, 2, 0, 0, 0, 0, 0, 1274, 0, 4, 0, 0, 0, 1416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4029, 0, 0, 4096, 8, 2, 16, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4066, 0, 74, 0, 0, 0, 0, 0, 2995, 17, 3998, 0, 0, 25, 0, 0, 0, 1015, 0, 0, 0, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 1, 0, 4090, 0, 0, 4096, 0, 0, 6, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 1589, 0, 0, 0, 0, 0, 81, 81, 3704, 0, 0, 1, 0, 0, 0, 1250, 26, 0, 0, 0, 0, 1464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 3707, 0, 0, 4096, 0, 50, 30, 0, 0, 4096, 0, 0, 0, 103, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4093, 0, 7, 0, 0, 0, 0, 0, 2613, 0, 4089, 0, 0, 3, 0, 0, 0, 803, 0, 0, 0, 0, 0, 886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3664, 1, 0, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 463, 0, 0, 4015, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3792, 6, 798, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2214, 1, 1596, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1274, 0, 4, 0, 0, 785, 450, 300, 81, 30, 0, 0, 0, 0, 1416, 0, 0, 0, 0, 0, 0, 3493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3798, 4544, 4089, 4597, 3813, 2924, 4909, 4094] avg=4096.0000 max=4909 max_rank=6 min=2924 min_rank=5 imbalance_ratio=1.198486 max_load_reduction=27859 max_load_reduction_pct=0.850189 moved_to_echo=15740 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 4083, 1, 0, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3633, 1, 16, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 199, 319, 2508, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3797, 61, 0, 1, 0, 1234, 452, 287, 79, 30, 0, 0, 0, 0, 326, 0, 0, 4, 1, 0, 0, 3508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4219, 3916, 4065, 3651, 3043, 5945, 3839, 4090] avg=4096.0000 max=5945 max_rank=5 min=3043 min_rank=4 imbalance_ratio=1.451416 max_load_reduction=26823 max_load_reduction_pct=0.818573 moved_to_echo=16917 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4065, 0, 8, 133, 14, 0, 0, 0, 0, 0, 0, 0, 0, 14, 152, 0, 0, 3999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3796, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 15, 2693, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3391, 58, 0, 0, 0, 1325, 453, 300, 81, 30, 0, 0, 0, 0, 400, 0, 0, 0, 0, 2, 1, 3510, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4220, 4165, 4100, 3799, 2839, 5638, 3913, 4094] avg=4096.0000 max=5638 max_rank=5 min=2839 min_rank=4 imbalance_ratio=1.376465 max_load_reduction=27130 max_load_reduction_pct=0.827942 moved_to_echo=16484 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 0, 4065, 0, 1, 134, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3796, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1265, 158, 2734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2699, 5, 0, 0, 0, 1345, 452, 300, 81, 30, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4206, 4015, 4067, 3805, 4157, 4913, 3510, 4095] avg=4096.0000 max=4913 max_rank=5 min=3510 min_rank=6 imbalance_ratio=1.199463 max_load_reduction=27855 max_load_reduction_pct=0.850067 moved_to_echo=16675 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 4090, 0, 0, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3796, 0, 1589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 81, 2483, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1250, 26, 0, 0, 0, 1221, 453, 300, 81, 30, 0, 0, 0, 0, 1464, 0, 0, 0, 0, 0, 0, 3510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4224, 4016, 4066, 5385, 2646, 3361, 4974, 4096] avg=4096.0000 max=5385 max_rank=3 min=2646 min_rank=4 imbalance_ratio=1.314697 max_load_reduction=27383 max_load_reduction_pct=0.835663 moved_to_echo=15077 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4029, 0, 0, 133, 8, 0, 0, 0, 0, 0, 0, 0, 0, 2, 16, 0, 0, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3768, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2995, 17, 2680, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 0, 0, 0, 0, 1318, 453, 298, 81, 30, 0, 0, 0, 0, 138, 0, 0, 0, 0, 0, 0, 3510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4170, 4033, 4067, 3842, 5717, 3195, 3648, 4096] avg=4096.0000 max=5717 max_rank=4 min=3195 min_rank=5 imbalance_ratio=1.395752 max_load_reduction=27051 max_load_reduction_pct=0.825531 moved_to_echo=16656 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3707, 0, 0, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 30, 0, 0, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3793, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2613, 0, 2741, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 803, 0, 0, 0, 0, 1348, 453, 300, 81, 30, 0, 0, 0, 0, 886, 0, 0, 0, 0, 0, 0, 3510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3840, 4095, 4169, 3800, 5357, 3015, 4396, 4096] avg=4096.0000 max=5357 max_rank=4 min=3015 min_rank=5 imbalance_ratio=1.307861 max_load_reduction=27411 max_load_reduction_pct=0.836517 moved_to_echo=16864 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 11, 3946, 0, 12, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3971, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 6, 0, 0, 0, 0, 0, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3494, 0, 375, 11, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2459, 397, 2327, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 1738, 17, 0, 4, 0, 1149, 454, 279, 84, 37, 0, 0, 0, 0, 2, 0, 2, 35, 7, 0, 36, 3497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 7, 0, 9, 0, 4084, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 3984, 4145, 3889, 5196, 3777, 3579, 4107] avg=4096.0000 max=5196 max_rank=4 min=3579 min_rank=6 imbalance_ratio=1.268555 max_load_reduction=27572 max_load_reduction_pct=0.841431 moved_to_echo=16659 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 0, 16000, 256, 128, 128, 0, 9728, 3712, 2432, 768, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 12032, 1024, 19840, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 128, 31744, 128, 128, 1152, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 4736, 0, 128, 128, 128, 128, 128, 28160, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 29952, 128, 2944, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 768, 0, 0, 32000, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 128, 128, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[384, 128, 0, 0, 0, 128, 0, 32512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 16, 0, 3, 0, 0, 0, 1, 0, 4083, 0, 0, 403, 2, 0, 4017, 0, 0, 0, 1, 4091, 0, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3654, 1, 4094, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 5, 0, 4053, 1, 0, 205, 838, 2, 4077, 2, 0, 0, 0, 4082, 3, 0, 917, 0, 0, 0, 0, 103, 0, 2, 1, 2, 3, 0, 0, 0, 3661, 0, 4093, 0, 0, 0, 1, 112, 0, 4005, 0, 0, 0, 0, 2514, 4078, 0, 2, 0, 0, 2, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4094, 0, 0, 41, 81, 0, 3647, 6, 1, 6, 0, 4089, 0, 0, 7, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 2, 0, 1, 3300, 0, 4092, 0, 0, 0, 0, 1261, 0, 4089, 1, 0, 0, 1, 3926, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 4, 0, 0, 4037, 0, 0, 1195, 238, 0, 3918, 0, 0, 1, 0, 4095, 0, 0, 3389, 0, 0, 0, 0, 7, 0, 0, 0, 1, 0, 0, 0, 0, 283, 0, 4096, 0, 0, 0, 0, 86, 0, 4002, 0, 0, 0, 0, 3319, 4094, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 4, 4096, 0, 0, 1664, 194, 0, 4071, 17, 0, 0, 0, 4093, 0, 0, 729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2252, 0, 4096, 0, 0, 0, 0, 89, 0, 4074, 0, 0, 0, 0, 3290, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 606, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 2947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2331, 0, 4096, 0, 0, 0, 0, 2, 0, 2664, 0, 0, 0, 0, 3738, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 3997, 0, 1, 725, 88, 15, 4028, 0, 6, 0, 0, 4076, 1, 0, 938, 0, 0, 0, 15, 6, 33, 1, 2, 7, 4, 5, 0, 0, 1584, 0, 4083, 3, 0, 0, 10, 1271, 0, 3938, 5, 12, 4, 0, 3812, 4046, 0, 0, 5, 9, 12, 9, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2, 4, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 407, 1, 0, 4077, 0, 0, 0, 0, 4093, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3209, 0, 4096, 0, 0, 0, 0, 737, 0, 3861, 0, 0, 0, 0, 4089, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 0, 0, 0, 0, 4035, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4094, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 41, 81, 0, 2973, 6, 1, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2963, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 2, 0, 2231, 1126, 651, 0, 0, 0, 0, 0, 1, 3300, 0, 1561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1261, 0, 54, 1, 0, 0, 1, 3926, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4338, 4122, 3108, 2970, 4026, 4862, 5243, 4099] avg=4096.0000 max=5243 max_rank=6 min=2970 min_rank=3 imbalance_ratio=1.280029 max_load_reduction=27525 max_load_reduction_pct=0.839996 moved_to_echo=18230 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4, 0, 0, 3952, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4053, 1, 0, 22, 0, 0, 0, 0, 0, 0, 0, 205, 838, 2, 3325, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2958, 3, 0, 917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 0, 2, 1, 2, 3, 0, 0, 2232, 1124, 727, 0, 0, 0, 0, 0, 0, 3661, 0, 1561, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 53, 0, 0, 0, 0, 2514, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 2, 0, 0, 2, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4256, 4081, 4372, 3878, 4194, 5223, 2679, 4085] avg=4096.0000 max=5223 max_rank=5 min=2679 min_rank=6 imbalance_ratio=1.275146 max_load_reduction=27545 max_load_reduction_pct=0.840607 moved_to_echo=16181 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 16, 0, 3, 4041, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4083, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 403, 2, 0, 3275, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2965, 0, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2233, 1126, 717, 0, 0, 0, 0, 0, 1, 3654, 1, 1560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4361, 4106, 3681, 3081, 4076, 5216, 4149, 4098] avg=4096.0000 max=5216 max_rank=5 min=3081 min_rank=3 imbalance_ratio=1.273438 max_load_reduction=27552 max_load_reduction_pct=0.840820 moved_to_echo=17539 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2, 0, 0, 3949, 301, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 4037, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 1195, 238, 0, 3195, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2968, 0, 0, 3389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 1, 0, 0, 0, 2234, 1127, 699, 0, 0, 0, 0, 0, 0, 283, 0, 1561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 53, 0, 0, 0, 0, 3319, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4252, 4063, 4629, 6357, 4068, 1844, 3458, 4097] avg=4096.0000 max=6357 max_rank=3 min=1844 min_rank=5 imbalance_ratio=1.552002 max_load_reduction=26411 max_load_reduction_pct=0.806000 moved_to_echo=13467 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 2628, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 606, 0, 0, 3340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2968, 0, 0, 2947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2234, 1128, 731, 0, 0, 0, 0, 0, 0, 2331, 0, 1561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 36, 0, 0, 0, 0, 3738, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2929, 4118, 3946, 5915, 4093, 3892, 3776, 4099] avg=4096.0000 max=5915 max_rank=3 min=2929 min_rank=0 imbalance_ratio=1.444092 max_load_reduction=26853 max_load_reduction_pct=0.819489 moved_to_echo=15860 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 2, 0, 4020, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 4096, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 1664, 194, 0, 3320, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2966, 0, 0, 729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2234, 1127, 726, 0, 0, 0, 0, 0, 0, 2252, 0, 1561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 54, 0, 0, 0, 0, 3290, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4323, 4123, 5195, 3695, 4087, 3813, 3433, 4099] avg=4096.0000 max=5195 max_rank=2 min=3433 min_rank=6 imbalance_ratio=1.268311 max_load_reduction=27573 max_load_reduction_pct=0.841461 moved_to_echo=15432 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2, 4, 0, 3810, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 407, 1, 0, 3325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2966, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2234, 1127, 727, 0, 0, 0, 0, 0, 0, 3209, 0, 1561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 737, 0, 51, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4117, 4115, 3733, 2968, 4089, 4770, 4877, 4099] avg=4096.0000 max=4877 max_rank=6 min=2968 min_rank=3 imbalance_ratio=1.190674 max_load_reduction=27891 max_load_reduction_pct=0.851166 moved_to_echo=17835 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 3889, 303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 3997, 0, 1, 27, 0, 0, 0, 0, 0, 0, 0, 725, 88, 15, 3270, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2950, 1, 0, 938, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 6, 33, 1, 2, 7, 4, 5, 0, 2229, 1126, 722, 0, 0, 0, 0, 0, 0, 1584, 0, 1551, 3, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 1271, 0, 49, 5, 12, 4, 0, 3812, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 0, 0, 5, 9, 12, 9, 2, 9, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4192, 4040, 4104, 3904, 4135, 3148, 5153, 4092] avg=4096.0000 max=5153 max_rank=6 min=3148 min_rank=5 imbalance_ratio=1.258057 max_load_reduction=27615 max_load_reduction_pct=0.842743 moved_to_echo=16528 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[256, 128, 128, 128, 128, 128, 128, 0, 17920, 9088, 5760, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 20352, 128, 12544, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[23808, 128, 0, 9088, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3584, 0, 512, 128, 128, 128, 128, 28800, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[5248, 1536, 128, 26112, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 128, 128, 128, 30336, 2432, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[32768, 0, 128, 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 0, 128, 128, 128, 32640, 128, 128, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33792 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1, 3618, 12, 0, 0, 5, 0, 1, 44, 3546, 1, 0, 0, 0, 0, 1, 1, 14, 0, 218, 0, 4, 0, 0, 3981, 4096, 4091, 3, 0, 0, 1, 0, 0, 0, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2628, 0, 0, 0, 4093, 1, 104, 0, 0, 0, 3818, 0, 0, 2473, 0, 0, 0, 1, 0, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 4095, 13, 3, 0, 79, 18, 0, 3, 3876, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4050, 4088, 4088, 0, 0, 0, 0, 110, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 118, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 4001, 0, 0, 4082, 0, 0, 0, 0, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 3934, 10, 1, 0, 110, 0, 0, 428, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4091, 4096, 4095, 0, 0, 0, 2, 0, 0, 0, 4, 0, 0, 0, 0, 3, 0, 0, 0, 0, 65, 0, 0, 0, 4095, 0, 481, 1, 0, 0, 4087, 0, 0, 3170, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 4032, 1, 0, 0, 93, 0, 0, 459, 2221, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 4095, 4096, 3873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 1514, 0, 0, 0, 4096, 0, 113, 0, 0, 0, 4078, 0, 0, 4066, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 4024, 0, 0, 0, 6, 0, 0, 122, 3099, 0, 4, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4085, 4096, 3726, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 16, 0, 0, 0, 0, 2184, 0, 0, 0, 4094, 0, 280, 0, 0, 0, 4094, 0, 0, 2923, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 3995, 0, 1, 0, 3, 0, 2, 393, 3836, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 12, 0, 8, 0, 4094, 4075, 4094, 0, 0, 2, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 487, 0, 3, 0, 4089, 0, 594, 0, 0, 0, 4071, 0, 0, 2995, 0, 1, 0, 0, 0, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 3645, 87, 0, 0, 93, 0, 10, 235, 3978, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 11, 0, 3908, 4057, 4084, 0, 0, 2, 34, 0, 7, 0, 3, 0, 0, 0, 0, 2, 0, 4, 16, 9, 362, 0, 0, 1, 4070, 0, 175, 31, 0, 0, 4069, 1, 25, 3706, 11, 0, 71, 7, 28, 16] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 3745, 0, 0, 0, 29, 0, 0, 121, 3633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4093, 4096, 4096, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 729, 0, 0, 0, 4096, 0, 35, 0, 0, 0, 4047, 0, 0, 3994, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 3618, 12, 0, 0, 5, 0, 1, 130, 0, 0, 0, 0, 0, 0, 0, 44, 3546, 1, 0, 0, 0, 0, 1, 300, 25, 0, 0, 0, 0, 0, 0, 1, 14, 0, 218, 0, 4, 0, 0, 4053, 13, 0, 0, 0, 0, 0, 0, 58, 0, 4091, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2628, 0, 0, 0, 2986, 43, 0, 0, 0, 0, 0, 0, 0, 1, 104, 0, 0, 0, 3663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2473, 0, 0, 0, 1, 0, 5, 637, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3767, 3917, 4303, 4153, 4087, 5657, 3768, 3116] avg=4096.0000 max=5657 max_rank=5 min=3116 min_rank=7 imbalance_ratio=1.381104 max_load_reduction=27111 max_load_reduction_pct=0.827362 moved_to_echo=16628 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 13, 3, 0, 79, 18, 0, 136, 0, 0, 0, 0, 0, 0, 0, 3, 3876, 1, 2, 0, 0, 0, 0, 306, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4045, 14, 0, 0, 0, 0, 0, 0, 59, 0, 4088, 0, 0, 0, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 0, 0, 0, 3037, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 2, 648, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4344, 4214, 4061, 4257, 4122, 3198, 3840, 4732] avg=4096.0000 max=4732 max_rank=7 min=3198 min_rank=5 imbalance_ratio=1.155273 max_load_reduction=28036 max_load_reduction_pct=0.855591 moved_to_echo=15892 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4032, 1, 0, 0, 93, 0, 0, 139, 0, 0, 0, 0, 0, 0, 0, 459, 2221, 0, 0, 0, 0, 0, 0, 309, 26, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 4053, 13, 0, 0, 0, 0, 0, 0, 60, 0, 3873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1514, 0, 0, 0, 3071, 43, 0, 0, 0, 0, 0, 0, 0, 0, 113, 0, 0, 0, 3913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 655, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4265, 3015, 4083, 3933, 4097, 4628, 4026, 4721] avg=4096.0000 max=4721 max_rank=7 min=3015 min_rank=1 imbalance_ratio=1.152588 max_load_reduction=28047 max_load_reduction_pct=0.855927 moved_to_echo=17472 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3934, 10, 1, 0, 110, 0, 0, 139, 0, 0, 0, 0, 0, 0, 0, 428, 4094, 0, 0, 0, 0, 0, 0, 309, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4053, 14, 0, 0, 0, 0, 0, 0, 60, 0, 4095, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 3, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 3068, 43, 0, 0, 0, 0, 0, 0, 0, 0, 481, 1, 0, 0, 3922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3170, 0, 0, 0, 0, 0, 0, 654, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4194, 4857, 4068, 4157, 4088, 3176, 4404, 3824] avg=4096.0000 max=4857 max_rank=1 min=3176 min_rank=5 imbalance_ratio=1.185791 max_load_reduction=27911 max_load_reduction_pct=0.851776 moved_to_echo=15492 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3995, 0, 1, 0, 3, 0, 2, 139, 0, 0, 0, 0, 0, 0, 0, 393, 3836, 0, 0, 0, 0, 1, 0, 309, 26, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 12, 0, 8, 0, 4032, 14, 0, 0, 0, 0, 0, 0, 60, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 487, 0, 3, 0, 3070, 43, 0, 0, 0, 0, 0, 0, 0, 0, 594, 0, 0, 0, 3906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2995, 0, 1, 0, 0, 0, 4, 655, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4140, 4565, 4067, 4156, 4078, 3607, 4500, 3655] avg=4096.0000 max=4565 max_rank=1 min=3607 min_rank=5 imbalance_ratio=1.114502 max_load_reduction=28203 max_load_reduction_pct=0.860687 moved_to_echo=15840 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4024, 0, 0, 0, 6, 0, 0, 139, 0, 0, 0, 0, 0, 0, 0, 122, 3099, 0, 4, 0, 0, 0, 0, 308, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4053, 13, 0, 0, 0, 0, 0, 0, 59, 0, 3726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 16, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2184, 0, 0, 0, 3064, 43, 0, 0, 0, 0, 0, 0, 0, 0, 280, 0, 0, 0, 3929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2923, 0, 1, 0, 0, 0, 0, 654, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4169, 3559, 4075, 3785, 4102, 5291, 4209, 3578] avg=4096.0000 max=5291 max_rank=5 min=3559 min_rank=1 imbalance_ratio=1.291748 max_load_reduction=27477 max_load_reduction_pct=0.838531 moved_to_echo=17180 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3745, 0, 0, 0, 29, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 121, 3633, 0, 0, 0, 0, 0, 0, 309, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4053, 14, 0, 0, 0, 0, 0, 0, 59, 0, 4096, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 729, 0, 0, 0, 3070, 43, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 3883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3994, 0, 0, 0, 0, 0, 0, 655, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3912, 4089, 4071, 4161, 4126, 3842, 3918, 4649] avg=4096.0000 max=4649 max_rank=7 min=3842 min_rank=5 imbalance_ratio=1.135010 max_load_reduction=28119 max_load_reduction_pct=0.858124 moved_to_echo=16535 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3645, 87, 0, 0, 93, 0, 10, 142, 0, 0, 0, 0, 0, 0, 0, 235, 3978, 0, 0, 0, 0, 0, 10, 298, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 4015, 14, 0, 0, 0, 0, 0, 0, 46, 0, 4084, 0, 0, 2, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 3, 0, 0, 0, 0, 2, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 4, 16, 9, 362, 0, 0, 1, 2935, 42, 0, 0, 0, 0, 0, 0, 0, 0, 175, 31, 0, 0, 3896, 1, 0, 0, 0, 0, 0, 0, 0, 0, 25, 3706, 11, 0, 71, 7, 28, 16, 629, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3977, 4552, 4040, 4166, 4068, 3369, 4103, 4493] avg=4096.0000 max=4552 max_rank=1 min=3369 min_rank=5 imbalance_ratio=1.111328 max_load_reduction=28216 max_load_reduction_pct=0.861084 moved_to_echo=16033 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 27520, 128, 128, 128, 128, 128, 128, 5248, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 8192, 0, 128, 128, 24320, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 31104, 128, 128, 0, 512, 128, 128, 1152, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[1920, 28288, 128, 128, 0, 0, 128, 128, 2560, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 256, 128, 128, 128, 0, 32384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[512, 0, 32256, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 1792, 128, 0, 0, 30976, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 0, 0, 0, 128, 32640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 0, 4096, 3974, 0, 0, 4096, 0, 6, 0, 0, 0, 1, 59, 0, 0, 4088, 1550, 0, 0, 0, 0, 0, 0, 4, 0, 47, 0, 2601, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2472, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 4096, 3372, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 193, 0, 0, 0, 0, 171, 0, 0, 0, 237, 0, 3991, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 190, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4087, 4, 4091, 3751, 0, 0, 4091, 3, 1, 2, 0, 0, 0, 2, 0, 0, 4094, 24, 0, 0, 0, 0, 32, 0, 10, 2, 1, 0, 3938, 4067, 2, 0, 0, 0, 0, 8, 0, 0, 0, 488, 0, 0, 5, 36, 0, 2, 0, 0, 2, 0, 0, 40, 0, 2, 0, 0, 0, 0, 3983, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[72, 0, 1, 0, 4091, 0, 4090, 3180, 0, 0, 4095, 1, 0, 1, 0, 0, 0, 81, 0, 10, 4053, 508, 0, 0, 0, 1, 386, 0, 0, 0, 24, 0, 3928, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 1, 0, 0, 4072, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 4094, 3729, 0, 0, 4096, 0, 0, 0, 3, 1, 1, 0, 0, 0, 4095, 2822, 0, 0, 0, 0, 3, 0, 2, 0, 0, 0, 3214, 4077, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 2516, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3989, 0, 4066, 3812, 0, 0, 4066, 6, 0, 11, 0, 0, 0, 121, 12, 0, 4070, 2243, 0, 0, 141, 0, 96, 0, 0, 6, 3, 9, 1347, 3853, 8, 5, 0, 0, 0, 68, 0, 0, 0, 1120, 0, 1, 37, 1, 0, 8, 19, 0, 12, 0, 20, 25, 12, 1, 0, 0, 0, 0, 3580, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 4096, 3736, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 140, 0, 0, 4096, 1333, 0, 0, 0, 0, 23, 0, 0, 0, 38, 0, 3888, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3129, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[13, 0, 0, 0, 4094, 0, 4096, 2564, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 3, 0, 0, 4096, 2316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3343, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[72, 0, 1, 0, 572, 0, 0, 3180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 10, 2648, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 386, 0, 0, 0, 24, 0, 3519, 420, 13, 0, 0, 0, 0, 0, 3928, 809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 0, 2849, 804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 4072, 0, 601, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3825, 4097, 3247, 4363, 4737, 3744, 4080, 4675] avg=4096.0000 max=4737 max_rank=4 min=3247 min_rank=2 imbalance_ratio=1.156494 max_load_reduction=28031 max_load_reduction_pct=0.855438 moved_to_echo=17236 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 572, 0, 0, 3974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 59, 0, 0, 2671, 1550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 47, 0, 3520, 422, 14, 0, 0, 0, 0, 0, 2601, 812, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1586, 0, 0, 0, 0, 2862, 811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2472, 0, 606, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4546, 4102, 4281, 4007, 3413, 5259, 4082, 3078] avg=4096.0000 max=5259 max_rank=5 min=3078 min_rank=7 imbalance_ratio=1.283936 max_load_reduction=27509 max_load_reduction_pct=0.839508 moved_to_echo=15832 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 572, 4, 0, 3751, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2675, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 10, 2, 1, 0, 3515, 419, 13, 0, 0, 0, 0, 0, 3938, 806, 2, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 0, 0, 5, 36, 2842, 812, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 40, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3983, 0, 607, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4327, 4097, 2701, 3992, 4754, 4183, 4122, 4592] avg=4096.0000 max=4754 max_rank=4 min=2701 min_rank=2 imbalance_ratio=1.160645 max_load_reduction=28014 max_load_reduction_pct=0.854919 moved_to_echo=17651 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 573, 0, 0, 3372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2677, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 171, 0, 0, 0, 237, 0, 3522, 422, 13, 0, 0, 0, 0, 0, 3991, 812, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 190, 0, 40, 0, 0, 2861, 812, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 607, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3945, 4096, 2870, 4365, 4803, 3903, 4083, 4703] avg=4096.0000 max=4803 max_rank=4 min=2870 min_rank=2 imbalance_ratio=1.172607 max_load_reduction=27965 max_load_reduction_pct=0.853424 moved_to_echo=17492 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 572, 0, 0, 3729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2676, 2822, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 0, 0, 3522, 420, 13, 0, 0, 0, 0, 0, 3214, 808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 2849, 812, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 18, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2516, 0, 607, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4301, 4100, 5499, 3960, 4022, 3663, 4100, 3123] avg=4096.0000 max=5499 max_rank=2 min=3123 min_rank=7 imbalance_ratio=1.342529 max_load_reduction=27269 max_load_reduction_pct=0.832184 moved_to_echo=14908 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[13, 0, 0, 0, 572, 0, 0, 2564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2677, 2316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3522, 422, 14, 0, 0, 0, 0, 0, 3343, 812, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2862, 812, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 607, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3149, 4098, 4996, 3958, 4155, 3674, 4082, 4656] avg=4096.0000 max=4996 max_rank=2 min=3149 min_rank=0 imbalance_ratio=1.219727 max_load_reduction=27772 max_load_reduction_pct=0.847534 moved_to_echo=16567 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 573, 0, 0, 3736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 2677, 1333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 38, 0, 3523, 422, 13, 0, 0, 0, 0, 0, 3888, 811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2861, 812, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3129, 0, 607, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4309, 4098, 4150, 4019, 4699, 3674, 4083, 3736] avg=4096.0000 max=4699 max_rank=4 min=3674 min_rank=5 imbalance_ratio=1.147217 max_load_reduction=28069 max_load_reduction_pct=0.856598 moved_to_echo=16192 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 554, 0, 0, 3812, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 6, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, 12, 0, 2648, 2243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 0, 96, 0, 0, 6, 3, 9, 3435, 400, 14, 0, 0, 0, 0, 0, 1347, 757, 8, 5, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1120, 0, 1, 37, 1, 2696, 813, 0, 0, 0, 0, 0, 0, 0, 8, 19, 0, 12, 0, 20, 25, 4052, 0, 0, 0, 0, 0, 0, 0, 12, 1, 0, 0, 0, 0, 3580, 0, 609, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4366, 4080, 5024, 4104, 2185, 4668, 4136, 4205] avg=4096.0000 max=5024 max_rank=2 min=2185 min_rank=4 imbalance_ratio=1.226562 max_load_reduction=27744 max_load_reduction_pct=0.846680 moved_to_echo=15194 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 512, 128, 128, 21376, 11008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[26368, 6528, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 0, 3584, 128, 128, 128, 128, 22784, 6528, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 0, 4608, 128, 0, 28160, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 0, 0, 27904, 0, 4864, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[256, 128, 768, 0, 128, 128, 384, 128, 28160, 3456, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 128, 128, 128, 128, 32640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 0, 32768, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[28, 2872, 0, 3933, 2, 0, 0, 0, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4076, 3687, 1, 1, 1, 0, 1, 0, 0, 15, 0, 0, 0, 0, 7, 3, 0, 0, 4, 2, 0, 0, 0, 0, 3484, 0, 197, 0, 0, 4095, 0, 44, 0, 0, 0, 0, 0, 0, 10, 42, 1, 617, 0, 1508, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1, 2850, 0, 2268, 0, 0, 0, 0, 0, 3328, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4088, 4056, 0, 0, 0, 0, 1771, 0, 0, 7, 1, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2453, 0, 2936, 0, 0, 4095, 0, 10, 0, 0, 0, 4, 0, 0, 0, 8, 0, 0, 0, 754, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1, 827, 1, 4093, 0, 1, 0, 0, 2, 3452, 0, 0, 0, 0, 0, 2, 0, 4085, 1, 4076, 4092, 0, 1, 2, 0, 266, 0, 2, 0, 0, 0, 8, 0, 19, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2954, 0, 3111, 2, 0, 4086, 0, 0, 0, 0, 0, 2, 3, 0, 0, 8, 0, 0, 0, 1669, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 2308, 0, 2594, 0, 0, 0, 0, 0, 3040, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3919, 0, 0, 0, 0, 39, 0, 0, 2, 0, 0, 0, 0, 22, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3284, 1, 3101, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 10, 0, 86, 0, 2072, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 3885, 0, 3204, 0, 0, 0, 0, 0, 3419, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3056, 0, 0, 0, 0, 426, 0, 0, 0, 0, 0, 0, 0, 742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1644, 0, 4090, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 6, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 2897, 9, 4053, 0, 27, 0, 0, 14, 710, 5, 0, 5, 0, 0, 16, 0, 4054, 0, 3991, 4075, 0, 280, 0, 5, 679, 0, 8, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 15, 0, 0, 3778, 0, 466, 14, 21, 4074, 0, 0, 0, 0, 0, 0, 11, 0, 423, 0, 6, 0, 0, 3125, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[5, 1930, 0, 4027, 0, 0, 0, 0, 0, 257, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3942, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3938, 0, 3605, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 15, 0, 2532, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 1414, 0, 3316, 0, 0, 0, 0, 0, 3677, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 2870, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2795, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 33, 42, 0, 162, 0, 1967, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[28, 2872, 0, 1962, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 0, 0, 0, 0, 0, 0, 1540, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 349, 3687, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 15, 0, 0, 0, 3552, 130, 0, 0, 0, 0, 0, 0, 0, 7, 3, 0, 0, 4, 2, 0, 3961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1592, 0, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 42, 1, 617, 0, 1508, 0, 1971, 352, 175, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4864, 5591, 4039, 3698, 3977, 1789, 4134, 4676] avg=4096.0000 max=5591 max_rank=1 min=1789 min_rank=5 imbalance_ratio=1.364990 max_load_reduction=27177 max_load_reduction_pct=0.829376 moved_to_echo=14576 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 2850, 0, 1132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3328, 0, 0, 0, 0, 0, 0, 1084, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 350, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1771, 0, 0, 7, 1, 0, 0, 3563, 130, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 3966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1121, 0, 2936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 10, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 754, 0, 1136, 248, 175, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3983, 4417, 4406, 5472, 4008, 4057, 4104, 2321] avg=4096.0000 max=5472 max_rank=3 min=2321 min_rank=7 imbalance_ratio=1.335938 max_load_reduction=27296 max_load_reduction_pct=0.833008 moved_to_echo=14490 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 827, 1, 2042, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3452, 0, 0, 0, 0, 0, 2, 1305, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 349, 4092, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, 0, 2, 0, 0, 0, 8, 3552, 130, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 3955, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1350, 0, 3111, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 1669, 0, 2051, 299, 175, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2872, 4765, 4445, 3958, 3974, 4465, 4087, 4202] avg=4096.0000 max=4765 max_rank=1 min=2872 min_rank=0 imbalance_ratio=1.163330 max_load_reduction=28003 max_load_reduction_pct=0.854584 moved_to_echo=16728 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 3885, 0, 1599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3419, 0, 0, 0, 0, 0, 0, 726, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 350, 3056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 426, 0, 0, 0, 0, 0, 0, 3570, 129, 0, 0, 0, 0, 0, 0, 0, 742, 0, 0, 0, 0, 0, 0, 3967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 752, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 6, 0, 1605, 166, 176, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5484, 4150, 3406, 4125, 4709, 4842, 4091, 1961] avg=4096.0000 max=5484 max_rank=0 min=1961 min_rank=7 imbalance_ratio=1.338867 max_load_reduction=27284 max_load_reduction_pct=0.832642 moved_to_echo=15603 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 2308, 0, 1294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3040, 0, 0, 0, 0, 0, 0, 1451, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 350, 3919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 2, 0, 0, 0, 3570, 130, 0, 0, 0, 0, 0, 0, 0, 22, 0, 1, 0, 0, 0, 0, 3966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1501, 1, 3101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 10, 0, 86, 0, 2072, 0, 1300, 332, 176, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3602, 4496, 4269, 3741, 3989, 4603, 4091, 3977] avg=4096.0000 max=4603 max_rank=5 min=3602 min_rank=0 imbalance_ratio=1.123779 max_load_reduction=28165 max_load_reduction_pct=0.859528 moved_to_echo=16660 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[5, 1930, 0, 2009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 257, 0, 0, 0, 0, 0, 0, 1740, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 350, 3942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 3570, 130, 0, 0, 0, 0, 0, 0, 0, 205, 0, 0, 0, 0, 0, 0, 3966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1800, 0, 3605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 15, 0, 2532, 0, 2018, 398, 176, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3944, 2002, 4292, 3711, 4171, 5405, 4091, 5152] avg=4096.0000 max=5405 max_rank=5 min=2002 min_rank=1 imbalance_ratio=1.319580 max_load_reduction=27363 max_load_reduction_pct=0.835052 moved_to_echo=18819 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1414, 0, 1654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3677, 0, 0, 0, 0, 0, 0, 1235, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 350, 2870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 3570, 130, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 3966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1278, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 42, 0, 162, 0, 1967, 0, 1662, 282, 176, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3068, 4917, 3220, 3761, 4014, 5373, 4091, 4324] avg=4096.0000 max=5373 max_rank=5 min=3068 min_rank=0 imbalance_ratio=1.311768 max_load_reduction=27395 max_load_reduction_pct=0.836029 moved_to_echo=17802 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 2897, 9, 2018, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 710, 5, 0, 5, 0, 0, 16, 1674, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 336, 4075, 0, 280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 679, 0, 8, 0, 0, 0, 0, 3481, 129, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3925, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 1718, 0, 466, 14, 21, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 423, 0, 6, 0, 0, 3125, 6, 2035, 386, 174, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4951, 2430, 4691, 4302, 3926, 2234, 4079, 6155] avg=4096.0000 max=6155 max_rank=7 min=2234 min_rank=5 imbalance_ratio=1.502686 max_load_reduction=26613 max_load_reduction_pct=0.812164 moved_to_echo=16394 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 512, 128, 128, 896, 0, 13696, 128, 13824, 2560, 1408, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 22016, 128, 0, 128, 0, 0, 128, 10880, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 19072, 128, 13824, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 0, 11136, 128, 21632, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 3328, 0, 128, 128, 128, 0, 128, 28544, 1152, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 0, 128, 2816, 29824, 128, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 1152, 128, 128, 0, 128, 128, 0, 31744, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 21, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 0, 4096, 4071, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 58, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 188, 4096, 3814, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[2, 0, 0, 252, 0, 3, 343, 36, 0, 0, 0, 0, 0, 3, 1, 0, 2, 0, 0, 2, 0, 4, 0, 4093, 2, 0, 4026, 4076, 0, 0, 0, 2, 0, 4082, 0, 0, 1, 22, 0, 2, 0, 2, 0, 4080, 0, 3, 1, 0, 3, 4094, 3568, 0, 4, 4, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 0, 736, 3, 0, 1, 0, 0, 1, 0, 0, 0, 0, 7, 1, 0, 0, 7, 0, 4087, 0, 0, 4072, 4096, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 2, 0, 0, 27, 0, 0, 4094, 0, 0, 0, 8, 1784, 4095, 1553, 0, 0, 0, 0, 4094, 0, 2, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1267, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 4087, 4091, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 9, 0, 3, 4096, 2793, 0, 0, 0, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 50, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 4091, 0, 0, 4081, 4096, 3, 3, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 4093, 4044, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1815, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 4048, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 1466, 4096, 883, 0, 0, 0, 0, 4033, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 96, 0, 0, 2101, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 4095, 0, 0, 4090, 4080, 1, 1, 0, 0, 0, 4034, 0, 0, 0, 28, 0, 0, 4, 0, 0, 4093, 0, 0, 0, 0, 16, 4094, 1955, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[9, 5, 0, 3, 0, 16, 324, 3, 0, 0, 0, 0, 33, 18, 0, 0, 0, 0, 0, 0, 0, 19, 0, 4075, 4, 2, 4065, 4048, 0, 0, 16, 14, 26, 3990, 0, 1, 12, 68, 7, 18, 18, 0, 0, 3943, 0, 0, 0, 0, 0, 4078, 3881, 0, 0, 0, 4, 4035, 7, 0, 0, 0, 3, 9, 14, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 0, 736, 3, 3208, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 4058, 25, 1, 0, 0, 0, 0, 0, 0, 7, 1, 0, 0, 7, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 4094, 0, 0, 0, 8, 4, 4, 3, 0, 0, 0, 0, 0, 1784, 0, 1553, 0, 0, 0, 0, 857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3949, 4086, 4099, 4111, 4093, 4140, 4194, 4096] avg=4096.0000 max=4194 max_rank=6 min=3949 min_rank=0 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16523 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 252, 0, 3, 343, 36, 3178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 4012, 24, 1, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 4, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 14, 4076, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 1, 22, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4080, 0, 3, 1, 0, 4, 4, 3, 0, 0, 0, 0, 0, 3, 0, 3568, 0, 4, 4, 0, 849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3814, 4041, 4098, 4094, 4103, 4097, 4428, 4093] avg=4096.0000 max=4428 max_rank=6 min=3814 min_rank=0 imbalance_ratio=1.081055 max_load_reduction=28340 max_load_reduction_pct=0.864868 moved_to_echo=16721 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 21, 0, 0, 57, 0, 3210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 25, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4, 4, 3, 0, 0, 0, 0, 0, 188, 0, 3814, 0, 0, 0, 0, 857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3288, 4108, 4102, 4085, 4125, 4106, 4859, 4095] avg=4096.0000 max=4859 max_rank=6 min=3288 min_rank=0 imbalance_ratio=1.186279 max_load_reduction=27909 max_load_reduction_pct=0.851715 moved_to_echo=17185 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1267, 86, 3168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 24, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4091, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 9, 0, 4, 4, 3, 0, 0, 0, 0, 0, 3, 0, 2793, 0, 0, 0, 0, 846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4521, 4098, 4095, 4108, 4093, 4116, 3642, 4095] avg=4096.0000 max=4521 max_rank=0 min=3642 min_rank=6 imbalance_ratio=1.103760 max_load_reduction=28247 max_load_reduction_pct=0.862030 moved_to_echo=15946 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1815, 42, 3161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 24, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 4, 4, 3, 0, 0, 0, 0, 0, 1466, 0, 883, 0, 0, 0, 0, 844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5018, 4106, 4093, 4062, 4092, 4109, 3193, 4095] avg=4096.0000 max=5018 max_rank=0 min=3193 min_rank=6 imbalance_ratio=1.225098 max_load_reduction=27750 max_load_reduction_pct=0.846863 moved_to_echo=15489 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 50, 2, 3209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 25, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 4, 3, 0, 0, 0, 0, 0, 5, 0, 4044, 0, 0, 0, 0, 857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3261, 4093, 4102, 4116, 4091, 4107, 4906, 4092] avg=4096.0000 max=4906 max_rank=6 min=3261 min_rank=0 imbalance_ratio=1.197754 max_load_reduction=27862 max_load_reduction_pct=0.850281 moved_to_echo=17196 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 96, 0, 0, 2101, 1, 3193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4076, 24, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4080, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4030, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4093, 0, 0, 0, 0, 4, 4, 3, 0, 0, 0, 0, 0, 16, 0, 1955, 0, 0, 0, 0, 854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5391, 4102, 4095, 4096, 4058, 4108, 2825, 4093] avg=4096.0000 max=5391 max_rank=0 min=2825 min_rank=6 imbalance_ratio=1.316162 max_load_reduction=27377 max_load_reduction_pct=0.835480 moved_to_echo=15084 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[9, 5, 0, 3, 0, 16, 324, 3, 3166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 18, 0, 0, 4053, 28, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 12, 4048, 0, 0, 16, 14, 0, 0, 0, 0, 0, 0, 0, 0, 26, 3981, 0, 1, 12, 68, 7, 18, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 3943, 0, 0, 0, 0, 9, 5, 10, 0, 0, 0, 0, 0, 0, 0, 3881, 0, 0, 0, 4, 836, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 3, 9, 14, 0, 4076, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3526, 4134, 4084, 4096, 4113, 3985, 4721, 4109] avg=4096.0000 max=4721 max_rank=6 min=3526 min_rank=0 imbalance_ratio=1.152588 max_load_reduction=28047 max_load_reduction_pct=0.855927 moved_to_echo=16928 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 0, 384, 128, 128, 6784, 256, 25600, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3584, 0, 22528, 0, 128, 128, 128, 6912, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 0, 32640, 0, 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 32640, 0, 128, 128, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 0, 0, 128, 128, 128, 128, 32512, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 0, 0, 128, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 32640, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4085, 2, 0, 4083, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 3971, 4096, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 149, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 5, 0, 0, 4096, 5, 0, 4087, 0, 0, 0, 0, 0, 4095, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4093, 4090, 0, 0, 0, 4072, 1, 0, 0, 0, 0, 0, 30, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 4094, 4, 0, 4078, 0, 311, 0, 0, 0, 4058, 2, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 1, 3299, 4092, 0, 0, 0, 4095, 38, 0, 1, 0, 0, 0, 459, 0, 4091, 6, 1, 0, 28, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 4096, 0, 0, 0, 4074, 44, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 123, 0, 0, 2, 3914, 0, 5, 4063, 0, 111, 0, 0, 0, 4081, 0, 0, 30, 21, 0, 0, 0, 0, 0, 0, 0, 16, 3, 4042, 4072, 0, 0, 24, 3498, 494, 0, 15, 0, 0, 1, 10, 0, 4081, 1, 20, 0, 13, 0, 0, 0, 0, 4075, 4, 16, 0, 3, 11, 0, 0, 12, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 2, 0, 0, 0, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3204, 4096, 0, 0, 0, 4096, 219, 0, 0, 0, 0, 0, 678, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 2, 0, 0, 4096, 0, 0, 4096, 0, 5, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3767, 4095, 0, 0, 0, 4096, 271, 0, 0, 0, 0, 0, 53, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 4085, 1, 0, 4093, 0, 12, 0, 0, 0, 4093, 0, 0, 0, 4, 0, 1, 0, 5, 0, 0, 0, 2, 0, 4049, 4093, 0, 0, 1, 3989, 126, 0, 3, 0, 0, 2, 0, 0, 4093, 45, 0, 0, 0, 0, 0, 0, 0, 4059, 2, 2, 3, 2, 2, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 4068, 3, 0, 0, 0, 0, 0, 0, 4063, 4, 0, 0, 0, 311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 1, 3299, 179, 61, 30, 10, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 38, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 459, 0, 3912, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 4095, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 4089, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4378, 4058, 3592, 4073, 4378, 4126, 4091] avg=4096.0000 max=4378 max_rank=1 min=3592 min_rank=3 imbalance_ratio=1.068848 max_load_reduction=28390 max_load_reduction_pct=0.866394 moved_to_echo=16668 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4073, 3, 0, 0, 0, 0, 0, 0, 4054, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 3971, 179, 61, 30, 10, 0, 0, 0, 0, 0, 0, 0, 0, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 0, 3916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4056, 4093, 4256, 4030, 4065, 4099, 4093] avg=4096.0000 max=4256 max_rank=3 min=4030 min_rank=4 imbalance_ratio=1.039062 max_load_reduction=28512 max_load_reduction_pct=0.870117 moved_to_echo=16287 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 5, 0, 0, 4076, 2, 0, 0, 0, 0, 0, 0, 4065, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4093, 179, 61, 30, 11, 0, 0, 0, 0, 0, 0, 0, 0, 4011, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 3917, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4070, 4095, 4375, 4012, 3948, 4097, 4088] avg=4096.0000 max=4375 max_rank=3 min=3948 min_rank=5 imbalance_ratio=1.068115 max_load_reduction=28393 max_load_reduction_pct=0.866486 moved_to_echo=16145 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 4085, 2, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 179, 61, 30, 11, 0, 0, 0, 0, 0, 0, 0, 0, 4013, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4064, 4106, 4342, 4057, 3918, 4099, 4094] avg=4096.0000 max=4342 max_rank=3 min=3918 min_rank=5 imbalance_ratio=1.060059 max_load_reduction=28426 max_load_reduction_pct=0.867493 moved_to_echo=16168 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4086, 3, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3204, 179, 61, 30, 10, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 678, 0, 3917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4066, 4101, 3484, 4254, 4595, 4086, 4093] avg=4096.0000 max=4595 max_rank=5 min=3484 min_rank=3 imbalance_ratio=1.121826 max_load_reduction=28173 max_load_reduction_pct=0.859772 moved_to_echo=17028 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 2, 0, 0, 4086, 3, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3767, 179, 61, 30, 10, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 3917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4070, 4092, 4047, 4306, 3970, 4099, 4092] avg=4096.0000 max=4306 max_rank=4 min=3970 min_rank=5 imbalance_ratio=1.051270 max_load_reduction=28462 max_load_reduction_pct=0.868591 moved_to_echo=16467 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 4083, 3, 0, 0, 0, 0, 0, 0, 4054, 1, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 2, 0, 4049, 179, 59, 30, 10, 0, 0, 0, 0, 0, 0, 0, 1, 3930, 126, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3914, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 2, 2, 1, 0, 0, 0, 0, 0, 0, 2, 3, 2, 2, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4067, 4096, 4334, 4060, 3961, 4064, 4099] avg=4096.0000 max=4334 max_rank=3 min=3961 min_rank=5 imbalance_ratio=1.058105 max_load_reduction=28434 max_load_reduction_pct=0.867737 moved_to_echo=16184 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 123, 0, 0, 2, 4053, 3, 0, 0, 0, 0, 0, 0, 3881, 0, 5, 0, 0, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 30, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 3, 4042, 180, 55, 32, 10, 0, 0, 0, 0, 0, 0, 0, 24, 3443, 494, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 10, 0, 3901, 1, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 4075, 4, 5, 1, 0, 0, 0, 0, 0, 0, 16, 0, 3, 11, 0, 0, 12, 7, 4069, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4181, 3997, 4127, 4338, 3976, 3933, 4098, 4118] avg=4096.0000 max=4338 max_rank=3 min=3933 min_rank=5 imbalance_ratio=1.059082 max_load_reduction=28430 max_load_reduction_pct=0.867615 moved_to_echo=16125 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 128, 1408, 0, 31360, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 0, 128, 128, 128, 0, 128, 32640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32384, 128, 128, 0, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 0, 128, 128, 30592, 1536, 512, 256, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 0, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 31616, 1280, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 32768, 128, 128, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 0, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 4086, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 12, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4087, 4012, 0, 7, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 4093, 20, 0, 0, 0, 0, 4, 4090, 0, 6, 2, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 4026, 0, 3, 2, 1, 0, 0, 2, 0, 0, 1, 0, 0, 4093, 3, 0, 2, 0, 0, 0, 4092, 3995, 3, 103, 0, 0, 0, 0, 53, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 4095, 1, 0, 0, 0, 1, 0, 4094, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 2, 3, 0, 0, 3691, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 4056, 0, 0, 1, 0, 0, 0, 4095, 4084, 0, 0, 0, 1, 0, 0, 405, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4095, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 4041, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 4018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 4096, 4096, 0, 50, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 4069, 185, 1, 0, 0, 7, 0, 4065, 0, 1, 27, 0, 2, 4067, 0, 0, 0, 0, 0, 1, 8, 3882, 0, 0, 0, 16, 0, 0, 2, 0, 0, 0, 0, 0, 4049, 26, 6, 26, 0, 33, 0, 4073, 3832, 16, 0, 0, 16, 7, 17, 187, 0, 0, 2, 34, 0, 0, 36, 0, 0, 0, 3, 4072, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 4095, 4094, 0, 21, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 4091, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 50, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4081, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 3691, 0, 85, 14, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 4093, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 4084, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 405, 29, 0, 0, 0, 0, 3979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4097, 3795, 4097, 4060, 4114, 4413, 4109] avg=4096.0000 max=4413 max_rank=6 min=3795 min_rank=2 imbalance_ratio=1.077393 max_load_reduction=28355 max_load_reduction_pct=0.865326 moved_to_echo=16696 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4079, 20, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 2, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 0, 85, 14, 0, 0, 0, 0, 0, 0, 3, 2, 1, 0, 0, 2, 0, 0, 4089, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4093, 3, 0, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 3995, 3, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 4, 0, 0, 0, 3976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4091, 4125, 4099, 4102, 4130, 4033, 4085] avg=4096.0000 max=4130 max_rank=5 min=4033 min_rank=6 imbalance_ratio=1.008301 max_load_reduction=28638 max_load_reduction_pct=0.873962 moved_to_echo=16350 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4096, 0, 85, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 4012, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 3971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4096, 4207, 4097, 4097, 4049, 4055, 4095] avg=4096.0000 max=4207 max_rank=2 min=4049 min_rank=5 imbalance_ratio=1.027100 max_load_reduction=28561 max_load_reduction_pct=0.871613 moved_to_echo=16296 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4082, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 85, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4097, 4195, 4097, 4099, 4116, 3982, 4098] avg=4096.0000 max=4195 max_rank=2 min=3982 min_rank=6 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16295 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4018, 0, 85, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 4096, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 3980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4027, 4097, 4118, 4097, 4100, 4175, 4061, 4093] avg=4096.0000 max=4175 max_rank=5 min=4027 min_rank=0 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 85, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 7, 0, 0, 0, 0, 3979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4096, 4144, 4098, 4099, 4125, 4036, 4093] avg=4096.0000 max=4144 max_rank=2 min=4036 min_rank=6 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 85, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 4094, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4097, 4173, 4097, 4100, 4144, 3981, 4095] avg=4096.0000 max=4173 max_rank=2 min=3981 min_rank=6 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16320 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4048, 185, 1, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 27, 0, 2, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8, 3882, 0, 92, 21, 7, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 2, 0, 0, 4065, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 26, 6, 26, 0, 4, 0, 0, 0, 0, 0, 0, 0, 33, 0, 18, 3832, 16, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 7, 17, 187, 0, 0, 2, 34, 0, 3960, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 3, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4241, 4097, 4011, 4086, 4111, 3915, 4207, 4100] avg=4096.0000 max=4241 max_rank=0 min=3915 min_rank=5 imbalance_ratio=1.035400 max_load_reduction=28527 max_load_reduction_pct=0.870575 moved_to_echo=16333 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 256, 32384, 128, 256, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 32640, 256, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 0, 128, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 768, 256, 128, 128, 128, 0, 31872, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 0, 0, 32768, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 0, 128, 128, 128, 128, 32000, 0, 768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True + [2026-09-21 08:25:38] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 262.0 | throughput per GPU (TFLOP/s/GPU): 123.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.720596E+00 | loss scale: 1.0 | grad norm: 4.095 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[177, 329, 858, 0, 0, 635, 803, 509, 2, 719, 16, 1, 18, 101, 234, 99, 4, 773, 853, 1961, 25, 103, 193, 3, 94, 7, 67, 287, 816, 374, 1585, 1402, 523, 413, 548, 1550, 125, 1116, 896, 899, 1137, 615, 1413, 54, 1186, 1010, 10, 13, 916, 545, 622, 32, 935, 861, 1439, 69, 1384, 11, 249, 9, 4, 5, 1098, 33] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[94, 112, 1433, 0, 0, 641, 817, 356, 0, 1045, 30, 5, 9, 4, 504, 27, 1, 859, 1200, 1846, 28, 51, 254, 2, 57, 20, 57, 199, 748, 757, 864, 1564, 926, 816, 455, 802, 204, 1100, 460, 791, 1638, 554, 2165, 60, 468, 705, 14, 67, 682, 579, 494, 53, 397, 1609, 1965, 29, 1288, 34, 229, 2, 20, 27, 541, 10] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[125, 240, 894, 1, 1, 483, 763, 415, 1, 571, 36, 2, 57, 3, 517, 84, 2, 1181, 586, 1970, 31, 269, 201, 0, 34, 21, 21, 416, 848, 1101, 1180, 1647, 764, 510, 485, 1143, 136, 1252, 420, 692, 1579, 827, 1382, 70, 671, 767, 22, 41, 1168, 547, 679, 48, 686, 1423, 1587, 43, 1089, 30, 416, 2, 14, 12, 543, 19] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[129, 100, 1138, 0, 0, 673, 736, 452, 3, 462, 44, 1, 7, 33, 214, 111, 3, 1014, 756, 2209, 13, 239, 255, 12, 51, 2, 76, 351, 1058, 608, 1056, 2002, 1235, 398, 553, 925, 152, 720, 525, 1068, 1353, 479, 1708, 71, 784, 567, 6, 27, 1221, 558, 647, 31, 807, 1026, 1653, 22, 978, 13, 188, 3, 6, 6, 1223, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[220, 397, 616, 0, 0, 848, 927, 442, 1, 583, 20, 0, 7, 67, 252, 131, 1, 721, 711, 1724, 23, 377, 242, 30, 100, 8, 48, 317, 1270, 464, 1010, 1577, 524, 439, 701, 1180, 88, 1233, 699, 1037, 1478, 661, 1583, 28, 1266, 932, 16, 56, 952, 535, 237, 25, 541, 1443, 1600, 58, 1112, 36, 309, 6, 28, 15, 797, 19] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[108, 238, 1210, 3, 1, 698, 813, 669, 13, 918, 32, 1, 15, 8, 188, 106, 9, 801, 543, 1856, 19, 121, 243, 6, 108, 11, 123, 401, 941, 428, 826, 1469, 623, 480, 520, 1348, 214, 1421, 692, 711, 1507, 558, 1589, 67, 759, 1004, 26, 20, 669, 746, 446, 51, 842, 1543, 1848, 12, 1107, 41, 306, 14, 50, 24, 589, 15] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[44, 271, 925, 1, 0, 957, 741, 252, 0, 954, 35, 1, 13, 52, 530, 44, 3, 812, 932, 1690, 55, 146, 261, 4, 92, 28, 92, 233, 1347, 370, 1089, 1789, 638, 336, 379, 1193, 259, 845, 816, 1042, 1790, 326, 1824, 51, 648, 868, 18, 25, 738, 745, 728, 21, 667, 1238, 1561, 30, 1333, 54, 297, 2, 7, 39, 475, 12] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[77, 589, 1200, 1, 1, 857, 1275, 363, 0, 330, 126, 7, 14, 206, 664, 191, 29, 737, 1024, 1536, 57, 237, 283, 31, 140, 133, 92, 226, 935, 543, 820, 1050, 1079, 404, 433, 1092, 214, 1260, 441, 519, 1342, 858, 1494, 162, 713, 836, 28, 89, 876, 643, 775, 128, 926, 611, 1629, 29, 1258, 51, 321, 28, 27, 14, 697, 17] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[177, 329, 858, 0, 0, 635, 803, 509, 382, 228, 0, 0, 0, 0, 0, 0, 2, 719, 16, 1, 18, 101, 234, 99, 1439, 968, 0, 0, 0, 0, 0, 0, 4, 773, 853, 1961, 25, 103, 193, 3, 48, 19, 0, 0, 0, 0, 0, 0, 94, 7, 67, 287, 816, 374, 1585, 878, 0, 0, 0, 0, 0, 0, 0, 0, 523, 413, 548, 0, 125, 686, 896, 899, 0, 0, 0, 0, 0, 0, 0, 0, 1137, 615, 199, 54, 1186, 1010, 10, 13, 0, 0, 0, 0, 0, 0, 0, 0, 916, 545, 622, 32, 935, 842, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 1384, 11, 249, 9, 4, 5, 1098, 33, 1550, 296, 246, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3921, 3597, 3982, 4108, 4090, 4224, 3961, 4885] avg=4096.0000 max=4885 max_rank=7 min=3597 min_rank=1 imbalance_ratio=1.192627 max_load_reduction=27883 max_load_reduction_pct=0.850922 moved_to_echo=17160 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[125, 240, 894, 1, 1, 483, 763, 415, 429, 268, 0, 0, 0, 0, 0, 0, 1, 571, 36, 2, 57, 3, 517, 84, 1587, 947, 0, 0, 0, 0, 0, 0, 2, 1181, 586, 1970, 31, 269, 201, 0, 54, 31, 0, 0, 0, 0, 0, 0, 34, 21, 21, 416, 848, 1101, 1180, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 764, 510, 485, 0, 136, 769, 420, 692, 0, 0, 0, 0, 0, 0, 0, 0, 1579, 827, 194, 70, 671, 767, 22, 41, 0, 0, 0, 0, 0, 0, 0, 0, 1168, 547, 679, 48, 686, 1392, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 1089, 30, 416, 2, 14, 12, 543, 19, 1143, 348, 241, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3619, 3805, 4325, 4652, 3776, 4171, 4563, 3857] avg=4096.0000 max=4652 max_rank=3 min=3619 min_rank=0 imbalance_ratio=1.135742 max_load_reduction=28116 max_load_reduction_pct=0.858032 moved_to_echo=16367 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[94, 112, 1433, 0, 0, 641, 817, 356, 377, 255, 0, 0, 0, 0, 0, 0, 0, 1045, 30, 5, 9, 4, 504, 27, 1965, 1483, 0, 0, 0, 0, 0, 0, 1, 859, 1200, 1846, 28, 51, 254, 2, 48, 35, 0, 0, 0, 0, 0, 0, 57, 20, 57, 199, 748, 757, 864, 979, 0, 0, 0, 0, 0, 0, 0, 0, 926, 816, 455, 0, 204, 675, 460, 791, 0, 0, 0, 0, 0, 0, 0, 0, 1638, 554, 304, 60, 468, 705, 14, 67, 0, 0, 0, 0, 0, 0, 0, 0, 682, 579, 494, 53, 397, 1574, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 1288, 34, 229, 2, 20, 27, 541, 10, 802, 330, 378, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 5072, 4324, 3681, 4327, 3810, 3808, 3661] avg=4096.0000 max=5072 max_rank=1 min=3661 min_rank=7 imbalance_ratio=1.238281 max_load_reduction=27696 max_load_reduction_pct=0.845215 moved_to_echo=15606 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[129, 100, 1138, 0, 0, 673, 736, 452, 247, 326, 0, 0, 0, 0, 0, 0, 3, 462, 44, 1, 7, 33, 214, 111, 1653, 1170, 0, 0, 0, 0, 0, 0, 3, 1014, 756, 2209, 13, 239, 255, 12, 31, 22, 0, 0, 0, 0, 0, 0, 51, 2, 76, 351, 1058, 608, 1056, 1253, 0, 0, 0, 0, 0, 0, 0, 0, 1235, 398, 553, 0, 152, 442, 525, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 1353, 479, 240, 71, 784, 567, 6, 27, 0, 0, 0, 0, 0, 0, 0, 0, 1221, 558, 647, 31, 807, 1004, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 978, 13, 188, 3, 6, 6, 1223, 7, 925, 423, 298, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3801, 3698, 4554, 4455, 4373, 3527, 4290, 4070] avg=4096.0000 max=4554 max_rank=2 min=3527 min_rank=5 imbalance_ratio=1.111816 max_load_reduction=28214 max_load_reduction_pct=0.861023 moved_to_echo=16260 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[44, 271, 925, 1, 0, 957, 741, 252, 289, 291, 0, 0, 0, 0, 0, 0, 0, 954, 35, 1, 13, 52, 530, 44, 1561, 1249, 0, 0, 0, 0, 0, 0, 3, 812, 932, 1690, 55, 146, 261, 4, 36, 27, 0, 0, 0, 0, 0, 0, 92, 28, 92, 233, 1347, 370, 1089, 1120, 0, 0, 0, 0, 0, 0, 0, 0, 638, 336, 379, 0, 259, 520, 816, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1790, 326, 257, 51, 648, 868, 18, 25, 0, 0, 0, 0, 0, 0, 0, 0, 738, 745, 728, 21, 667, 1211, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 1333, 54, 297, 2, 7, 39, 475, 12, 1193, 378, 318, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3771, 4439, 3966, 4371, 3990, 3983, 4140, 4108] avg=4096.0000 max=4439 max_rank=1 min=3771 min_rank=0 imbalance_ratio=1.083740 max_load_reduction=28329 max_load_reduction_pct=0.864532 moved_to_echo=16221 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[108, 238, 1210, 3, 1, 698, 813, 669, 487, 239, 0, 0, 0, 0, 0, 0, 13, 918, 32, 1, 15, 8, 188, 106, 1848, 1088, 0, 0, 0, 0, 0, 0, 9, 801, 543, 1856, 19, 121, 243, 6, 62, 34, 0, 0, 0, 0, 0, 0, 108, 11, 123, 401, 941, 428, 826, 920, 0, 0, 0, 0, 0, 0, 0, 0, 623, 480, 520, 0, 214, 872, 692, 711, 0, 0, 0, 0, 0, 0, 0, 0, 1507, 558, 224, 67, 759, 1004, 26, 20, 0, 0, 0, 0, 0, 0, 0, 0, 669, 746, 446, 51, 842, 1509, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 1107, 41, 306, 14, 50, 24, 589, 15, 1348, 310, 277, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4466, 4217, 3694, 3758, 4112, 4165, 4275, 4081] avg=4096.0000 max=4466 max_rank=0 min=3694 min_rank=2 imbalance_ratio=1.090332 max_load_reduction=28302 max_load_reduction_pct=0.863708 moved_to_echo=16633 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[220, 397, 616, 0, 0, 848, 927, 442, 427, 262, 0, 0, 0, 0, 0, 0, 1, 583, 20, 0, 7, 67, 252, 131, 1600, 1089, 0, 0, 0, 0, 0, 0, 1, 721, 711, 1724, 23, 377, 242, 30, 57, 35, 0, 0, 0, 0, 0, 0, 100, 8, 48, 317, 1270, 464, 1010, 979, 0, 0, 0, 0, 0, 0, 0, 0, 524, 439, 701, 0, 88, 749, 699, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1478, 661, 213, 28, 1266, 932, 16, 56, 0, 0, 0, 0, 0, 0, 0, 0, 952, 535, 237, 25, 541, 1408, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 1112, 36, 309, 6, 28, 15, 797, 19, 1180, 336, 281, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4139, 3750, 3921, 4196, 4237, 4650, 3756, 4119] avg=4096.0000 max=4650 max_rank=5 min=3750 min_rank=1 imbalance_ratio=1.135254 max_load_reduction=28118 max_load_reduction_pct=0.858093 moved_to_echo=16762 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[77, 589, 1200, 1, 1, 857, 1275, 363, 432, 171, 0, 0, 0, 0, 0, 0, 0, 330, 126, 7, 14, 206, 664, 191, 1629, 1023, 0, 0, 0, 0, 0, 0, 29, 737, 1024, 1536, 57, 237, 283, 31, 55, 13, 0, 0, 0, 0, 0, 0, 140, 133, 92, 226, 935, 543, 820, 658, 0, 0, 0, 0, 0, 0, 0, 0, 1079, 404, 433, 0, 214, 773, 441, 519, 0, 0, 0, 0, 0, 0, 0, 0, 1342, 858, 210, 162, 713, 836, 28, 89, 0, 0, 0, 0, 0, 0, 0, 0, 876, 643, 775, 128, 926, 598, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 1258, 51, 321, 28, 27, 14, 697, 17, 1092, 221, 261, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4966, 4190, 4002, 3547, 3863, 4238, 3975, 3987] avg=4096.0000 max=4966 max_rank=0 min=3547 min_rank=3 imbalance_ratio=1.212402 max_load_reduction=27802 max_load_reduction_pct=0.848450 moved_to_echo=16063 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1024, 2304, 8320, 128, 128, 5888, 6912, 3584, 3072, 2048, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[6400, 3840, 4096, 0, 1408, 5504, 4992, 6784, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[11904, 4992, 1920, 640, 6528, 6784, 256, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[7296, 4992, 4736, 512, 5888, 9600, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 6912, 6656, 14848, 256, 1664, 2048, 128, 512, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[768, 256, 640, 2432, 8064, 4736, 8448, 7936, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 5632, 384, 128, 256, 512, 3200, 896, 13312, 9088, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[9600, 384, 2432, 128, 256, 256, 6016, 256, 9344, 2688, 2304, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1059, 0, 0, 0, 0, 1695, 34, 806, 17, 33, 2260, 1150, 1, 0, 296, 0, 0, 0, 122, 0, 212, 27, 51, 0, 5, 526, 0, 0, 67, 1148, 2760, 0, 1, 0, 0, 0, 1816, 4027, 2961, 0, 947, 1, 1303, 0, 0, 1, 2, 0, 0, 0, 142, 2738, 1215, 0, 13, 1, 835, 8, 681, 419, 1617, 0, 1771, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1126, 0, 3, 0, 0, 1166, 4, 1545, 273, 7, 2837, 2347, 5, 0, 1436, 1, 0, 0, 99, 26, 552, 12, 899, 0, 0, 2191, 0, 0, 4, 456, 872, 34, 0, 0, 0, 0, 2819, 2434, 1101, 0, 168, 5, 14, 0, 0, 0, 306, 2, 3, 4, 963, 430, 1227, 0, 4, 0, 626, 7, 2356, 1313, 441, 0, 2650, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[412, 0, 0, 0, 0, 1865, 0, 780, 5, 0, 3528, 1254, 1, 0, 563, 0, 0, 0, 374, 9, 1176, 155, 1605, 0, 0, 2225, 1, 0, 250, 2197, 1041, 0, 0, 0, 3, 1, 2095, 2578, 3139, 1, 779, 1, 212, 0, 1, 0, 132, 1, 4, 0, 301, 1795, 703, 0, 9, 0, 128, 2, 1273, 1581, 210, 0, 378, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1599, 0, 18, 0, 0, 1998, 0, 1732, 0, 14, 3664, 217, 0, 0, 778, 0, 0, 0, 158, 9, 87, 59, 95, 7, 0, 588, 0, 2, 2, 3734, 1786, 0, 0, 0, 0, 0, 1687, 2645, 3367, 0, 398, 0, 317, 0, 0, 0, 197, 0, 1, 10, 160, 3982, 787, 0, 2, 3, 777, 0, 890, 703, 236, 0, 59, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1371, 0, 0, 0, 0, 615, 1, 953, 11, 108, 3435, 677, 7, 0, 296, 0, 0, 2, 65, 0, 1651, 74, 46, 0, 1, 1387, 2, 0, 3, 976, 2736, 0, 0, 9, 0, 0, 1666, 3936, 3626, 1, 104, 0, 393, 3, 0, 0, 126, 0, 1, 0, 702, 553, 1851, 0, 0, 0, 431, 0, 1158, 776, 523, 0, 2492, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[848, 0, 1, 0, 0, 8, 1, 32, 0, 1, 3274, 889, 0, 0, 2099, 0, 0, 0, 165, 46, 474, 17, 26, 0, 0, 3191, 0, 0, 4, 3459, 262, 0, 0, 0, 0, 0, 2078, 3972, 1328, 0, 796, 0, 149, 0, 0, 0, 73, 0, 0, 1, 87, 277, 1226, 0, 0, 0, 1246, 1, 1787, 688, 2748, 0, 1514, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[105, 0, 0, 0, 0, 3755, 1, 1011, 12, 32, 3752, 65, 0, 0, 136, 0, 0, 0, 0, 33, 2042, 2, 17, 0, 0, 2361, 0, 1, 4, 2066, 2884, 0, 2, 7, 22, 0, 1724, 1670, 1011, 2, 2096, 0, 155, 0, 0, 0, 188, 0, 0, 0, 1978, 222, 199, 0, 10, 0, 1146, 0, 436, 1464, 2048, 1, 108, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[563, 0, 0, 0, 1, 2073, 2, 990, 10, 200, 3282, 1905, 1, 1, 170, 2, 0, 0, 77, 259, 1, 37, 63, 0, 0, 2420, 0, 1, 1, 228, 1871, 1, 0, 0, 0, 0, 1261, 3959, 3519, 2, 31, 0, 1400, 0, 1, 0, 2426, 0, 80, 0, 40, 1681, 479, 0, 22, 4, 1825, 10, 240, 904, 559, 0, 164, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1059, 0, 0, 0, 0, 1695, 34, 806, 366, 183, 0, 0, 0, 0, 0, 0, 17, 33, 1542, 1150, 1, 0, 296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 0, 212, 27, 51, 0, 3502, 0, 0, 0, 0, 0, 0, 0, 5, 139, 0, 0, 67, 1148, 2760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1816, 0, 2595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 947, 1, 1303, 0, 0, 1, 2, 0, 387, 1119, 525, 0, 0, 0, 0, 0, 0, 0, 142, 2738, 1215, 0, 13, 1, 535, 521, 0, 0, 0, 0, 0, 0, 835, 8, 681, 419, 1617, 0, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4143, 3039, 3914, 4119, 4412, 4285, 5165, 3691] avg=4096.0000 max=5165 max_rank=6 min=3039 min_rank=1 imbalance_ratio=1.260986 max_load_reduction=27603 max_load_reduction_pct=0.842377 moved_to_echo=17553 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[848, 0, 1, 0, 0, 8, 1, 32, 167, 270, 0, 0, 0, 0, 0, 0, 0, 1, 2225, 889, 0, 0, 2099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 46, 474, 17, 26, 0, 3455, 0, 0, 0, 0, 0, 0, 0, 0, 833, 0, 0, 4, 3459, 262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2078, 0, 1161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 796, 0, 149, 0, 0, 0, 73, 0, 2358, 961, 517, 0, 0, 0, 0, 0, 0, 1, 87, 277, 1226, 0, 0, 0, 779, 448, 0, 0, 0, 0, 0, 0, 1246, 1, 1787, 688, 2748, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1327, 5214, 4183, 4558, 3239, 4854, 2818, 6575] avg=4096.0000 max=6575 max_rank=7 min=1327 min_rank=0 imbalance_ratio=1.605225 max_load_reduction=26193 max_load_reduction_pct=0.799347 moved_to_echo=17486 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[105, 0, 0, 0, 0, 3755, 1, 1011, 125, 304, 0, 0, 0, 0, 0, 0, 12, 32, 2559, 65, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 2042, 2, 17, 0, 1453, 0, 0, 0, 0, 0, 0, 0, 0, 620, 0, 1, 4, 2066, 2884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7, 22, 0, 1724, 0, 886, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2096, 0, 155, 0, 0, 0, 188, 0, 1741, 68, 217, 0, 0, 0, 0, 0, 0, 0, 1978, 222, 199, 0, 10, 0, 889, 31, 0, 0, 0, 0, 0, 0, 1146, 0, 436, 1464, 2048, 1, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5301, 2804, 3547, 5575, 2643, 4465, 3329, 5104] avg=4096.0000 max=5575 max_rank=3 min=2643 min_rank=4 imbalance_ratio=1.361084 max_load_reduction=27193 max_load_reduction_pct=0.829865 moved_to_echo=15541 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1599, 0, 18, 0, 0, 1998, 0, 1732, 416, 297, 0, 0, 0, 0, 0, 0, 0, 14, 2499, 217, 0, 0, 778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 9, 87, 59, 95, 7, 2300, 0, 0, 0, 0, 0, 0, 0, 0, 155, 0, 2, 2, 3734, 1786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1687, 0, 2951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 398, 0, 317, 0, 0, 0, 197, 0, 433, 37, 345, 0, 0, 0, 0, 0, 1, 10, 160, 3982, 787, 0, 2, 3, 868, 17, 0, 0, 0, 0, 0, 0, 777, 0, 890, 703, 236, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6060, 3508, 2715, 5679, 4638, 1727, 5830, 2611] avg=4096.0000 max=6060 max_rank=0 min=1727 min_rank=5 imbalance_ratio=1.479492 max_load_reduction=26708 max_load_reduction_pct=0.815063 moved_to_echo=14806 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[563, 0, 0, 0, 1, 2073, 2, 990, 435, 266, 0, 0, 0, 0, 0, 0, 10, 200, 2239, 1905, 1, 1, 170, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 259, 1, 37, 63, 0, 3443, 0, 0, 0, 0, 0, 0, 0, 0, 636, 0, 1, 1, 228, 1871, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1261, 0, 3084, 2, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 1400, 0, 1, 0, 2426, 0, 1784, 103, 516, 0, 0, 0, 0, 0, 80, 0, 40, 1681, 479, 0, 22, 4, 777, 48, 0, 0, 0, 0, 0, 0, 1825, 10, 240, 904, 559, 0, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4330, 4528, 3880, 2738, 4347, 6261, 3131, 3553] avg=4096.0000 max=6261 max_rank=5 min=2738 min_rank=3 imbalance_ratio=1.528564 max_load_reduction=26507 max_load_reduction_pct=0.808929 moved_to_echo=17292 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1371, 0, 0, 0, 0, 615, 1, 953, 448, 278, 0, 0, 0, 0, 0, 0, 11, 108, 2343, 677, 7, 0, 296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 65, 0, 1651, 74, 46, 0, 3424, 0, 0, 0, 0, 0, 0, 0, 1, 365, 2, 0, 3, 976, 2736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 1666, 0, 3178, 1, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 393, 3, 0, 0, 126, 0, 1022, 1575, 512, 0, 0, 0, 0, 0, 1, 0, 702, 553, 1851, 0, 0, 0, 814, 733, 0, 0, 0, 0, 0, 0, 431, 0, 1158, 776, 523, 0, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3666, 3442, 5262, 4083, 4854, 3735, 4654, 3072] avg=4096.0000 max=5262 max_rank=2 min=3072 min_rank=7 imbalance_ratio=1.284668 max_load_reduction=27506 max_load_reduction_pct=0.839417 moved_to_echo=16315 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1126, 0, 3, 0, 0, 1166, 4, 1545, 136, 230, 0, 0, 0, 0, 0, 0, 273, 7, 1935, 2347, 5, 0, 1436, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 26, 552, 12, 899, 0, 2118, 0, 0, 0, 0, 0, 0, 0, 0, 576, 0, 0, 4, 456, 872, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2819, 0, 965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 5, 14, 0, 0, 0, 306, 2, 1615, 1675, 316, 0, 0, 0, 0, 0, 3, 4, 963, 430, 1227, 0, 4, 0, 672, 779, 0, 0, 0, 0, 0, 0, 626, 7, 2356, 1313, 441, 0, 196, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4210, 6004, 3706, 1942, 3784, 4101, 4082, 4939] avg=4096.0000 max=6004 max_rank=1 min=1942 min_rank=3 imbalance_ratio=1.465820 max_load_reduction=26764 max_load_reduction_pct=0.816772 moved_to_echo=16906 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[412, 0, 0, 0, 0, 1865, 0, 780, 388, 286, 0, 0, 0, 0, 0, 0, 5, 0, 2406, 1254, 1, 0, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 374, 9, 1176, 155, 1605, 0, 2242, 0, 0, 0, 0, 0, 0, 0, 0, 585, 1, 0, 250, 2197, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 2095, 0, 2751, 1, 0, 0, 0, 0, 0, 0, 0, 0, 779, 1, 212, 0, 1, 0, 132, 1, 1640, 238, 336, 0, 0, 0, 0, 0, 4, 0, 301, 1795, 703, 0, 9, 0, 836, 111, 0, 0, 0, 0, 0, 0, 128, 2, 1273, 1581, 210, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3731, 4229, 5561, 4074, 4851, 3340, 3759, 3223] avg=4096.0000 max=5561 max_rank=2 min=3223 min_rank=7 imbalance_ratio=1.357666 max_load_reduction=27207 max_load_reduction_pct=0.830292 moved_to_echo=15173 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 4480, 11776, 7808, 0, 128, 128, 6272, 2688, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[7168, 0, 128, 0, 128, 13184, 128, 7936, 2560, 2176, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[7040, 128, 8832, 7936, 8448, 128, 768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[5376, 128, 3968, 128, 128, 128, 3456, 128, 11008, 5888, 3328, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[384, 512, 17792, 8576, 128, 128, 5888, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 3968, 128, 128, 384, 14336, 14336, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 15232, 0, 17664, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 1152, 384, 6272, 384, 2816, 128, 22016, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3435, 0, 276, 0, 833, 0, 0, 96, 4, 3326, 0, 460, 736, 1, 3522, 140, 79, 0, 0, 0, 1, 0, 0, 86, 4, 6, 0, 0, 0, 2835, 0, 6, 0, 196, 20, 0, 0, 0, 0, 0, 3301, 2, 0, 0, 47, 0, 3927, 2641, 0, 3205, 568, 0, 0, 771, 0, 0, 58, 0, 2186, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[2, 6, 0, 0, 2535, 0, 75, 1, 1256, 0, 0, 146, 3, 1597, 1, 236, 323, 0, 3915, 0, 2, 2, 0, 1, 0, 5, 0, 367, 0, 1, 0, 0, 0, 3730, 14, 0, 0, 195, 2, 0, 2, 2, 1, 0, 3930, 0, 27, 0, 18, 11, 3964, 1169, 1, 1835, 4012, 0, 0, 37, 0, 0, 2022, 1072, 250, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 8, 0, 2, 2423, 0, 2197, 5, 2639, 0, 4, 1918, 0, 763, 2, 303, 1074, 5, 3839, 112, 0, 0, 0, 0, 0, 2, 1, 626, 0, 0, 0, 4, 6, 1042, 0, 0, 0, 439, 83, 0, 0, 0, 4, 29, 3714, 0, 0, 0, 32, 42, 4094, 2447, 0, 499, 3700, 0, 0, 9, 0, 0, 332, 35, 332, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 5, 1, 3456, 0, 1809, 0, 85, 0, 0, 226, 0, 1643, 0, 146, 2043, 23, 3997, 0, 0, 0, 0, 1, 0, 0, 0, 20, 92, 10, 0, 1, 0, 3518, 0, 0, 0, 166, 86, 1, 0, 0, 0, 11, 3726, 0, 0, 0, 176, 13, 4026, 1117, 0, 938, 3980, 2, 0, 6, 0, 0, 204, 9, 1231, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2364, 0, 1736, 0, 1871, 0, 0, 27, 0, 2413, 0, 178, 3016, 28, 3794, 0, 2, 0, 0, 0, 2, 0, 0, 28, 0, 0, 0, 3, 19, 826, 1, 0, 355, 1818, 78, 0, 0, 0, 0, 6, 2246, 0, 0, 0, 273, 0, 3988, 3018, 2, 62, 3755, 0, 0, 210, 0, 0, 153, 86, 408, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 509, 0, 1065, 0, 3198, 0, 0, 5, 0, 3057, 0, 1147, 3090, 9, 3583, 1, 6, 0, 0, 0, 0, 0, 0, 380, 6, 2, 0, 0, 0, 1107, 0, 0, 0, 2, 3, 0, 0, 0, 0, 1, 3980, 0, 1, 0, 14, 0, 4022, 1581, 0, 760, 3927, 0, 0, 1, 0, 0, 584, 84, 642, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 7, 0, 18, 3142, 0, 2015, 0, 723, 0, 26, 2037, 2, 2034, 0, 46, 1624, 0, 2652, 16, 0, 0, 0, 0, 0, 0, 0, 9, 0, 1, 0, 0, 12, 1037, 90, 0, 1, 3, 0, 0, 0, 0, 0, 4, 3205, 0, 2, 0, 31, 171, 4081, 56, 190, 3602, 3885, 0, 0, 0, 0, 0, 334, 72, 1640, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3621, 0, 265, 0, 3213, 1, 0, 1, 0, 3081, 0, 35, 459, 0, 1901, 248, 0, 0, 0, 0, 0, 0, 0, 240, 2, 1, 0, 12, 1, 1501, 0, 4, 1, 1770, 612, 0, 0, 0, 0, 0, 2396, 0, 0, 0, 510, 19, 4067, 2150, 0, 2085, 3892, 4, 0, 2, 2, 0, 15, 0, 657, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 6, 0, 0, 2535, 0, 75, 1, 159, 0, 0, 0, 0, 0, 0, 0, 1256, 0, 0, 146, 3, 1062, 1, 236, 0, 0, 0, 0, 0, 0, 0, 0, 323, 0, 2840, 0, 2, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 367, 0, 1, 0, 0, 3802, 0, 0, 0, 0, 0, 0, 0, 0, 3730, 14, 0, 0, 195, 2, 0, 819, 794, 0, 0, 0, 0, 0, 0, 2, 2, 1, 0, 3930, 0, 27, 0, 376, 281, 0, 0, 0, 0, 0, 0, 18, 11, 0, 1169, 1, 1835, 587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 2022, 1072, 250, 0, 2606, 162, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2778, 2704, 3168, 4175, 5554, 4619, 3621, 6149] avg=4096.0000 max=6149 max_rank=7 min=2704 min_rank=1 imbalance_ratio=1.501221 max_load_reduction=26619 max_load_reduction_pct=0.812347 moved_to_echo=19943 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 0, 2, 2423, 0, 2197, 5, 76, 0, 0, 0, 0, 0, 0, 0, 2639, 0, 4, 1918, 0, 508, 2, 303, 0, 0, 0, 0, 0, 0, 0, 0, 1074, 5, 2785, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 626, 0, 0, 0, 4, 3927, 0, 0, 0, 0, 0, 0, 0, 6, 1042, 0, 0, 0, 439, 83, 0, 756, 779, 0, 0, 0, 0, 0, 0, 0, 0, 4, 29, 3714, 0, 0, 0, 179, 275, 0, 0, 0, 0, 0, 0, 32, 42, 0, 2447, 0, 499, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 332, 35, 332, 2, 2403, 167, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4711, 5374, 3976, 4560, 3105, 4201, 3561, 3280] avg=4096.0000 max=5374 max_rank=1 min=3105 min_rank=4 imbalance_ratio=1.312012 max_load_reduction=27394 max_load_reduction_pct=0.835999 moved_to_echo=14147 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 5, 1, 3456, 0, 1809, 0, 164, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 226, 0, 1092, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 2043, 23, 2900, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 92, 10, 0, 1, 3861, 0, 0, 0, 0, 0, 0, 0, 0, 3518, 0, 0, 0, 166, 86, 1, 813, 811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 3726, 0, 0, 0, 387, 286, 0, 0, 0, 0, 0, 0, 176, 13, 0, 1117, 0, 938, 582, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 204, 9, 1231, 0, 2585, 165, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5435, 1549, 4967, 3984, 5395, 4410, 2828, 4200] avg=4096.0000 max=5435 max_rank=0 min=1549 min_rank=1 imbalance_ratio=1.326904 max_load_reduction=27333 max_load_reduction_pct=0.834137 moved_to_echo=16833 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 0, 18, 3142, 0, 2015, 0, 203, 0, 0, 0, 0, 0, 0, 0, 723, 0, 26, 2037, 2, 1352, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 1624, 0, 1924, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 1, 0, 0, 3915, 0, 0, 0, 0, 0, 0, 0, 12, 1037, 90, 0, 1, 3, 0, 0, 793, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3205, 0, 2, 0, 479, 190, 0, 0, 0, 0, 0, 0, 31, 171, 0, 56, 190, 3602, 569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 334, 72, 1640, 0, 2523, 166, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5385, 4186, 3564, 3925, 2474, 3880, 4619, 4735] avg=4096.0000 max=5385 max_rank=0 min=2474 min_rank=4 imbalance_ratio=1.314697 max_load_reduction=27383 max_load_reduction_pct=0.835663 moved_to_echo=15708 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2364, 0, 1736, 0, 240, 0, 0, 0, 0, 0, 0, 0, 1871, 0, 0, 27, 0, 1604, 0, 178, 0, 0, 0, 0, 0, 0, 0, 0, 3016, 28, 2752, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 28, 0, 0, 0, 3, 3825, 0, 0, 0, 0, 0, 0, 0, 19, 826, 1, 0, 355, 1818, 78, 0, 767, 770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2246, 0, 0, 0, 569, 272, 0, 0, 0, 0, 0, 0, 273, 0, 0, 3018, 2, 62, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 210, 0, 0, 153, 86, 408, 2, 2439, 163, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4340, 3680, 5798, 3858, 4634, 3093, 3904, 3461] avg=4096.0000 max=5798 max_rank=2 min=3093 min_rank=5 imbalance_ratio=1.415527 max_load_reduction=26970 max_load_reduction_pct=0.823059 moved_to_echo=15092 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3435, 0, 276, 0, 332, 0, 0, 0, 0, 0, 0, 0, 833, 0, 0, 96, 4, 2210, 0, 460, 0, 0, 0, 0, 0, 0, 0, 0, 736, 1, 2555, 140, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 86, 4, 6, 0, 0, 3766, 0, 0, 0, 0, 0, 0, 0, 0, 2835, 0, 6, 0, 196, 20, 0, 116, 715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3301, 2, 0, 0, 784, 252, 0, 0, 0, 0, 0, 0, 47, 0, 0, 2641, 0, 3205, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 771, 0, 0, 58, 0, 2186, 0, 368, 161, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4043, 3603, 3511, 3863, 3888, 4339, 5977, 3544] avg=4096.0000 max=5977 max_rank=6 min=3511 min_rank=2 imbalance_ratio=1.459229 max_load_reduction=26791 max_load_reduction_pct=0.817596 moved_to_echo=17748 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3621, 0, 265, 0, 307, 0, 0, 0, 0, 0, 0, 0, 3213, 1, 0, 1, 0, 2048, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 459, 0, 1380, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 2, 1, 0, 12, 3902, 0, 0, 0, 0, 0, 0, 0, 1, 1501, 0, 4, 1, 1770, 612, 0, 795, 385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2396, 0, 0, 0, 726, 136, 0, 0, 0, 0, 0, 0, 510, 19, 0, 2150, 0, 2085, 569, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 15, 0, 657, 0, 2528, 165, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4193, 5298, 2087, 4157, 5069, 3258, 5337, 3369] avg=4096.0000 max=5337 max_rank=6 min=2087 min_rank=2 imbalance_ratio=1.302979 max_load_reduction=27431 max_load_reduction_pct=0.837128 moved_to_echo=17033 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 509, 0, 1065, 0, 308, 0, 0, 0, 0, 0, 0, 0, 3198, 0, 0, 5, 0, 2024, 0, 1147, 0, 0, 0, 0, 0, 0, 0, 0, 3090, 9, 2591, 1, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 380, 6, 2, 0, 0, 3858, 0, 0, 0, 0, 0, 0, 0, 0, 1107, 0, 0, 0, 2, 3, 0, 806, 731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3980, 0, 1, 0, 725, 261, 0, 0, 0, 0, 0, 0, 14, 0, 0, 1581, 0, 760, 566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 584, 84, 642, 0, 2555, 164, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1883, 6374, 5697, 4246, 2649, 4968, 2921, 4030] avg=4096.0000 max=6374 max_rank=1 min=1883 min_rank=0 imbalance_ratio=1.556152 max_load_reduction=26394 max_load_reduction_pct=0.805481 moved_to_echo=14568 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 15616, 128, 128, 384, 4608, 896, 128, 5760, 5632, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[12416, 128, 19840, 640, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[13824, 128, 128, 4480, 128, 11904, 128, 2560, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 1152, 128, 0, 3712, 1408, 7424, 128, 18048, 1408, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1152, 256, 0, 14208, 256, 13056, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 21504, 0, 9472, 128, 1792, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 26624, 128, 128, 0, 4352, 2048, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 1792, 128, 128, 0, 128, 30976, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33792 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3292, 0, 15, 0, 1652, 3823, 0, 0, 2881, 0, 0, 0, 0, 2329, 2928, 0, 1, 0, 0, 0, 0, 0, 5, 0, 267, 0, 4078, 0, 0, 3030, 0, 0, 0, 0, 0, 2, 0, 2591, 0, 0, 0, 0, 0, 0, 1461, 0, 0, 4090, 0, 76, 0, 0, 0, 0, 0, 247, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 7, 0, 0, 2384, 4, 0, 0, 1595, 2106, 0, 7, 50, 0, 22, 19, 1, 4036, 4090, 2, 0, 1, 1, 0, 10, 6, 2, 0, 1021, 0, 4090, 0, 0, 2963, 0, 0, 12, 0, 0, 17, 0, 3197, 0, 22, 0, 0, 2, 0, 2940, 14, 0, 4051, 4, 78, 0, 0, 3, 2, 0, 3, 0, 6, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[12, 0, 0, 0, 0, 598, 1, 0, 0, 7, 3518, 0, 0, 3586, 0, 13, 0, 0, 3961, 3881, 0, 1, 0, 0, 0, 0, 0, 0, 2, 3, 0, 4088, 0, 0, 3898, 0, 0, 0, 0, 0, 2, 0, 3884, 0, 0, 0, 0, 0, 0, 1001, 3, 1, 4066, 0, 90, 1, 0, 0, 0, 2, 149, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1, 1, 4, 0, 0, 3325, 0, 21, 9, 154, 2134, 0, 0, 254, 0, 51, 0, 0, 2176, 3793, 0, 21, 0, 0, 0, 0, 0, 0, 0, 285, 0, 4088, 0, 0, 3824, 0, 1, 0, 7, 0, 40, 0, 4091, 13, 0, 0, 0, 0, 15, 3987, 0, 0, 4096, 0, 299, 0, 5, 3, 0, 0, 67, 0, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3609, 0, 0, 0, 841, 868, 0, 0, 3350, 0, 3, 0, 0, 3137, 3487, 7, 0, 0, 0, 0, 0, 9, 12, 1, 256, 0, 4094, 0, 0, 3643, 0, 7, 0, 0, 0, 0, 0, 4004, 56, 0, 0, 0, 0, 0, 1018, 0, 0, 4075, 0, 134, 0, 0, 0, 0, 0, 155, 2, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 2764, 0, 2, 0, 15, 2935, 0, 0, 2688, 0, 48, 0, 0, 3682, 3530, 0, 21, 0, 0, 0, 0, 0, 0, 8, 135, 0, 4095, 0, 0, 3763, 0, 0, 0, 0, 0, 0, 0, 3699, 71, 0, 0, 1, 0, 0, 856, 0, 0, 3530, 0, 9, 0, 0, 0, 0, 0, 902, 0, 0, 13] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 2108, 1613, 0, 0, 176, 0, 1, 0, 0, 2963, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 591, 0, 4086, 0, 0, 3095, 0, 0, 0, 0, 0, 0, 0, 2087, 3, 0, 0, 0, 0, 0, 3174, 0, 0, 4081, 0, 554, 0, 0, 0, 0, 0, 72, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3031, 0, 3, 0, 1842, 2555, 0, 0, 3623, 0, 40, 0, 17, 538, 3573, 0, 0, 0, 0, 1, 0, 0, 0, 15, 274, 0, 2841, 0, 11, 3942, 0, 0, 0, 0, 0, 269, 0, 885, 0, 0, 0, 0, 1, 0, 4072, 0, 0, 4027, 0, 17, 0, 0, 12, 0, 4, 1175, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 4, 0, 0, 3325, 0, 21, 1100, 108, 0, 0, 0, 0, 0, 0, 9, 154, 848, 0, 0, 254, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2176, 1274, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 285, 0, 3881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3824, 0, 1, 0, 7, 0, 324, 207, 0, 0, 0, 0, 0, 0, 40, 0, 4091, 13, 0, 0, 0, 0, 854, 0, 0, 0, 0, 0, 0, 0, 15, 3987, 0, 0, 1660, 0, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3, 0, 0, 67, 0, 3, 0, 2519, 1336, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4560, 1316, 3471, 4166, 4363, 4998, 5961, 3933] avg=4096.0000 max=5961 max_rank=6 min=1316 min_rank=1 imbalance_ratio=1.455322 max_load_reduction=26807 max_load_reduction_pct=0.818085 moved_to_echo=19255 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3292, 0, 15, 1099, 193, 0, 0, 0, 0, 0, 0, 0, 1652, 1517, 0, 0, 2881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2329, 984, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 267, 0, 3872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3030, 0, 0, 0, 0, 0, 582, 206, 0, 0, 0, 0, 0, 0, 2, 0, 2591, 0, 0, 0, 0, 0, 1531, 0, 0, 0, 0, 0, 0, 0, 0, 1461, 0, 0, 1657, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 0, 0, 0, 1944, 1334, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4599, 6050, 3314, 4144, 3818, 4124, 3194, 3525] avg=4096.0000 max=6050 max_rank=1 min=3194 min_rank=6 imbalance_ratio=1.477051 max_load_reduction=26718 max_load_reduction_pct=0.815369 moved_to_echo=14661 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 7, 0, 0, 2384, 4, 0, 1088, 106, 0, 0, 0, 0, 0, 0, 0, 1595, 837, 0, 7, 50, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 19, 1, 4036, 1374, 2, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 6, 2, 0, 1021, 0, 3883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2963, 0, 0, 12, 0, 0, 320, 207, 0, 0, 0, 0, 0, 0, 17, 0, 3197, 0, 22, 0, 0, 2, 843, 0, 0, 0, 0, 0, 0, 0, 0, 2940, 14, 0, 1642, 4, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 3, 0, 6, 0, 2716, 1321, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3589, 2511, 5434, 4922, 3502, 4081, 4678, 4051] avg=4096.0000 max=5434 max_rank=2 min=2511 min_rank=1 imbalance_ratio=1.326660 max_load_reduction=27334 max_load_reduction_pct=0.834167 moved_to_echo=16312 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 0, 0, 0, 598, 1, 0, 1092, 178, 0, 0, 0, 0, 0, 0, 0, 7, 1396, 0, 0, 3586, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3961, 1304, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 3881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3898, 0, 0, 0, 0, 0, 535, 207, 0, 0, 0, 0, 0, 0, 2, 0, 3884, 0, 0, 0, 0, 0, 1409, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 3, 1, 1648, 0, 90, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 149, 0, 0, 0, 2577, 1326, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1881, 5002, 5266, 3886, 4640, 5295, 2744, 4054] avg=4096.0000 max=5295 max_rank=5 min=1881 min_rank=0 imbalance_ratio=1.292725 max_load_reduction=27473 max_load_reduction_pct=0.838409 moved_to_echo=16733 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3609, 0, 0, 1095, 44, 0, 0, 0, 0, 0, 0, 0, 841, 345, 0, 0, 3350, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3137, 1171, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 12, 1, 256, 0, 3887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3643, 0, 7, 0, 0, 0, 132, 207, 0, 0, 0, 0, 0, 0, 0, 0, 4004, 56, 0, 0, 0, 0, 347, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 0, 0, 1651, 0, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 2, 0, 0, 2316, 1329, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4748, 4539, 4315, 4165, 3989, 4407, 2803, 3802] avg=4096.0000 max=4748 max_rank=0 min=2803 min_rank=6 imbalance_ratio=1.159180 max_load_reduction=28020 max_load_reduction_pct=0.855103 moved_to_echo=15001 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3031, 0, 3, 1082, 129, 0, 0, 0, 0, 0, 0, 0, 1842, 1014, 0, 0, 3623, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 538, 1200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 15, 274, 0, 2697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 3942, 0, 0, 0, 0, 0, 389, 144, 0, 0, 0, 0, 0, 0, 269, 0, 885, 0, 0, 0, 0, 1, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 1631, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4, 1175, 0, 0, 0, 2373, 1314, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4245, 6519, 1755, 2987, 4486, 2178, 5720, 4878] avg=4096.0000 max=6519 max_rank=1 min=1755 min_rank=2 imbalance_ratio=1.591553 max_load_reduction=26249 max_load_reduction_pct=0.801056 moved_to_echo=17262 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 2764, 0, 2, 948, 148, 0, 0, 0, 0, 0, 0, 0, 15, 1166, 0, 0, 2688, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3682, 1186, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 135, 0, 3888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3763, 0, 0, 0, 0, 0, 446, 207, 0, 0, 0, 0, 0, 0, 0, 0, 3699, 71, 0, 0, 1, 0, 1175, 0, 0, 0, 0, 0, 0, 0, 0, 856, 0, 0, 1431, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 902, 0, 0, 13, 2344, 1151, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3863, 3917, 4889, 4031, 4416, 4946, 2296, 4410] avg=4096.0000 max=4946 max_rank=5 min=2296 min_rank=6 imbalance_ratio=1.207520 max_load_reduction=27822 max_load_reduction_pct=0.849060 moved_to_echo=16068 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 1101, 86, 0, 0, 0, 0, 0, 0, 0, 2108, 629, 0, 0, 176, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2963, 1361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 591, 0, 3876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3095, 0, 0, 0, 0, 0, 249, 210, 0, 0, 0, 0, 0, 0, 0, 0, 2087, 3, 0, 0, 0, 0, 649, 0, 0, 0, 0, 0, 0, 0, 0, 3174, 0, 0, 1644, 0, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 2707, 1336, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5283, 2914, 4324, 4467, 3554, 2739, 5372, 4115] avg=4096.0000 max=5372 max_rank=6 min=2739 min_rank=5 imbalance_ratio=1.311523 max_load_reduction=27396 max_load_reduction_pct=0.836060 moved_to_echo=15780 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 2816, 128, 128, 128, 19584, 10496, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 18560, 128, 128, 13056, 128, 1280, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 22912, 9856, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 0, 23168, 128, 128, 8704, 1024, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 8320, 7808, 0, 128, 16640, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[384, 0, 24448, 256, 128, 0, 128, 128, 7936, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 28160, 0, 128, 128, 128, 0, 3072, 1664, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 2944, 0, 29952, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33792 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 48, 0, 21, 2995, 0, 0, 1728, 4085, 0, 0, 0, 0, 0, 0, 0, 13, 4, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2661, 0, 0, 0, 0, 0, 0, 0, 0, 2, 27, 3642, 4094, 0, 3886, 2602, 0, 0, 1, 0, 0, 671, 0, 1517, 0, 690, 0, 0, 0, 0, 0, 0, 4080] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 17, 0, 3, 3757, 0, 0, 1546, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 136, 0, 1, 0, 0, 15, 4, 0, 1279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 4091, 4093, 0, 3885, 1147, 0, 3, 0, 0, 0, 82, 0, 453, 0, 4002, 1, 0, 0, 0, 0, 0, 3899] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[5, 195, 0, 1, 2520, 0, 0, 2564, 4093, 6, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 1, 0, 180, 0, 0, 0, 39, 124, 11, 2, 2846, 0, 1, 3, 0, 0, 0, 0, 4, 0, 10, 4071, 3978, 12, 4076, 20, 0, 0, 0, 1, 0, 25, 6, 112, 0, 3766, 0, 0, 0, 0, 0, 0, 4091] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 275, 0, 1, 2337, 1, 0, 2783, 4010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 69, 0, 0, 1368, 0, 0, 3, 0, 0, 0, 0, 0, 0, 841, 3950, 4087, 0, 3804, 1157, 0, 6, 0, 0, 10, 2, 15, 721, 0, 3231, 0, 0, 0, 0, 0, 0, 4054] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[3, 1, 0, 0, 3843, 0, 0, 258, 4060, 16, 0, 7, 0, 3, 2, 0, 4, 0, 0, 10, 7, 0, 1, 318, 0, 6, 0, 0, 0, 9, 0, 2931, 0, 1, 0, 12, 0, 0, 1, 0, 0, 49, 4088, 4081, 0, 4081, 262, 0, 2, 0, 0, 1, 861, 0, 696, 0, 3280, 3, 0, 0, 7, 0, 0, 3864] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[5, 0, 0, 0, 2003, 0, 0, 2362, 4040, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 0, 0, 0, 0, 2, 0, 0, 2159, 0, 0, 10, 0, 0, 0, 0, 1, 9, 0, 4010, 3994, 0, 4079, 1516, 0, 0, 0, 0, 0, 176, 30, 456, 0, 3685, 0, 0, 0, 0, 0, 0, 4062] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[44, 965, 0, 1, 19, 0, 0, 2990, 4084, 0, 25, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 14, 0, 797, 1, 0, 0, 0, 487, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 4063, 3936, 1, 2314, 3347, 0, 0, 0, 0, 0, 822, 14, 459, 0, 2738, 0, 0, 0, 0, 0, 0, 1515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 28, 0, 0, 3752, 0, 0, 1844, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 2181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 4069, 4096, 0, 3984, 847, 0, 0, 0, 0, 0, 214, 0, 5, 0, 3547, 0, 0, 0, 0, 0, 0, 4069] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3, 1, 0, 0, 2731, 0, 0, 258, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 16, 0, 7, 0, 3, 2, 0, 23, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 10, 7, 0, 1, 318, 3969, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 9, 0, 2931, 876, 1089, 0, 0, 0, 0, 0, 0, 0, 1, 0, 12, 0, 0, 1, 0, 4081, 44, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 2798, 262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 861, 0, 696, 0, 2844, 407, 75, 0, 0, 0, 0, 0, 3280, 3, 0, 0, 7, 0, 0, 1020, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2993, 4111, 4309, 4911, 4139, 3109, 4886, 4310] avg=4096.0000 max=4911 max_rank=3 min=2993 min_rank=0 imbalance_ratio=1.198975 max_load_reduction=27857 max_load_reduction_pct=0.850128 moved_to_echo=16444 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 48, 0, 21, 2128, 0, 0, 1728, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 13, 4, 0, 0, 0, 0, 0, 1, 3536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2661, 834, 849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 38, 0, 0, 0, 0, 0, 0, 2, 27, 0, 0, 0, 2665, 2602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 671, 0, 1517, 0, 3003, 387, 68, 0, 0, 0, 0, 0, 690, 0, 0, 0, 0, 0, 0, 1077, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3925, 4103, 3554, 4344, 4132, 5296, 5647, 1767] avg=4096.0000 max=5647 max_rank=6 min=1767 min_rank=7 imbalance_ratio=1.378662 max_load_reduction=27121 max_load_reduction_pct=0.827667 moved_to_echo=16842 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[5, 195, 0, 1, 1791, 0, 0, 2564, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 6, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 1, 0, 180, 3954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 124, 11, 2, 2846, 875, 714, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 4, 3978, 43, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 12, 2795, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 25, 6, 112, 0, 3011, 406, 74, 0, 0, 0, 0, 0, 3766, 0, 0, 0, 0, 0, 0, 1080, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4556, 4114, 4140, 4611, 4029, 2837, 3635, 4846] avg=4096.0000 max=4846 max_rank=7 min=2837 min_rank=5 imbalance_ratio=1.183105 max_load_reduction=27922 max_load_reduction_pct=0.852112 moved_to_echo=15347 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 275, 0, 1, 1661, 1, 0, 2783, 0, 0, 0, 0, 0, 0, 0, 0, 4010, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 3836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 1368, 816, 662, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4087, 42, 0, 0, 0, 0, 0, 0, 0, 841, 0, 0, 0, 2609, 1157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 10, 2, 15, 721, 0, 2984, 379, 72, 0, 0, 0, 0, 0, 3231, 0, 0, 0, 0, 0, 0, 1070, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4721, 4024, 3879, 2915, 4132, 4607, 4189, 4301] avg=4096.0000 max=4721 max_rank=0 min=2915 min_rank=3 imbalance_ratio=1.152588 max_load_reduction=28047 max_load_reduction_pct=0.855927 moved_to_echo=17229 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 17, 0, 3, 2669, 0, 0, 1546, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 136, 3972, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 15, 4, 0, 1279, 834, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 43, 0, 0, 0, 0, 0, 0, 0, 255, 0, 0, 0, 2664, 1147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 82, 0, 453, 0, 2870, 387, 76, 0, 0, 0, 0, 0, 4002, 1, 0, 0, 0, 0, 0, 1029, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4235, 4119, 4111, 3198, 4136, 4066, 3871, 5032] avg=4096.0000 max=5032 max_rank=7 min=3198 min_rank=3 imbalance_ratio=1.228516 max_load_reduction=27736 max_load_reduction_pct=0.846436 moved_to_echo=17105 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[44, 965, 0, 1, 14, 0, 0, 2990, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 14, 0, 797, 3946, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 487, 0, 0, 4092, 496, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 3936, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1588, 3347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 822, 14, 459, 0, 1115, 230, 74, 0, 0, 0, 0, 0, 2738, 0, 0, 0, 0, 0, 0, 400, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4014, 4109, 4763, 5081, 4013, 4936, 2714, 3138] avg=4096.0000 max=5081 max_rank=3 min=2714 min_rank=6 imbalance_ratio=1.240479 max_load_reduction=27687 max_load_reduction_pct=0.844940 moved_to_echo=14801 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 0, 0, 1424, 0, 0, 2362, 0, 0, 0, 0, 0, 0, 0, 0, 4040, 0, 15, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 3894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2159, 875, 567, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 1, 3994, 43, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 2797, 1516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 30, 456, 0, 2990, 407, 73, 0, 0, 0, 0, 0, 3685, 0, 0, 0, 0, 0, 0, 1072, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3791, 4067, 4048, 3603, 4048, 4322, 4132, 4757] avg=4096.0000 max=4757 max_rank=7 min=3603 min_rank=3 imbalance_ratio=1.161377 max_load_reduction=28011 max_load_reduction_pct=0.854828 moved_to_echo=17259 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 28, 0, 0, 2661, 0, 0, 1844, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 3952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2181, 858, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 43, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 2724, 847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 214, 0, 5, 0, 2999, 402, 74, 0, 0, 0, 0, 0, 3547, 0, 0, 0, 0, 0, 0, 1070, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4533, 4121, 3964, 4105, 4139, 3595, 3694, 4617] avg=4096.0000 max=4617 max_rank=7 min=3595 min_rank=5 imbalance_ratio=1.127197 max_load_reduction=28151 max_load_reduction_pct=0.859100 moved_to_echo=16045 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 0, 128, 768, 128, 128, 19584, 6528, 6144, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 2944, 128, 4480, 0, 21888, 3072, 640, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 1536, 0, 128, 15104, 128, 0, 16128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[24960, 128, 0, 0, 128, 0, 0, 7936, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 1280, 0, 0, 128, 20736, 11008, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 128, 128, 128, 1664, 31104, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 0, 0, 128, 128, 32384, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[32640, 128, 128, 128, 128, 128, 128, 0, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 4059, 1, 126, 3, 0, 0, 0, 0, 3941, 0, 452, 0, 0, 0, 543, 42, 264, 0, 0, 0, 7, 3907, 1, 0, 0, 0, 264, 0, 0, 4090, 0, 0, 0, 0, 7, 0, 0, 1, 0, 0, 2, 2945, 59, 0, 3606, 0, 0, 4096, 0, 0, 0, 0, 0, 258, 0, 0, 4094, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[7, 0, 0, 183, 0, 6, 0, 0, 0, 12, 0, 4086, 0, 1134, 0, 0, 7, 1442, 3580, 0, 5, 0, 437, 0, 133, 0, 0, 0, 0, 576, 3, 8, 4074, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3935, 4, 11, 2337, 0, 1, 4088, 0, 3, 0, 1, 8, 2949, 0, 0, 3722, 0, 7, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1, 0, 0, 3611, 0, 182, 0, 0, 0, 0, 0, 4075, 0, 95, 3, 0, 0, 2440, 372, 24, 0, 0, 0, 0, 205, 22, 0, 0, 0, 144, 0, 0, 4002, 0, 0, 0, 0, 9, 133, 0, 0, 6, 0, 1, 4019, 71, 0, 3590, 0, 0, 4072, 0, 0, 0, 0, 0, 4058, 0, 0, 1633, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 3895, 0, 82, 0, 0, 1, 0, 0, 4040, 0, 0, 0, 0, 0, 316, 2829, 56, 0, 0, 2, 0, 132, 0, 21, 0, 0, 1251, 0, 0, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2844, 395, 0, 3431, 0, 0, 4056, 0, 0, 0, 0, 0, 2690, 0, 0, 2707, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 234, 0, 551, 0, 0, 2, 0, 0, 3924, 0, 20, 11, 0, 1, 776, 3875, 1, 0, 0, 0, 0, 342, 51, 0, 1, 0, 51, 0, 0, 4066, 0, 0, 0, 0, 1, 221, 4, 0, 0, 0, 0, 3401, 762, 0, 2213, 0, 0, 4095, 1, 0, 0, 0, 0, 4096, 4, 0, 4064, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 1012, 0, 24, 0, 0, 0, 0, 0, 3967, 2, 47, 2, 1, 0, 1987, 1244, 13, 0, 1, 14, 0, 2336, 6, 0, 0, 0, 552, 1, 0, 4067, 0, 0, 0, 0, 2, 35, 0, 0, 0, 0, 0, 3700, 25, 0, 4043, 0, 1, 4087, 0, 0, 0, 0, 1, 2290, 0, 0, 3307, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 508, 0, 91, 0, 0, 0, 0, 0, 4087, 0, 115, 0, 0, 0, 15, 3959, 9, 0, 0, 0, 0, 53, 161, 0, 0, 0, 628, 0, 0, 3535, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4086, 965, 0, 2625, 0, 0, 4096, 0, 0, 0, 0, 0, 4057, 0, 0, 3777, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 3, 8, 2806, 0, 0, 7, 25, 0, 0, 0, 3987, 0, 1453, 0, 0, 0, 53, 2619, 0, 0, 3, 302, 0, 1160, 443, 32, 0, 0, 0, 0, 0, 3960, 0, 0, 0, 15, 0, 11, 0, 0, 1, 3, 0, 3726, 422, 0, 1575, 0, 0, 4012, 0, 0, 1, 1, 0, 3702, 0, 0, 2438, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4059, 1, 126, 3, 0, 2161, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3612, 0, 452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 42, 264, 0, 0, 0, 7, 555, 249, 0, 0, 0, 0, 0, 0, 3907, 1, 0, 0, 0, 264, 0, 0, 2084, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 7, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 681, 59, 0, 3606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 258, 0, 0, 1378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6530, 4064, 1660, 6256, 4159, 4349, 4114, 1636] avg=4096.0000 max=6530 max_rank=0 min=1636 min_rank=7 imbalance_ratio=1.594238 max_load_reduction=26238 max_load_reduction_pct=0.800720 moved_to_echo=14258 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 0, 183, 0, 6, 0, 0, 1965, 240, 0, 0, 0, 0, 0, 0, 0, 12, 0, 3746, 0, 1134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1442, 3580, 0, 5, 0, 437, 0, 505, 258, 0, 0, 0, 0, 0, 0, 133, 0, 0, 0, 0, 576, 3, 8, 2785, 0, 0, 0, 0, 0, 0, 0, 4074, 9, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 910, 4, 11, 2337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4088, 0, 3, 0, 1, 8, 18, 0, 0, 0, 0, 0, 0, 0, 2949, 0, 0, 1252, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2401, 4892, 6234, 3505, 4147, 3262, 4119, 4208] avg=4096.0000 max=6234 max_rank=2 min=2401 min_rank=0 imbalance_ratio=1.521973 max_load_reduction=26534 max_load_reduction_pct=0.809753 moved_to_echo=15736 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 234, 0, 551, 0, 0, 2145, 207, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3597, 0, 20, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 776, 3875, 1, 0, 0, 0, 0, 551, 248, 0, 0, 0, 0, 0, 0, 342, 51, 0, 1, 0, 51, 0, 0, 2407, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 1, 221, 4, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 787, 762, 0, 2213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 1368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3137, 3630, 5452, 2852, 4353, 3762, 4114, 5468] avg=4096.0000 max=5468 max_rank=7 min=2852 min_rank=3 imbalance_ratio=1.334961 max_load_reduction=27300 max_load_reduction_pct=0.833130 moved_to_echo=17697 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 3611, 0, 182, 0, 0, 862, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3735, 0, 95, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2440, 372, 24, 0, 0, 0, 0, 221, 258, 0, 0, 0, 0, 0, 0, 205, 22, 0, 0, 0, 144, 0, 0, 2844, 0, 0, 0, 0, 0, 0, 0, 4002, 0, 0, 0, 0, 9, 133, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1, 930, 71, 0, 3590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4901, 3833, 3315, 3215, 4208, 4598, 4090, 4608] avg=4096.0000 max=4901 max_rank=0 min=3215 min_rank=3 imbalance_ratio=1.196533 max_load_reduction=27867 max_load_reduction_pct=0.850433 moved_to_echo=17504 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1012, 0, 24, 0, 0, 1745, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3636, 2, 47, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1987, 1244, 13, 0, 1, 14, 0, 448, 251, 0, 0, 0, 0, 0, 0, 2336, 6, 0, 0, 0, 552, 1, 0, 2619, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 2, 35, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 855, 25, 0, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4087, 0, 0, 0, 0, 1, 18, 0, 0, 0, 0, 0, 0, 0, 2290, 0, 0, 1114, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3007, 3688, 3958, 5514, 4166, 4923, 4107, 3405] avg=4096.0000 max=5514 max_rank=3 min=3007 min_rank=0 imbalance_ratio=1.346191 max_load_reduction=27254 max_load_reduction_pct=0.831726 moved_to_echo=16601 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 8, 2806, 0, 0, 7, 25, 1287, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3655, 0, 1453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 2619, 0, 0, 3, 302, 0, 331, 252, 0, 0, 0, 0, 0, 0, 1160, 443, 32, 0, 0, 0, 0, 0, 2637, 0, 0, 0, 0, 0, 0, 0, 3960, 0, 0, 0, 15, 0, 11, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 862, 422, 0, 1575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4012, 0, 0, 1, 1, 0, 18, 0, 0, 0, 0, 0, 0, 0, 3702, 0, 0, 820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4363, 5108, 3560, 4272, 4048, 2863, 4032, 4522] avg=4096.0000 max=5108 max_rank=1 min=2863 min_rank=5 imbalance_ratio=1.247070 max_load_reduction=27660 max_load_reduction_pct=0.844116 moved_to_echo=15465 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3895, 0, 82, 0, 0, 1429, 173, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 316, 2829, 56, 0, 0, 2, 0, 367, 256, 0, 0, 0, 0, 0, 0, 132, 0, 21, 0, 0, 1251, 0, 0, 2013, 0, 0, 0, 0, 0, 0, 0, 4020, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 658, 395, 0, 3431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 2690, 0, 0, 911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5579, 3704, 3826, 3417, 4083, 4484, 4074, 3601] avg=4096.0000 max=5579 max_rank=0 min=3417 min_rank=3 imbalance_ratio=1.362061 max_load_reduction=27189 max_load_reduction_pct=0.829742 moved_to_echo=16242 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 508, 0, 91, 0, 0, 1997, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3734, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 3959, 9, 0, 0, 0, 0, 517, 263, 0, 0, 0, 0, 0, 0, 53, 161, 0, 0, 0, 628, 0, 0, 2895, 0, 0, 0, 0, 0, 0, 0, 3535, 0, 0, 0, 0, 0, 1, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 937, 965, 0, 2625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 0, 1263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2850, 3849, 4763, 3737, 3604, 4527, 4118, 5320] avg=4096.0000 max=5320 max_rank=7 min=2850 min_rank=0 imbalance_ratio=1.298828 max_load_reduction=27448 max_load_reduction_pct=0.837646 moved_to_echo=17569 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[24192, 128, 0, 8704, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 7680, 18560, 384, 128, 128, 768, 128, 3584, 2048, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 6656, 2816, 128, 23424, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 16384, 128, 1152, 128, 128, 13696, 1792, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[8320, 768, 128, 128, 0, 3584, 128, 128, 20352, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 0, 29440, 128, 3328, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[31872, 128, 0, 0, 128, 128, 512, 128, 512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[0, 128, 32640, 128, 128, 128, 128, 128, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33792 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[2, 3, 2, 0, 2548, 1797, 0, 0, 0, 0, 0, 0, 246, 186, 0, 0, 14, 0, 4, 0, 0, 0, 1, 0, 0, 0, 3298, 4086, 0, 2728, 0, 4080, 0, 0, 0, 0, 4096, 0, 2831, 0, 0, 0, 0, 0, 0, 0, 0, 26, 1, 0, 11, 0, 0, 3042, 3752, 0, 0, 0, 0, 0, 0, 14, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1708, 10, 0, 0, 0, 0, 0, 0, 3812, 1047, 0, 0, 5, 0, 0, 0, 0, 0, 0, 143, 0, 3, 4093, 4096, 0, 3126, 0, 3989, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 2552, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 8, 15, 3402, 1307, 0, 0, 12, 0, 3, 0, 79, 542, 0, 29, 0, 2, 0, 0, 0, 2, 1, 0, 0, 0, 3989, 4080, 0, 2196, 0, 4039, 15, 0, 10, 0, 4083, 2, 1451, 0, 0, 0, 0, 1, 35, 0, 26, 0, 0, 0, 0, 0, 13, 3281, 4084, 6, 0, 0, 8, 3, 0, 22, 8, 14] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 10, 0, 2692, 500, 0, 0, 0, 0, 0, 0, 2981, 676, 0, 1, 0, 0, 0, 7, 0, 1, 0, 653, 0, 0, 3787, 4093, 2, 1261, 0, 4059, 4, 0, 0, 0, 4096, 0, 61, 3, 1, 1, 140, 0, 0, 0, 52, 8, 0, 0, 0, 0, 0, 3654, 3811, 0, 0, 0, 0, 0, 0, 214, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 2922, 1512, 0, 1, 0, 0, 0, 0, 1430, 1805, 0, 0, 0, 0, 0, 2, 0, 0, 0, 5, 0, 0, 4096, 4093, 1, 700, 0, 4071, 0, 0, 0, 0, 4095, 1, 110, 0, 0, 0, 679, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 3167, 4070, 1, 0, 0, 0, 0, 0, 2, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 218, 0, 2228, 605, 0, 0, 0, 0, 0, 0, 847, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 3848, 4096, 0, 4090, 0, 4095, 0, 0, 0, 0, 4096, 0, 1470, 0, 0, 0, 5, 0, 0, 0, 0, 1, 0, 0, 20, 0, 0, 3162, 3802, 0, 0, 0, 0, 0, 0, 17, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 415, 0, 2575, 238, 0, 0, 0, 0, 0, 0, 32, 994, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 4096, 0, 3822, 2, 4093, 4, 0, 0, 0, 4095, 0, 113, 0, 0, 0, 1, 0, 0, 0, 40, 35, 0, 0, 74, 0, 0, 2999, 3541, 0, 0, 0, 0, 0, 0, 1516, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[6, 0, 0, 0, 1453, 1902, 0, 0, 0, 0, 1, 0, 3469, 3357, 0, 1, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4093, 4082, 0, 3385, 0, 4081, 0, 0, 0, 0, 4067, 0, 34, 0, 0, 4, 0, 0, 0, 0, 39, 19, 0, 0, 0, 0, 0, 248, 2462, 0, 53, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8, 15, 3402, 1307, 0, 0, 584, 0, 0, 0, 0, 0, 0, 0, 12, 0, 3, 0, 79, 542, 0, 29, 1334, 172, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 1, 0, 3977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1461, 0, 0, 2196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 10, 0, 3327, 2, 1451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 35, 0, 26, 0, 3830, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 3281, 1455, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3, 0, 22, 8, 14, 2528, 1295, 209, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5316, 2171, 3982, 3657, 4805, 3995, 4755, 4087] avg=4096.0000 max=5316 max_rank=0 min=2171 min_rank=1 imbalance_ratio=1.297852 max_load_reduction=27452 max_load_reduction_pct=0.837769 moved_to_echo=17642 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1708, 10, 0, 0, 586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3812, 1047, 0, 0, 1333, 172, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 143, 3994, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1499, 0, 0, 3126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 3, 3782, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2552, 1453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2594, 1294, 207, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2304, 6364, 4142, 4628, 3339, 3891, 4005, 4095] avg=4096.0000 max=6364 max_rank=1 min=2304 min_rank=0 imbalance_ratio=1.553711 max_load_reduction=26404 max_load_reduction_pct=0.805786 moved_to_echo=15330 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2, 3, 2, 0, 2548, 1797, 0, 0, 586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 186, 0, 0, 1226, 172, 0, 0, 0, 0, 0, 0, 14, 0, 4, 0, 0, 0, 1, 0, 3984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1208, 0, 0, 2728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3338, 0, 2831, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 3870, 102, 0, 0, 0, 0, 0, 0, 1, 0, 11, 0, 0, 3042, 1336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 2090, 1190, 210, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4938, 1830, 4003, 3936, 6169, 3998, 4390, 3504] avg=4096.0000 max=6169 max_rank=4 min=1830 min_rank=1 imbalance_ratio=1.506104 max_load_reduction=26599 max_load_reduction_pct=0.811737 moved_to_echo=18061 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 10, 0, 2692, 500, 0, 0, 586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2981, 676, 0, 1, 1245, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 1, 0, 653, 3990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1387, 0, 2, 1261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 3338, 0, 61, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 140, 0, 0, 0, 52, 8, 3849, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3654, 1358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 214, 0, 0, 2400, 1208, 210, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3788, 5075, 4651, 2650, 3406, 4154, 5012, 4032] avg=4096.0000 max=5075 max_rank=1 min=2650 min_rank=3 imbalance_ratio=1.239014 max_load_reduction=27693 max_load_reduction_pct=0.845123 moved_to_echo=16604 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 2922, 1512, 0, 1, 586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1430, 1805, 0, 0, 1330, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 5, 3990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1500, 0, 1, 700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3337, 1, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 679, 0, 0, 0, 1, 0, 3860, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3167, 1449, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 2596, 1291, 211, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5023, 4737, 3997, 2201, 3448, 4643, 4618, 4101] avg=4096.0000 max=5023 max_rank=0 min=2201 min_rank=3 imbalance_ratio=1.226318 max_load_reduction=27745 max_load_reduction_pct=0.846710 moved_to_echo=16810 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 415, 0, 2575, 238, 0, 0, 586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 994, 0, 0, 1157, 172, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 3993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1492, 0, 0, 3822, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 3337, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 40, 35, 3881, 103, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 2999, 1261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1516, 0, 0, 2582, 1123, 212, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3814, 2355, 4002, 5316, 3454, 4060, 4334, 5433] avg=4096.0000 max=5433 max_rank=7 min=2355 min_rank=1 imbalance_ratio=1.326416 max_load_reduction=27335 max_load_reduction_pct=0.834198 moved_to_echo=17281 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 0, 0, 1453, 1902, 0, 0, 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3469, 3357, 0, 1, 804, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 3980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1499, 0, 0, 3385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3314, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 39, 19, 3871, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 248, 878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 2594, 780, 210, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3943, 7803, 3992, 4884, 3348, 4035, 1126, 3637] avg=4096.0000 max=7803 max_rank=1 min=1126 min_rank=6 imbalance_ratio=1.905029 max_load_reduction=24965 max_load_reduction_pct=0.761871 moved_to_echo=12146 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 218, 0, 2228, 605, 0, 0, 591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 847, 163, 0, 0, 1247, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1406, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3329, 0, 1470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 1, 3884, 102, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 3162, 1346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 2442, 1209, 211, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3642, 2433, 3999, 5496, 4799, 3992, 4528, 3879] avg=4096.0000 max=5496 max_rank=3 min=2433 min_rank=1 imbalance_ratio=1.341797 max_load_reduction=27272 max_load_reduction_pct=0.832275 moved_to_echo=17198 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 0, 12928, 8832, 0, 128, 9728, 1408, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 0, 1792, 128, 128, 19840, 9472, 1792, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 128, 22144, 10624, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 11520, 0, 128, 21376, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 768, 128, 19584, 7936, 0, 128, 4736, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 26752, 128, 6144, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 896, 128, 128, 0, 256, 128, 30848, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 0, 128, 128, 896, 32000, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 9, 0, 0, 1, 13, 2, 0, 2, 0, 0, 0, 4000, 0, 0, 0, 0, 0, 9, 0, 0, 4096, 0, 0, 0, 16, 32, 0, 0, 0, 2, 0, 0, 4055, 4090, 4088, 39, 0, 0, 0, 1623, 0, 0, 0, 3666, 240, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 0, 0, 2660, 24] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 7, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 2350, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 10, 69, 0, 0, 0, 0, 0, 0, 4092, 4096, 3738, 7, 0, 0, 0, 378, 0, 0, 0, 4084, 3493, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2188, 42] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 34, 0, 0, 27, 0, 0, 0, 1, 0, 0, 0, 2655, 0, 0, 0, 0, 2, 2, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4077, 4041, 4088, 15, 2, 0, 0, 2510, 0, 1, 1, 3746, 2107, 0, 21, 0, 0, 0, 1, 1, 0, 0, 1, 4093, 0, 1, 2, 1241, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 54, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 3927, 0, 0, 0, 0, 0, 78, 0, 0, 3935, 0, 0, 1, 0, 230, 0, 0, 0, 0, 0, 0, 1617, 4011, 3491, 377, 0, 0, 0, 2399, 0, 0, 0, 4032, 4041, 6, 0, 0, 0, 0, 0, 4, 1, 0, 0, 4088, 0, 0, 0, 45, 427] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[3, 5, 0, 2, 0, 0, 2, 0, 0, 7, 0, 6, 0, 18, 3723, 0, 0, 12, 18, 0, 0, 0, 0, 4084, 0, 1, 0, 0, 0, 1, 0, 8, 0, 0, 0, 4087, 4058, 4070, 0, 0, 13, 0, 3288, 11, 0, 2, 3030, 2226, 0, 0, 0, 8, 0, 0, 0, 0, 0, 4, 4081, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 0, 0, 0, 33, 0, 0, 1, 0, 0, 0, 3970, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 16, 60, 0, 0, 0, 0, 0, 0, 4015, 4086, 3978, 1, 0, 0, 0, 1288, 0, 0, 0, 2754, 3123, 0, 3, 0, 0, 2, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 737, 539] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3896, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2723, 4094, 3474, 0, 0, 0, 0, 3803, 0, 0, 0, 3834, 1658, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4093, 0, 0, 0, 736, 131] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 4, 0, 2, 0, 0, 0, 0, 0, 0, 4057, 0, 0, 0, 0, 0, 75, 0, 0, 4036, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 3865, 4074, 3643, 26, 41, 0, 27, 412, 0, 3, 0, 2958, 2463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 0, 0, 0, 2647, 401] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 7, 0, 0, 0, 3, 4063, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2350, 0, 410, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 69, 0, 0, 0, 2748, 1233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 457, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 378, 0, 0, 0, 0, 3071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 4096, 21, 0, 0, 0, 0, 0, 0, 0, 0, 2628, 0, 0, 0, 2188, 42, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4606, 2995, 4108, 4060, 4556, 3449, 4136, 4858] avg=4096.0000 max=4858 max_rank=7 min=2995 min_rank=1 imbalance_ratio=1.186035 max_load_reduction=27910 max_load_reduction_pct=0.851746 moved_to_echo=16999 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3, 5, 0, 2, 0, 0, 2, 0, 3014, 581, 0, 0, 0, 0, 0, 0, 0, 7, 0, 6, 0, 18, 3723, 0, 261, 235, 0, 0, 0, 0, 0, 0, 0, 12, 18, 0, 0, 0, 0, 4084, 8, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 8, 2992, 1229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 3288, 11, 0, 2, 0, 1957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 4058, 16, 0, 0, 0, 0, 0, 0, 0, 4, 2617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3607, 4250, 4122, 4231, 4584, 5271, 4082, 2621] avg=4096.0000 max=5271 max_rank=5 min=2621 min_rank=7 imbalance_ratio=1.286865 max_load_reduction=27497 max_load_reduction_pct=0.839142 moved_to_echo=16558 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 9, 0, 0, 1, 13, 3647, 583, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 4000, 0, 28, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 32, 0, 0, 0, 3005, 1233, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4055, 0, 500, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1623, 0, 0, 0, 0, 212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4090, 19, 0, 0, 0, 0, 0, 0, 0, 0, 2628, 0, 0, 0, 2660, 24, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4253, 4267, 4105, 4286, 4596, 1835, 4114, 5312] avg=4096.0000 max=5312 max_rank=7 min=1835 min_rank=5 imbalance_ratio=1.296875 max_load_reduction=27456 max_load_reduction_pct=0.837891 moved_to_echo=15857 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 54, 0, 0, 0, 4, 4012, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3927, 0, 474, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 3935, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 230, 0, 0, 0, 2566, 1231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1617, 0, 427, 377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2399, 0, 0, 0, 0, 3553, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4, 1, 4011, 20, 0, 0, 0, 0, 0, 0, 0, 0, 2622, 0, 0, 0, 45, 427, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4568, 4636, 4027, 4028, 2421, 5952, 4042, 3094] avg=4096.0000 max=5952 max_rank=5 min=2421 min_rank=4 imbalance_ratio=1.453125 max_load_reduction=26816 max_load_reduction_pct=0.818359 moved_to_echo=15509 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 34, 0, 0, 27, 0, 3726, 583, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2655, 0, 247, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 4095, 7, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 3005, 1232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 500, 15, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2510, 0, 1, 1, 0, 1853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 1, 1, 0, 4041, 20, 0, 0, 0, 0, 0, 0, 0, 1, 2626, 0, 1, 2, 1241, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4370, 3138, 4106, 4239, 4594, 4365, 4084, 3872] avg=4096.0000 max=4594 max_rank=4 min=3138 min_rank=1 imbalance_ratio=1.121582 max_load_reduction=28174 max_load_reduction_pct=0.859802 moved_to_echo=16915 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 4, 0, 2, 2943, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 289, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 4036, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 2678, 1211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3865, 0, 445, 26, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 412, 0, 3, 0, 0, 2166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 15, 0, 0, 0, 0, 0, 0, 0, 0, 2582, 0, 0, 0, 2647, 401, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3470, 4577, 4119, 3898, 4377, 2608, 4089, 5630] avg=4096.0000 max=5630 max_rank=7 min=2608 min_rank=5 imbalance_ratio=1.374512 max_load_reduction=27138 max_load_reduction_pct=0.828186 moved_to_echo=16704 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 231, 0, 0, 0, 0, 3815, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3896, 0, 197, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2559, 1237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2723, 0, 415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3803, 0, 0, 0, 0, 1451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 19, 0, 0, 0, 0, 0, 0, 0, 0, 2616, 0, 0, 0, 736, 131, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4546, 4333, 4103, 3796, 3138, 5254, 4115, 3483] avg=4096.0000 max=5254 max_rank=5 min=3138 min_rank=4 imbalance_ratio=1.282715 max_load_reduction=27514 max_load_reduction_pct=0.839661 moved_to_echo=15990 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 0, 0, 0, 33, 2739, 568, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3970, 0, 366, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 60, 0, 0, 0, 2924, 1230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4015, 0, 486, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1288, 0, 0, 0, 0, 2746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2, 0, 0, 0, 4086, 15, 0, 0, 0, 0, 0, 0, 0, 0, 2622, 0, 0, 0, 737, 539, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3348, 4572, 4078, 4230, 4502, 4034, 4106, 3898] avg=4096.0000 max=4572 max_rank=1 min=3348 min_rank=0 imbalance_ratio=1.116211 max_load_reduction=28196 max_load_reduction_pct=0.860474 moved_to_echo=16540 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 15744, 128, 128, 128, 0, 17024, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 512, 128, 0, 128, 22528, 9856, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 0, 384, 128, 128, 128, 128, 28032, 4480, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 20992, 0, 128, 128, 10368, 1664, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 0, 28544, 0, 3840, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 0, 128, 28672, 0, 2304, 1920, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 256, 0, 0, 32512, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 128, 128, 128, 128, 128, 32640, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33792 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 28, 0, 2954, 0, 0, 4093, 7, 2729, 106, 0, 0, 3783, 0, 0, 0, 106, 0, 0, 4092, 0, 0, 0, 4095, 0, 0, 4083, 0, 3033, 0, 0, 0, 0, 0, 2, 3477, 83, 0, 0, 2, 0, 0, 0, 77, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 2109, 0, 0, 4095, 2, 3281, 341, 0, 0, 3534, 0, 0, 0, 225, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 1162, 0, 0, 0, 0, 0, 0, 1858, 3297, 0, 0, 1, 0, 0, 0, 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 23, 4, 3911, 9, 0, 4071, 0, 2727, 3, 0, 0, 3102, 0, 12, 0, 12, 23, 0, 4083, 0, 0, 4, 4036, 0, 0, 4071, 0, 1079, 24, 0, 10, 0, 0, 1, 2208, 1310, 21, 5, 106, 0, 0, 0, 1160, 713, 0, 8, 0, 0, 0, 0, 0, 13, 9, 0, 0, 0, 0, 1, 0, 0, 0, 9, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 1, 0, 3001, 0, 0, 4096, 0, 1895, 89, 0, 0, 4067, 0, 0, 0, 25, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 3999, 0, 84, 0, 0, 0, 0, 0, 0, 3901, 3373, 0, 0, 9, 0, 0, 0, 30, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 1, 0, 4066, 0, 0, 4096, 16, 3115, 42, 0, 0, 4027, 0, 0, 0, 10, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 3505, 0, 939, 0, 0, 0, 0, 0, 0, 3053, 221, 0, 0, 10, 0, 0, 0, 1475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 3985, 0, 0, 4096, 7, 448, 0, 0, 0, 4093, 0, 0, 1, 0, 0, 0, 4093, 0, 0, 0, 4090, 0, 0, 4075, 0, 2518, 0, 0, 0, 0, 1, 0, 3368, 1657, 0, 0, 10, 2, 0, 0, 199, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 27, 1, 4076, 1, 0, 4094, 0, 2505, 45, 0, 0, 3854, 2, 0, 1, 0, 0, 0, 4094, 0, 0, 1, 4095, 0, 0, 4083, 0, 2113, 0, 0, 0, 1, 0, 0, 1051, 2016, 0, 0, 18, 0, 0, 0, 687, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 760, 0, 997, 0, 0, 4096, 0, 370, 1334, 0, 0, 4080, 0, 0, 0, 322, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 3791, 0, 0, 0, 0, 0, 0, 589, 3319, 0, 0, 0, 0, 0, 0, 738, 15, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4, 56, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 23, 4, 3911, 9, 0, 844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2727, 3, 0, 0, 1395, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 23, 0, 0, 0, 0, 4, 3955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2292, 0, 1079, 24, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1959, 1310, 21, 5, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1160, 713, 0, 8, 0, 1779, 1026, 249, 81, 0, 0, 0, 0, 0, 0, 0, 0, 13, 9, 0, 0, 3226, 681, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 9, 0, 4083, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4791, 4137, 3994, 3405, 3402, 5016, 3929, 4094] avg=4096.0000 max=5016 max_rank=5 min=3402 min_rank=4 imbalance_ratio=1.224609 max_load_reduction=27752 max_load_reduction_pct=0.846924 moved_to_echo=16441 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2109, 0, 0, 849, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3281, 341, 0, 0, 1589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 225, 0, 0, 0, 0, 0, 0, 4014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2306, 0, 1162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1649, 3297, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 575, 0, 0, 0, 0, 1790, 1169, 209, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3245, 776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2960, 5211, 4239, 3468, 4947, 3825, 4021, 4097] avg=4096.0000 max=5211 max_rank=1 min=2960 min_rank=0 imbalance_ratio=1.272217 max_load_reduction=27557 max_load_reduction_pct=0.840973 moved_to_echo=16890 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 28, 0, 2954, 0, 0, 849, 7, 0, 0, 0, 0, 0, 0, 0, 0, 2729, 106, 0, 0, 1701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2299, 0, 3033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3085, 83, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 14, 0, 0, 0, 1784, 1252, 392, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3243, 830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3838, 4536, 4119, 5332, 3172, 3601, 4077, 4093] avg=4096.0000 max=5332 max_rank=3 min=3172 min_rank=4 imbalance_ratio=1.301758 max_load_reduction=27436 max_load_reduction_pct=0.837280 moved_to_echo=14943 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3985, 0, 0, 849, 7, 0, 0, 0, 0, 0, 0, 0, 0, 448, 0, 0, 0, 1840, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2294, 0, 2518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2989, 1657, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 199, 106, 0, 0, 0, 1781, 1354, 379, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 3246, 899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4841, 2289, 4008, 4812, 4657, 3903, 4163, 4095] avg=4096.0000 max=4841 max_rank=0 min=2289 min_rank=1 imbalance_ratio=1.181885 max_load_reduction=27927 max_load_reduction_pct=0.852264 moved_to_echo=16818 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 27, 1, 4076, 1, 0, 849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2505, 45, 0, 0, 1733, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2299, 0, 2113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 933, 2016, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 687, 0, 0, 0, 0, 1784, 1275, 118, 82, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 3244, 846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4954, 4286, 4014, 4412, 2968, 3946, 4093, 4095] avg=4096.0000 max=4954 max_rank=0 min=2968 min_rank=4 imbalance_ratio=1.209473 max_load_reduction=27814 max_load_reduction_pct=0.848816 moved_to_echo=15102 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 4066, 0, 0, 849, 16, 0, 0, 0, 0, 0, 0, 0, 0, 3115, 42, 0, 0, 1810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 4014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1973, 0, 939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2709, 221, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1475, 0, 0, 0, 0, 1532, 1333, 344, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3246, 884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4932, 4967, 4024, 2912, 2940, 4766, 4130, 4097] avg=4096.0000 max=4967 max_rank=1 min=2912 min_rank=3 imbalance_ratio=1.212646 max_load_reduction=27801 max_load_reduction_pct=0.848419 moved_to_echo=15933 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 760, 0, 997, 0, 0, 849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 370, 1334, 0, 0, 1834, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 322, 0, 0, 0, 0, 0, 0, 4014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2306, 0, 3791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 3319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 738, 15, 0, 0, 0, 1790, 1350, 66, 82, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4, 56, 3246, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2606, 3538, 4336, 6097, 3842, 4041, 4211, 4097] avg=4096.0000 max=6097 max_rank=3 min=2606 min_rank=0 imbalance_ratio=1.488525 max_load_reduction=26671 max_load_reduction_pct=0.813934 moved_to_echo=16191 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 3001, 0, 0, 844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1895, 89, 0, 0, 1820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 4009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2246, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3458, 3373, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 6, 0, 0, 0, 1753, 1351, 443, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3248, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3846, 3804, 4034, 2330, 6840, 3670, 4144, 4100] avg=4096.0000 max=6840 max_rank=4 min=2330 min_rank=3 imbalance_ratio=1.669922 max_load_reduction=25928 max_load_reduction_pct=0.791260 moved_to_echo=18754 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 0, 4992, 896, 0, 128, 0, 14080, 10112, 2304, 768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 18048, 0, 14720, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[17152, 2048, 0, 0, 13824, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 17408, 15360, 128, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 0, 128, 128, 128, 128, 128, 128, 25984, 6784, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 896, 128, 25216, 128, 0, 6784, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 128, 0, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[768, 128, 0, 0, 0, 0, 128, 32128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 49, 1, 0, 0, 0, 0, 0, 54, 0, 3401, 0, 0, 199, 1223, 14, 3431, 0, 0, 0, 0, 4069, 0, 0, 3052, 4, 0, 0, 0, 0, 10, 0, 0, 11, 0, 0, 1, 0, 42, 0, 4043, 0, 22, 20, 0, 1699, 0, 3935, 51, 20, 5, 0, 3315, 4036, 0, 0, 0, 9, 18, 14, 20] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 2, 0, 3342, 0, 0, 1159, 1035, 0, 3729, 0, 0, 0, 1, 4080, 0, 0, 2062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 119, 0, 4096, 0, 0, 0, 0, 864, 0, 4093, 0, 0, 0, 0, 4080, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 27, 3914, 0, 0, 884, 2439, 0, 2548, 0, 0, 0, 2, 4064, 0, 0, 2090, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 131, 0, 4096, 0, 0, 0, 0, 658, 0, 3948, 0, 0, 0, 0, 3840, 4094, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 4, 125, 0, 3993, 1, 0, 0, 0, 4096, 0, 0, 3014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2, 0, 4096, 0, 0, 0, 0, 1132, 0, 4096, 0, 0, 0, 0, 4043, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3904, 0, 0, 903, 211, 0, 4077, 5, 0, 0, 0, 4095, 0, 0, 2744, 0, 0, 0, 3, 0, 4, 0, 8, 0, 0, 0, 0, 0, 67, 0, 4095, 0, 0, 0, 0, 2290, 0, 4095, 0, 1, 1, 0, 2172, 4090, 0, 0, 0, 1, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 28, 12, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 3603, 238, 0, 3870, 10, 0, 0, 14, 4096, 0, 0, 122, 0, 0, 0, 0, 303, 0, 0, 0, 0, 0, 0, 0, 8, 2, 0, 4096, 0, 0, 0, 0, 97, 0, 4006, 0, 0, 0, 0, 4081, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 3907, 0, 0, 90, 3178, 0, 3824, 0, 0, 0, 0, 4096, 0, 0, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 315, 0, 4096, 0, 0, 0, 0, 1220, 0, 4085, 0, 0, 0, 0, 3587, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 1, 8, 4095, 0, 0, 1995, 1192, 0, 3942, 2, 0, 0, 0, 4082, 0, 0, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 4096, 0, 0, 0, 0, 310, 0, 4064, 0, 0, 0, 0, 3703, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 49, 1, 0, 0, 3935, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 3401, 0, 0, 131, 99, 0, 0, 0, 0, 0, 0, 199, 1223, 14, 1662, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2288, 0, 0, 3052, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 11, 0, 0, 1, 1733, 1781, 374, 0, 0, 0, 0, 0, 0, 42, 0, 3944, 0, 22, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1699, 0, 0, 51, 20, 5, 0, 2810, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 0, 0, 0, 9, 18, 14, 20, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4021, 3685, 3098, 5344, 3910, 4028, 4585, 4097] avg=4096.0000 max=5344 max_rank=3 min=3098 min_rank=2 imbalance_ratio=1.304688 max_load_reduction=27424 max_load_reduction_pct=0.836914 moved_to_echo=16620 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 7, 0, 0, 4093, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3342, 0, 0, 161, 100, 0, 0, 0, 0, 0, 0, 1159, 1035, 0, 1806, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2294, 0, 0, 2062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1884, 1786, 461, 0, 0, 0, 0, 0, 2, 119, 0, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 864, 0, 0, 0, 0, 0, 0, 3458, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4139, 3605, 4001, 4356, 4132, 4117, 4322, 4096] avg=4096.0000 max=4356 max_rank=3 min=3605 min_rank=1 imbalance_ratio=1.063477 max_load_reduction=28412 max_load_reduction_pct=0.867065 moved_to_echo=16667 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 31, 0, 3948, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 3914, 0, 0, 152, 100, 0, 0, 0, 0, 0, 0, 884, 2439, 0, 1234, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2285, 0, 0, 2090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1287, 1779, 433, 0, 0, 0, 0, 0, 0, 131, 0, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 658, 0, 0, 0, 0, 0, 0, 3255, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4006, 4193, 4559, 4375, 3501, 4127, 3913, 4094] avg=4096.0000 max=4559 max_rank=2 min=3501 min_rank=4 imbalance_ratio=1.113037 max_load_reduction=28209 max_load_reduction_pct=0.860870 moved_to_echo=15635 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 160, 100, 0, 0, 0, 0, 0, 0, 4, 125, 0, 1934, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2303, 0, 0, 3014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2017, 1793, 456, 0, 0, 0, 0, 0, 7, 2, 0, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1132, 0, 0, 0, 0, 0, 0, 3427, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4138, 4323, 2064, 5317, 4266, 4005, 4559, 4096] avg=4096.0000 max=5317 max_rank=3 min=2064 min_rank=2 imbalance_ratio=1.298096 max_load_reduction=27451 max_load_reduction_pct=0.837738 moved_to_echo=16926 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 4095, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3904, 0, 0, 86, 100, 0, 0, 0, 0, 0, 0, 903, 211, 0, 1975, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2302, 0, 0, 2744, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8, 0, 0, 0, 0, 2059, 1793, 245, 0, 0, 0, 0, 0, 0, 67, 0, 3995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2290, 0, 0, 0, 1, 1, 0, 1841, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4139, 4090, 3094, 5049, 4109, 4062, 4133, 4092] avg=4096.0000 max=5049 max_rank=3 min=3094 min_rank=2 imbalance_ratio=1.232666 max_load_reduction=27719 max_load_reduction_pct=0.845917 moved_to_echo=16396 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 28, 12, 0, 4006, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 161, 100, 0, 0, 0, 0, 0, 0, 3603, 238, 0, 1874, 10, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 2303, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 303, 0, 0, 0, 0, 0, 0, 0, 1955, 1793, 461, 0, 0, 0, 0, 0, 8, 2, 0, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 0, 0, 0, 0, 3459, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4347, 5739, 2425, 4512, 4006, 3556, 4096] avg=4096.0000 max=5739 max_rank=2 min=2425 min_rank=3 imbalance_ratio=1.401123 max_load_reduction=27029 max_load_reduction_pct=0.824860 moved_to_echo=16170 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3, 2, 0, 4064, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8, 4095, 0, 0, 146, 100, 0, 0, 0, 0, 0, 0, 1995, 1192, 0, 1910, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2295, 0, 0, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1991, 1787, 418, 0, 0, 0, 0, 0, 0, 127, 0, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 310, 0, 0, 0, 0, 0, 0, 3139, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4350, 5099, 3345, 4196, 4123, 3449, 4096] avg=4096.0000 max=5099 max_rank=2 min=3345 min_rank=3 imbalance_ratio=1.244873 max_load_reduction=27669 max_load_reduction_pct=0.844391 moved_to_echo=15864 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4085, 43, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 3907, 0, 0, 147, 106, 0, 0, 0, 0, 0, 0, 90, 3178, 0, 1846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2298, 0, 0, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1935, 1798, 409, 0, 0, 0, 0, 0, 0, 315, 0, 3985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1220, 0, 0, 0, 0, 0, 0, 3031, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4175, 5114, 2557, 4142, 4300, 4251, 4101] avg=4096.0000 max=5114 max_rank=2 min=2557 min_rank=3 imbalance_ratio=1.248535 max_load_reduction=27654 max_load_reduction_pct=0.843933 moved_to_echo=16794 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[384, 128, 0, 128, 128, 0, 0, 128, 14976, 14336, 3328, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[18432, 0, 0, 14464, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8960, 9728, 128, 14336, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 0, 0, 128, 128, 128, 0, 24448, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 896, 0, 32000, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 128, 128, 128, 30720, 0, 0, 1152, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 128, 128, 128, 0, 32384, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[32768, 0, 0, 0, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 1831, 0, 0, 0, 56, 0, 0, 3948, 160, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 3, 0, 0, 4091, 4095, 3553, 0, 0, 0, 0, 133, 0, 0, 0, 3, 0, 0, 0, 25, 0, 0, 0, 0, 4085, 0, 0, 0, 4096, 0, 26, 0, 0, 0, 3252, 0, 0, 3395, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 119, 4, 0, 0, 60, 0, 14, 4068, 233, 0, 0, 2, 0, 0, 0, 0, 0, 0, 27, 42, 0, 0, 10, 4078, 4000, 4068, 0, 0, 5, 48, 179, 6, 0, 0, 0, 0, 0, 0, 4, 0, 23, 23, 18, 3973, 0, 0, 4, 4072, 0, 0, 0, 1, 0, 4051, 4, 19, 3321, 5, 0, 172, 9, 106, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 290, 0, 0, 0, 198, 0, 0, 4008, 21, 0, 5, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 4063, 4096, 4082, 0, 0, 0, 0, 75, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3720, 0, 0, 0, 4093, 0, 71, 0, 0, 0, 4058, 0, 0, 3937, 0, 5, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 356, 0, 0, 0, 17, 0, 0, 4072, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 492, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3775, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 4096, 0, 0, 3549, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 3007, 20, 0, 0, 480, 12, 0, 3687, 42, 2, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4011, 4095, 3629, 0, 0, 0, 0, 467, 0, 0, 64, 0, 0, 0, 0, 102, 0, 0, 0, 0, 2086, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3949, 0, 0, 2998, 0, 12, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 1319, 1, 0, 0, 188, 0, 0, 3923, 1, 0, 23, 1, 0, 0, 0, 0, 26, 0, 2, 0, 0, 0, 0, 4093, 4096, 3201, 0, 0, 0, 0, 1016, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 3164, 0, 0, 0, 4096, 0, 34, 0, 0, 0, 3797, 0, 0, 3775, 0, 6, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 2668, 0, 0, 0, 294, 0, 1, 3875, 127, 0, 6, 0, 2, 0, 0, 0, 0, 1, 9, 0, 0, 7, 0, 4095, 4082, 3377, 0, 0, 0, 0, 155, 1, 0, 1, 0, 0, 1, 0, 2, 0, 0, 1, 1, 2667, 0, 0, 1, 4094, 0, 92, 0, 0, 0, 3878, 1, 0, 3325, 0, 0, 1, 1, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 3963, 8, 0, 0, 0, 20, 0, 3947, 576, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 4079, 4096, 302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 206, 0, 0, 0, 0, 3971, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3889, 0, 0, 3600, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 119, 4, 0, 0, 60, 0, 14, 2231, 0, 0, 0, 0, 0, 0, 0, 4068, 233, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 42, 0, 0, 10, 3988, 0, 0, 0, 0, 0, 0, 0, 489, 0, 4068, 0, 0, 5, 48, 179, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4, 3877, 134, 12, 0, 0, 0, 0, 0, 0, 23, 23, 18, 3973, 0, 0, 4, 657, 0, 0, 0, 0, 0, 0, 0, 195, 0, 0, 0, 1, 0, 4051, 4, 0, 0, 0, 0, 0, 0, 0, 0, 19, 3321, 5, 0, 172, 9, 106, 0, 566, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2428, 4304, 4067, 4789, 4033, 4698, 4251, 4198] avg=4096.0000 max=4789 max_rank=3 min=2428 min_rank=0 imbalance_ratio=1.169189 max_load_reduction=27979 max_load_reduction_pct=0.853851 moved_to_echo=17180 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1831, 0, 0, 0, 56, 0, 0, 2238, 0, 0, 0, 0, 0, 0, 0, 3948, 160, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 3, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 491, 0, 3553, 0, 0, 0, 0, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 25, 3900, 134, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 659, 0, 0, 0, 0, 0, 0, 0, 196, 0, 26, 0, 0, 0, 3252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3395, 0, 0, 0, 0, 0, 0, 568, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4109, 4102, 4177, 4074, 4744, 3474, 3963] avg=4096.0000 max=4744 max_rank=5 min=3474 min_rank=6 imbalance_ratio=1.158203 max_load_reduction=28024 max_load_reduction_pct=0.855225 moved_to_echo=16255 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3007, 20, 0, 0, 480, 12, 0, 2194, 0, 0, 0, 0, 0, 0, 0, 3687, 42, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 481, 0, 3629, 0, 0, 0, 0, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 102, 3900, 132, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2086, 0, 0, 0, 646, 0, 0, 0, 0, 0, 0, 0, 196, 0, 0, 0, 0, 0, 3949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2998, 0, 12, 0, 0, 0, 0, 557, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5713, 3732, 4092, 4577, 4210, 2732, 4145, 3567] avg=4096.0000 max=5713 max_rank=0 min=2732 min_rank=5 imbalance_ratio=1.394775 max_load_reduction=27055 max_load_reduction_pct=0.825653 moved_to_echo=14654 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1319, 1, 0, 0, 188, 0, 0, 2239, 0, 0, 0, 0, 0, 0, 0, 3923, 1, 0, 23, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 2, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 492, 0, 3201, 0, 0, 0, 0, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3900, 134, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3164, 0, 0, 0, 659, 0, 0, 0, 0, 0, 0, 0, 196, 0, 34, 0, 0, 0, 3797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3775, 0, 6, 0, 0, 0, 0, 568, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3747, 3949, 4111, 4709, 4053, 3823, 4027, 4349] avg=4096.0000 max=4709 max_rank=3 min=3747 min_rank=0 imbalance_ratio=1.149658 max_load_reduction=28059 max_load_reduction_pct=0.856293 moved_to_echo=16252 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 290, 0, 0, 0, 198, 0, 0, 2223, 0, 0, 0, 0, 0, 0, 0, 4008, 21, 0, 5, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 488, 0, 4082, 0, 0, 0, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3897, 133, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3720, 0, 0, 0, 654, 0, 0, 0, 0, 0, 0, 0, 196, 0, 71, 0, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3937, 0, 5, 0, 0, 0, 0, 564, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2711, 4035, 4129, 4645, 4043, 4374, 4325, 4506] avg=4096.0000 max=4645 max_rank=3 min=2711 min_rank=0 imbalance_ratio=1.134033 max_load_reduction=28123 max_load_reduction_pct=0.858246 moved_to_echo=17248 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3963, 8, 0, 0, 0, 20, 0, 2232, 0, 0, 0, 0, 0, 0, 0, 3947, 576, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 489, 0, 302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 206, 3900, 134, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3971, 0, 0, 0, 657, 0, 0, 0, 0, 0, 0, 0, 196, 0, 0, 0, 0, 0, 3889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3600, 0, 0, 0, 0, 0, 0, 566, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6223, 4526, 4097, 791, 4252, 4628, 4085, 4166] avg=4096.0000 max=6223 max_rank=0 min=791 min_rank=3 imbalance_ratio=1.519287 max_load_reduction=26545 max_load_reduction_pct=0.810089 moved_to_echo=17131 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 2668, 0, 0, 0, 294, 0, 1, 2240, 0, 0, 0, 0, 0, 0, 0, 3875, 127, 0, 6, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 9, 0, 0, 7, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 492, 0, 3377, 0, 0, 0, 0, 155, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 2, 3898, 134, 13, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2667, 0, 0, 1, 660, 0, 0, 0, 0, 0, 0, 0, 196, 0, 92, 0, 0, 0, 3878, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3325, 0, 0, 1, 1, 0, 2, 568, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5203, 4011, 4086, 4024, 4050, 3330, 4167, 3897] avg=4096.0000 max=5203 max_rank=0 min=3330 min_rank=5 imbalance_ratio=1.270264 max_load_reduction=27565 max_load_reduction_pct=0.841217 moved_to_echo=15444 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 356, 0, 0, 0, 17, 0, 0, 2245, 0, 0, 0, 0, 0, 0, 0, 4072, 23, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 468, 0, 4096, 0, 0, 0, 0, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3901, 139, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3775, 0, 0, 0, 664, 0, 0, 0, 0, 0, 0, 0, 195, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3549, 0, 0, 0, 0, 0, 0, 573, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2618, 4102, 4084, 5056, 4053, 4439, 4294, 4122] avg=4096.0000 max=5056 max_rank=3 min=2618 min_rank=0 imbalance_ratio=1.234375 max_load_reduction=27712 max_load_reduction_pct=0.845703 moved_to_echo=16908 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 13568, 128, 0, 0, 1408, 128, 128, 17920, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3968, 0, 26368, 0, 0, 128, 128, 2560, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 27904, 128, 128, 256, 128, 128, 128, 4608, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 27520, 0, 0, 128, 5376, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1664, 0, 256, 0, 128, 0, 30976, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[31616, 1280, 128, 128, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 0, 128, 0, 384, 31232, 1152, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 128, 128, 128, 128, 32640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[4, 0, 0, 0, 4070, 0, 4096, 4081, 0, 0, 4096, 0, 6, 0, 7, 0, 0, 229, 0, 0, 4078, 273, 0, 0, 0, 0, 0, 0, 2, 0, 1200, 0, 3045, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[16, 0, 0, 0, 3864, 0, 4077, 1860, 0, 0, 4096, 0, 246, 0, 6, 0, 0, 2540, 0, 2, 3994, 8, 0, 0, 282, 0, 0, 0, 7, 0, 7, 0, 3988, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 604, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 34, 3954, 0, 4034, 3605, 0, 0, 4045, 11, 285, 16, 0, 0, 0, 528, 10, 0, 4082, 1306, 0, 3, 174, 0, 571, 0, 189, 14, 1257, 11, 90, 3643, 5, 0, 5, 0, 0, 106, 0, 0, 0, 4079, 0, 0, 22, 319, 0, 12, 25, 0, 19, 0, 7, 302, 0, 3, 0, 2, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 4095, 3667, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 157, 0, 0, 4082, 3124, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 1961, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4089, 0, 4088, 3909, 0, 1, 4093, 0, 0, 1, 0, 0, 4, 40, 1, 0, 4094, 58, 0, 0, 23, 0, 1, 0, 0, 1, 3165, 0, 845, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 0, 1, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 159, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 4092, 0, 4088, 3929, 0, 0, 4096, 0, 2, 0, 2, 0, 0, 90, 0, 0, 4082, 1158, 0, 0, 14, 0, 11, 0, 0, 0, 743, 0, 2314, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 15, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 4096, 3156, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 209, 0, 0, 4096, 1959, 0, 0, 0, 0, 9, 0, 0, 0, 1109, 0, 854, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 899, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 4096, 3758, 0, 0, 4096, 0, 325, 0, 6, 0, 0, 160, 0, 0, 4096, 1363, 0, 0, 0, 0, 1, 0, 0, 0, 1290, 0, 1601, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 34, 581, 0, 0, 3605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3939, 11, 285, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 10, 0, 2449, 1306, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 174, 0, 571, 0, 189, 14, 1257, 11, 1422, 1273, 0, 0, 0, 0, 0, 0, 90, 2026, 5, 0, 5, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 22, 319, 211, 106, 0, 0, 0, 0, 0, 0, 0, 12, 25, 0, 19, 0, 7, 302, 3998, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 0, 0, 0, 3373, 344, 36, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4220, 4251, 4296, 4911, 2232, 4737, 4363, 3758] avg=4096.0000 max=4911 max_rank=3 min=2232 min_rank=4 imbalance_ratio=1.198975 max_load_reduction=27857 max_load_reduction_pct=0.850128 moved_to_echo=15090 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 0, 0, 598, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3989, 0, 6, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 229, 0, 0, 2446, 273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1200, 0, 1421, 1431, 0, 0, 0, 0, 0, 0, 3045, 2277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3462, 0, 0, 0, 0, 211, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 3472, 387, 36, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4683, 4002, 2948, 4054, 5322, 3780, 4060, 3919] avg=4096.0000 max=5322 max_rank=4 min=2948 min_rank=2 imbalance_ratio=1.299316 max_load_reduction=27446 max_load_reduction_pct=0.837585 moved_to_echo=17081 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[16, 0, 0, 0, 567, 0, 0, 1860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3989, 0, 246, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2540, 0, 2, 2395, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 282, 0, 0, 0, 7, 0, 7, 0, 1392, 1431, 0, 0, 0, 0, 0, 0, 3988, 2278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3071, 0, 0, 0, 0, 207, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 604, 0, 3297, 387, 36, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2443, 4241, 4945, 3119, 6266, 3385, 4045, 4324] avg=4096.0000 max=6266 max_rank=4 min=2443 min_rank=0 imbalance_ratio=1.529785 max_load_reduction=26502 max_load_reduction_pct=0.808777 moved_to_echo=18020 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 601, 0, 0, 3667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3989, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 0, 0, 2449, 3124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 1422, 1431, 0, 0, 0, 0, 0, 0, 1961, 2278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3343, 0, 0, 0, 0, 211, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 3495, 387, 36, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4268, 3990, 5730, 2889, 4239, 3661, 4059, 3932] avg=4096.0000 max=5730 max_rank=2 min=2889 min_rank=3 imbalance_ratio=1.398926 max_load_reduction=27038 max_load_reduction_pct=0.825134 moved_to_echo=15891 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 601, 0, 0, 3929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3989, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 2449, 1158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 11, 0, 0, 0, 743, 0, 1422, 1424, 0, 0, 0, 0, 0, 0, 2314, 2267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 0, 0, 0, 211, 107, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 11, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 3491, 385, 36, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4532, 3993, 3697, 3614, 4581, 4360, 4064, 3927] avg=4096.0000 max=4581 max_rank=4 min=3614 min_rank=3 imbalance_ratio=1.118408 max_load_reduction=28187 max_load_reduction_pct=0.860199 moved_to_echo=16932 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 600, 0, 0, 3909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3986, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 40, 1, 0, 2455, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 1, 0, 0, 1, 3165, 0, 1427, 1431, 0, 0, 0, 0, 0, 0, 845, 2277, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 212, 107, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 2, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 159, 0, 3489, 387, 36, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4509, 3988, 2558, 6048, 3123, 4415, 4056, 4071] avg=4096.0000 max=6048 max_rank=3 min=2558 min_rank=2 imbalance_ratio=1.476562 max_load_reduction=26720 max_load_reduction_pct=0.815430 moved_to_echo=15665 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 601, 0, 0, 3758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3989, 0, 325, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 0, 2457, 1363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1290, 0, 1427, 1431, 0, 0, 0, 0, 0, 0, 1601, 2278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3686, 0, 0, 0, 0, 212, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 3495, 387, 36, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4359, 4320, 3980, 4149, 3879, 4005, 4060, 4016] avg=4096.0000 max=4359 max_rank=0 min=3879 min_rank=4 imbalance_ratio=1.064209 max_load_reduction=28409 max_load_reduction_pct=0.866974 moved_to_echo=15960 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 598, 0, 0, 3156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3982, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 209, 0, 0, 2446, 1959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 1109, 0, 1432, 1434, 0, 0, 0, 0, 0, 0, 854, 2272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 1, 218, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 899, 0, 3497, 390, 35, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3754, 3983, 4614, 3984, 3126, 4425, 4061, 4821] avg=4096.0000 max=4821 max_rank=7 min=3126 min_rank=4 imbalance_ratio=1.177002 max_load_reduction=27947 max_load_reduction_pct=0.852875 moved_to_echo=16433 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[512, 0, 640, 0, 256, 128, 8832, 128, 11392, 11392, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[14720, 18048, 128, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 3968, 128, 128, 19584, 9344, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 0, 128, 4864, 0, 0, 28032, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 0, 128, 0, 0, 1920, 0, 27648, 3072, 384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 0, 29952, 0, 0, 128, 384, 1792, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 128, 0, 128, 384, 32384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 31872, 128, 896, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[13, 465, 0, 766, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 3856, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 3059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3773, 0, 276, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 3949, 0, 188, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 502, 11, 2806, 0, 9, 0, 0, 18, 29, 7, 0, 8, 0, 0, 15, 0, 4053, 0, 3619, 4078, 0, 319, 0, 13, 481, 0, 20, 0, 0, 0, 0, 5, 0, 0, 0, 0, 1, 0, 0, 28, 0, 0, 3607, 0, 3860, 31, 26, 4024, 0, 0, 0, 0, 0, 0, 5, 0, 4072, 0, 0, 942, 0, 171, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 2186, 1, 599, 0, 0, 0, 0, 0, 94, 0, 0, 1, 1, 1, 1, 0, 4086, 0, 4078, 4092, 0, 0, 0, 0, 2619, 0, 1, 0, 0, 0, 0, 0, 1851, 0, 0, 0, 0, 0, 0, 1, 0, 0, 599, 0, 1996, 1, 0, 4090, 4, 0, 0, 1, 0, 0, 1, 0, 4090, 0, 1, 910, 0, 1462, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[4, 4043, 0, 1667, 0, 0, 0, 0, 0, 525, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4087, 3714, 0, 0, 0, 0, 405, 0, 0, 0, 0, 0, 0, 0, 138, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4000, 0, 690, 0, 0, 4096, 0, 7, 0, 1, 2, 0, 0, 0, 4019, 0, 0, 1273, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[7, 1437, 0, 1285, 0, 0, 0, 0, 0, 156, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 3924, 0, 0, 0, 0, 73, 0, 0, 0, 17, 0, 0, 0, 795, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 3053, 0, 0, 4084, 0, 25, 0, 0, 0, 0, 0, 0, 4049, 0, 0, 1366, 0, 235, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[37, 921, 0, 596, 0, 0, 0, 0, 0, 117, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4037, 3959, 0, 0, 0, 0, 1208, 0, 0, 0, 0, 1, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3811, 0, 2566, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 4047, 9, 0, 3146, 0, 36, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 1904, 0, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4078, 4076, 0, 0, 0, 0, 192, 0, 0, 0, 0, 0, 0, 0, 1967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3875, 0, 1756, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4004, 0, 0, 2450, 0, 70, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[342, 2085, 0, 1, 0, 0, 0, 0, 0, 351, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3177, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2576, 0, 1159, 0, 0, 4096, 0, 11, 0, 0, 6, 0, 0, 0, 3989, 0, 0, 4082, 0, 2696, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 502, 11, 2806, 0, 9, 0, 0, 1229, 240, 0, 0, 0, 0, 0, 0, 18, 29, 7, 0, 8, 0, 0, 15, 3893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177, 4078, 0, 319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 481, 0, 20, 0, 0, 0, 0, 2972, 160, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 1, 0, 0, 2602, 470, 0, 0, 0, 0, 0, 0, 28, 0, 0, 2375, 0, 3860, 31, 26, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 0, 0, 0, 0, 0, 0, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1230, 0, 0, 942, 0, 171, 8, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4797, 3970, 4574, 3646, 3078, 6320, 4032, 2351] avg=4096.0000 max=6320 max_rank=5 min=2351 min_rank=7 imbalance_ratio=1.542969 max_load_reduction=26448 max_load_reduction_pct=0.807129 moved_to_echo=15781 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[13, 465, 0, 766, 0, 0, 0, 0, 1285, 241, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 3935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 201, 3856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 3362, 161, 0, 0, 0, 0, 0, 0, 0, 3059, 0, 0, 0, 0, 0, 0, 2617, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2485, 0, 276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1238, 0, 0, 3949, 0, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2770, 3954, 4057, 3543, 6208, 2761, 4100, 5375] avg=4096.0000 max=6208 max_rank=4 min=2761 min_rank=5 imbalance_ratio=1.515625 max_load_reduction=26560 max_load_reduction_pct=0.810547 moved_to_echo=18444 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4, 4043, 0, 1667, 0, 0, 0, 0, 1363, 237, 0, 0, 0, 0, 0, 0, 0, 525, 0, 0, 0, 0, 0, 0, 3934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 3714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 405, 0, 0, 0, 0, 0, 0, 3356, 162, 0, 0, 0, 0, 0, 0, 0, 138, 0, 0, 0, 1, 0, 0, 2568, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2634, 0, 690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 0, 1, 2, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1214, 0, 0, 1273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7314, 4459, 3914, 3923, 3238, 3324, 4109, 2487] avg=4096.0000 max=7314 max_rank=0 min=2487 min_rank=7 imbalance_ratio=1.785645 max_load_reduction=25454 max_load_reduction_pct=0.776794 moved_to_echo=13158 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 2186, 1, 599, 0, 0, 0, 0, 204, 241, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 1, 1, 1, 1, 3924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 201, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2619, 0, 1, 0, 0, 0, 0, 3348, 162, 0, 0, 0, 0, 0, 0, 0, 1851, 0, 0, 0, 0, 0, 0, 2613, 529, 0, 0, 0, 0, 0, 0, 1, 0, 0, 395, 0, 1996, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1236, 0, 1, 910, 0, 1462, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3231, 4022, 4293, 6130, 4993, 2393, 4096, 3610] avg=4096.0000 max=6130 max_rank=3 min=2393 min_rank=5 imbalance_ratio=1.496582 max_load_reduction=26638 max_load_reduction_pct=0.812927 moved_to_echo=15092 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[7, 1437, 0, 1285, 0, 0, 0, 0, 1388, 238, 0, 0, 0, 0, 0, 0, 0, 156, 0, 0, 0, 0, 0, 0, 3935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 201, 3924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 17, 0, 0, 3359, 161, 0, 0, 0, 0, 0, 0, 0, 795, 0, 0, 0, 0, 0, 0, 2587, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2684, 0, 3053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 25, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1224, 0, 0, 1366, 0, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4355, 4091, 4125, 3610, 3913, 5737, 4112, 2825] avg=4096.0000 max=5737 max_rank=5 min=2825 min_rank=7 imbalance_ratio=1.400635 max_load_reduction=27031 max_load_reduction_pct=0.824921 moved_to_echo=16587 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[342, 2085, 0, 1, 0, 0, 0, 0, 877, 235, 0, 0, 0, 0, 0, 0, 0, 351, 0, 0, 0, 0, 0, 0, 3935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 201, 3177, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3363, 161, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 2549, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1697, 0, 1159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 11, 0, 0, 6, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1205, 0, 0, 4082, 0, 2696, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3540, 4286, 3380, 3524, 3084, 2856, 4115, 7983] avg=4096.0000 max=7983 max_rank=7 min=2856 min_rank=5 imbalance_ratio=1.948975 max_load_reduction=24785 max_load_reduction_pct=0.756378 moved_to_echo=18038 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[37, 921, 0, 596, 0, 0, 0, 0, 1298, 238, 0, 0, 0, 0, 0, 0, 0, 117, 0, 0, 0, 0, 0, 0, 3934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 198, 3959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1208, 0, 0, 0, 0, 1, 0, 3315, 162, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 2586, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2510, 0, 2566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1223, 9, 0, 3146, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3090, 4051, 4157, 4686, 3192, 5076, 4102, 4414] avg=4096.0000 max=5076 max_rank=5 min=3090 min_rank=0 imbalance_ratio=1.239258 max_load_reduction=27692 max_load_reduction_pct=0.845093 moved_to_echo=16784 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1904, 0, 204, 0, 0, 0, 0, 1324, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 0, 0, 0, 0, 0, 3353, 161, 0, 0, 0, 0, 0, 0, 0, 1967, 0, 0, 0, 0, 0, 0, 2562, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2545, 0, 1756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 1203, 0, 0, 2450, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3671, 3935, 4268, 3706, 5062, 4301, 4102, 3723] avg=4096.0000 max=5062 max_rank=4 min=3671 min_rank=0 imbalance_ratio=1.235840 max_load_reduction=27706 max_load_reduction_pct=0.845520 moved_to_echo=17188 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[512, 13568, 128, 7936, 0, 128, 0, 0, 9088, 1920, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 9856, 128, 128, 18176, 0, 4864, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 0, 17408, 0, 15360, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 7936, 0, 0, 0, 128, 0, 0, 20736, 4224, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 5120, 0, 128, 0, 128, 128, 0, 26496, 1408, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 1664, 30976, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 1408, 128, 0, 128, 128, 128, 128, 31488, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 0, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 978, 0, 4096, 0, 0, 4088, 4096, 0, 0, 0, 0, 0, 3848, 0, 0, 0, 0, 0, 0, 0, 13, 0, 450, 0, 0, 32, 0, 2749, 4096, 530, 0, 0, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2637, 0, 4086, 0, 0, 3979, 4095, 7, 0, 0, 0, 0, 2678, 0, 0, 0, 1, 0, 0, 46, 13, 0, 2218, 0, 0, 30, 0, 756, 4096, 31, 0, 0, 0, 0, 3999, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[14, 6, 0, 362, 0, 28, 4079, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 10, 3, 1612, 0, 4068, 133, 7, 4012, 3650, 0, 0, 21, 19, 56, 2669, 0, 18, 0, 609, 21, 17, 0, 0, 0, 2663, 0, 0, 21, 0, 1273, 4070, 35, 0, 0, 0, 0, 3268, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 4050, 19, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 886, 0, 4095, 0, 1, 4052, 4094, 0, 0, 1, 1, 1, 2808, 0, 0, 1, 36, 0, 1, 1, 5, 0, 2961, 0, 0, 101, 0, 418, 4090, 1196, 1, 0, 0, 0, 3937, 0, 0, 0, 0, 0, 0, 0, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 0, 4096, 0, 0, 3833, 3759, 4, 0, 0, 0, 0, 3487, 0, 0, 0, 0, 0, 0, 3, 67, 0, 4093, 0, 0, 0, 0, 319, 4094, 708, 0, 0, 0, 0, 3747, 0, 0, 0, 0, 0, 0, 0, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 67, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 4096, 0, 0, 4095, 4094, 0, 0, 0, 0, 0, 2442, 0, 0, 0, 46, 0, 0, 0, 0, 0, 3456, 0, 0, 0, 0, 1275, 4096, 898, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 997, 0, 4096, 0, 0, 4040, 3843, 3, 1, 0, 0, 0, 4023, 0, 0, 0, 13, 0, 0, 0, 0, 0, 1963, 0, 0, 13, 0, 1403, 4096, 120, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3778, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2103, 0, 4096, 0, 0, 3618, 3964, 1, 0, 0, 0, 0, 4017, 0, 0, 0, 0, 0, 0, 171, 42, 0, 1840, 0, 0, 141, 0, 3, 4096, 1833, 0, 0, 0, 0, 2894, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3739, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3959, 272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 978, 0, 2878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3848, 0, 0, 0, 0, 0, 0, 747, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 450, 0, 0, 32, 0, 1190, 449, 0, 0, 0, 0, 0, 0, 2749, 0, 530, 0, 0, 0, 0, 2591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3761, 4231, 3856, 4223, 4595, 2134, 5870, 4098] avg=4096.0000 max=5870 max_rank=6 min=2134 min_rank=5 imbalance_ratio=1.433105 max_load_reduction=26898 max_load_reduction_pct=0.820862 moved_to_echo=16697 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[14, 6, 0, 362, 0, 28, 4079, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 3886, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3, 1612, 0, 2858, 0, 0, 0, 0, 0, 0, 0, 0, 133, 7, 124, 3650, 0, 0, 21, 19, 0, 0, 0, 0, 0, 0, 0, 0, 56, 2669, 0, 18, 0, 609, 21, 17, 742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2663, 0, 0, 21, 0, 959, 446, 0, 0, 0, 0, 0, 0, 1273, 0, 35, 0, 0, 0, 0, 2090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4511, 4129, 4483, 3954, 4132, 4089, 3398, 4072] avg=4096.0000 max=4511 max_rank=0 min=3398 min_rank=6 imbalance_ratio=1.101318 max_load_reduction=28257 max_load_reduction_pct=0.862335 moved_to_echo=15691 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3854, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2637, 0, 2870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 4095, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2678, 0, 0, 0, 1, 0, 0, 746, 0, 0, 0, 0, 0, 0, 0, 46, 13, 0, 2218, 0, 0, 30, 0, 1174, 448, 0, 0, 0, 0, 0, 0, 756, 0, 31, 0, 0, 0, 0, 2557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4122, 5507, 4225, 3425, 3929, 3344, 4098] avg=4096.0000 max=5507 max_rank=2 min=3344 min_rank=6 imbalance_ratio=1.344482 max_load_reduction=27261 max_load_reduction_pct=0.831940 moved_to_echo=14796 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4042, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3712, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 0, 2878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 3759, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3487, 0, 0, 0, 0, 0, 0, 747, 0, 0, 0, 0, 0, 0, 0, 3, 67, 0, 4093, 0, 0, 0, 0, 1100, 449, 0, 0, 0, 0, 0, 0, 319, 0, 708, 0, 0, 0, 0, 2396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4094, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4064, 3963, 3389, 3882, 4234, 5712, 3423, 4101] avg=4096.0000 max=5712 max_rank=5 min=3389 min_rank=2 imbalance_ratio=1.394531 max_load_reduction=27056 max_load_reduction_pct=0.825684 moved_to_echo=17470 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 4050, 19, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 3924, 264, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 886, 0, 2877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 126, 4094, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2808, 0, 0, 1, 36, 0, 1, 747, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 2961, 0, 0, 101, 0, 1156, 449, 0, 0, 0, 0, 0, 0, 418, 0, 1196, 1, 0, 0, 0, 2517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4090, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4190, 3765, 4223, 3594, 4673, 4132, 4099] avg=4096.0000 max=4673 max_rank=5 min=3594 min_rank=4 imbalance_ratio=1.140869 max_load_reduction=28095 max_load_reduction_pct=0.857391 moved_to_echo=16498 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4066, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3913, 274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 997, 0, 2878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 3843, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 0, 0, 0, 13, 0, 0, 747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1963, 0, 0, 13, 0, 1201, 449, 0, 0, 0, 0, 0, 0, 1403, 0, 120, 0, 0, 0, 0, 2616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4187, 3875, 3972, 4783, 3626, 4139, 4098] avg=4096.0000 max=4783 max_rank=4 min=3626 min_rank=5 imbalance_ratio=1.167725 max_load_reduction=27985 max_load_reduction_pct=0.854034 moved_to_echo=16646 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3778, 170, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3504, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2103, 0, 2878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 3964, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4017, 0, 0, 0, 0, 0, 0, 747, 0, 0, 0, 0, 0, 0, 0, 171, 42, 0, 1840, 0, 0, 141, 0, 849, 449, 0, 0, 0, 0, 0, 0, 3, 0, 1833, 0, 0, 0, 0, 1851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3970, 3698, 4981, 4077, 4764, 3492, 3687, 4099] avg=4096.0000 max=4981 max_rank=2 min=3492 min_rank=5 imbalance_ratio=1.216064 max_load_reduction=27787 max_load_reduction_pct=0.847992 moved_to_echo=16042 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 67, 0, 0, 4068, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3970, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 2860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2442, 0, 0, 0, 46, 0, 0, 753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3456, 0, 0, 0, 0, 1203, 454, 0, 0, 0, 0, 0, 0, 1275, 0, 898, 0, 0, 0, 0, 2602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4164, 4248, 2912, 4212, 3241, 5113, 4775, 4103] avg=4096.0000 max=5113 max_rank=5 min=2912 min_rank=2 imbalance_ratio=1.248291 max_load_reduction=27655 max_load_reduction_pct=0.843964 moved_to_echo=17232 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 256, 0, 19712, 0, 0, 384, 0, 8832, 3712, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 0, 128, 128, 9856, 0, 23040, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 25984, 0, 128, 128, 768, 128, 128, 6016, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 0, 5376, 128, 0, 0, 0, 19328, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[256, 128, 1024, 31616, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 0, 30848, 2048, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 0, 512, 0, 128, 32000, 256, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 595, 0, 0, 27, 3526, 0, 10, 3374, 0, 119, 0, 0, 0, 4071, 0, 2, 125, 16, 0, 0, 0, 0, 0, 0, 0, 27, 2, 4077, 4072, 0, 0, 7, 3230, 1099, 0, 18, 0, 0, 7, 0, 0, 4071, 172, 22, 0, 37, 0, 0, 0, 0, 3991, 3, 25, 0, 0, 16, 0, 0, 23, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 55, 0, 0, 4096, 41, 0, 3856, 0, 0, 0, 0, 0, 4057, 0, 1, 0, 0, 0, 0, 4, 0, 12, 0, 0, 0, 0, 3968, 4077, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 313, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 69, 0, 0, 4081, 0, 1, 4077, 0, 0, 0, 0, 0, 4095, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3992, 4091, 0, 0, 1, 4015, 5, 0, 1, 1, 0, 1, 140, 0, 4094, 14, 0, 0, 0, 1, 0, 0, 0, 4056, 0, 1, 0, 1, 0, 1, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 17, 0, 0, 4096, 7, 0, 4089, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3690, 4087, 0, 0, 0, 4080, 0, 0, 0, 8, 0, 0, 411, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 26, 0, 0, 4096, 28, 0, 4013, 0, 18, 0, 0, 0, 4077, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 60, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 9, 0, 0, 4096, 8, 0, 3965, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3843, 4093, 0, 0, 0, 3835, 1, 0, 0, 0, 0, 0, 602, 0, 4096, 29, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4, 0, 0, 4096, 30, 0, 4041, 0, 0, 0, 0, 0, 4063, 0, 51, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 403, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3636, 0, 4093, 57, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3114, 4096, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 894, 0, 4096, 99, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 595, 0, 0, 27, 3406, 109, 0, 0, 0, 0, 0, 0, 110, 0, 10, 3374, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 2, 125, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 2, 4077, 692, 2, 0, 0, 0, 0, 0, 0, 14, 0, 0, 7, 3230, 1099, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 3270, 172, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 3991, 3, 16, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 16, 0, 0, 23, 4, 4058, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4137, 3613, 4196, 4800, 4368, 3471, 4047, 4136] avg=4096.0000 max=4800 max_rank=3 min=3471 min_rank=5 imbalance_ratio=1.171875 max_load_reduction=27968 max_load_reduction_pct=0.853516 moved_to_echo=16022 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 17, 0, 0, 3956, 109, 0, 0, 0, 0, 0, 0, 128, 7, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3690, 695, 2, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 411, 0, 3289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4224, 4076, 4387, 4094, 3708, 4112, 4085] avg=4096.0000 max=4387 max_rank=3 min=3708 min_rank=5 imbalance_ratio=1.071045 max_load_reduction=28381 max_load_reduction_pct=0.866119 moved_to_echo=15999 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 26, 0, 0, 3956, 109, 0, 0, 0, 0, 0, 0, 128, 28, 0, 4013, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 696, 2, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 3291, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4187, 4092, 4745, 4110, 3360, 4089, 4094] avg=4096.0000 max=4745 max_rank=3 min=3360 min_rank=5 imbalance_ratio=1.158447 max_load_reduction=28023 max_load_reduction_pct=0.855194 moved_to_echo=15653 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 55, 0, 0, 3956, 109, 0, 0, 0, 0, 0, 0, 128, 41, 0, 3856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 12, 0, 0, 0, 0, 3968, 696, 2, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 313, 0, 3291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4025, 4040, 4682, 4110, 3604, 4112, 4075] avg=4096.0000 max=4682 max_rank=3 min=3604 min_rank=5 imbalance_ratio=1.143066 max_load_reduction=28086 max_load_reduction_pct=0.857117 moved_to_echo=15901 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 69, 0, 0, 3942, 109, 0, 0, 0, 0, 0, 0, 127, 0, 1, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3992, 695, 2, 0, 0, 0, 0, 0, 0, 14, 0, 0, 1, 4015, 5, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 140, 0, 3290, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4056, 0, 16, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 4077, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4205, 4106, 4690, 4036, 3446, 4073, 4092] avg=4096.0000 max=4690 max_rank=3 min=3446 min_rank=5 imbalance_ratio=1.145020 max_load_reduction=28078 max_load_reduction_pct=0.856873 moved_to_echo=15647 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 9, 0, 0, 3956, 109, 0, 0, 0, 0, 0, 0, 128, 8, 0, 3965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3843, 696, 2, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 3835, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 602, 0, 3291, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4101, 4079, 4544, 3850, 3922, 4107, 4091] avg=4096.0000 max=4544 max_rank=3 min=3850 min_rank=4 imbalance_ratio=1.109375 max_load_reduction=28224 max_load_reduction_pct=0.861328 moved_to_echo=15970 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4, 0, 0, 3956, 109, 0, 0, 0, 0, 0, 0, 128, 30, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 51, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 403, 695, 2, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3636, 0, 3289, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4199, 4101, 1100, 4110, 6982, 4113, 4094] avg=4096.0000 max=6982 max_rank=5 min=1100 min_rank=3 imbalance_ratio=1.704590 max_load_reduction=25786 max_load_reduction_pct=0.786926 moved_to_echo=19299 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 3960, 114, 0, 0, 0, 0, 0, 0, 121, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3114, 700, 3, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 894, 0, 3282, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 15, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4214, 4078, 3820, 4090, 4275, 4115, 4101] avg=4096.0000 max=4275 max_rank=5 min=3820 min_rank=3 imbalance_ratio=1.043701 max_load_reduction=28493 max_load_reduction_pct=0.869537 moved_to_echo=16581 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 0, 128, 128, 27136, 5632, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 6016, 0, 26368, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32512, 0, 128, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 640, 256, 0, 128, 31104, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 0, 128, 31616, 1152, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1024, 128, 128, 31616, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 0, 0, 0, 32640, 128, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 128, 0, 128, 128, 32640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 3305, 4072, 0, 0, 0, 0, 0, 4058, 0, 12, 29, 0, 13, 4009, 0, 0, 0, 0, 0, 30, 9, 3246, 0, 0, 23, 19, 0, 0, 1, 0, 0, 0, 1, 0, 4052, 31, 4, 38, 0, 0, 0, 4065, 1411, 22, 0, 0, 21, 1, 1, 147, 0, 21, 1, 32, 0, 0, 51, 0, 0, 0, 0, 4043, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 4068, 3976, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 21, 0, 0, 4090, 2418, 0, 0, 0, 0, 0, 0, 86, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 4093, 4010, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 2396, 0, 26, 0, 0, 0, 0, 61, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 4014, 4076, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 4096, 1019, 0, 9, 0, 0, 0, 0, 3, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 3778, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 3906, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 434, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[2, 4054, 4088, 0, 0, 0, 1, 1, 4093, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 1008, 0, 2, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 4094, 1, 0, 1, 0, 0, 0, 4095, 2983, 1, 13, 0, 0, 0, 0, 135, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 4095, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 3115, 3402, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 6, 0, 0, 4057, 4079, 0, 37, 0, 0, 0, 0, 1146, 274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 277, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 4090, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1952, 0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 709, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3305, 249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 12, 29, 0, 13, 4009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 9, 3246, 0, 2362, 198, 7, 0, 0, 0, 0, 0, 0, 23, 19, 0, 0, 1, 0, 0, 4053, 5, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4045, 31, 4, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1552, 1411, 22, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 147, 0, 21, 1, 32, 0, 3818, 151, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 3845, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3554, 4068, 5852, 4101, 4119, 3006, 4172, 3896] avg=4096.0000 max=5852 max_rank=2 min=3006 min_rank=5 imbalance_ratio=1.428711 max_load_reduction=26916 max_load_reduction_pct=0.821411 moved_to_echo=15193 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4093, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1507, 0, 2380, 200, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1564, 2396, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 14, 0, 0, 0, 0, 3760, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 3896, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4338, 4101, 4094, 4096, 4089, 3986, 3987, 4077] avg=4096.0000 max=4338 max_rank=0 min=3986 min_rank=5 imbalance_ratio=1.059082 max_load_reduction=28430 max_load_reduction_pct=0.867615 moved_to_echo=16139 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4068, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1677, 0, 2376, 200, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1562, 2418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 48, 0, 0, 0, 0, 3728, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3896, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4311, 4101, 4260, 4096, 4110, 3980, 4014, 3896] avg=4096.0000 max=4311 max_rank=0 min=3896 min_rank=7 imbalance_ratio=1.052490 max_load_reduction=28457 max_load_reduction_pct=0.868439 moved_to_echo=16000 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 231, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 2380, 200, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1564, 3906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 3542, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 434, 3896, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4329, 4101, 2636, 4096, 4087, 5470, 3719, 4330] avg=4096.0000 max=5470 max_rank=5 min=2636 min_rank=2 imbalance_ratio=1.335449 max_load_reduction=27298 max_load_reduction_pct=0.833069 moved_to_echo=17606 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4014, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3140, 0, 2380, 200, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1564, 1019, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 23, 0, 0, 0, 0, 3821, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3896, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4264, 4101, 5727, 4096, 4091, 2592, 3999, 3898] avg=4096.0000 max=5727 max_rank=2 min=2592 min_rank=5 imbalance_ratio=1.398193 max_load_reduction=27041 max_load_reduction_pct=0.825226 moved_to_echo=14580 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2, 4054, 250, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 0, 2379, 200, 7, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 1, 0, 0, 4088, 5, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4087, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1564, 2983, 1, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 0, 0, 0, 1, 0, 3833, 152, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3895, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4308, 4099, 3594, 4097, 4090, 4561, 4121, 3898] avg=4096.0000 max=4561 max_rank=5 min=3594 min_rank=2 imbalance_ratio=1.113525 max_load_reduction=28207 max_load_reduction_pct=0.860809 moved_to_echo=16670 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3115, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 2357, 200, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1549, 4079, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1146, 274, 0, 0, 0, 0, 3189, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 277, 3896, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3324, 4101, 2568, 4082, 4095, 5665, 4760, 4173] avg=4096.0000 max=5665 max_rank=5 min=2568 min_rank=2 imbalance_ratio=1.383057 max_load_reduction=27103 max_load_reduction_pct=0.827118 moved_to_echo=18693 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1438, 0, 2385, 205, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1555, 1952, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 3838, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 709, 3891, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4340, 4096, 4037, 4104, 4087, 3508, 3996, 4600] avg=4096.0000 max=4600 max_rank=7 min=3508 min_rank=5 imbalance_ratio=1.123047 max_load_reduction=28168 max_load_reduction_pct=0.859619 moved_to_echo=16191 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 128, 128, 12160, 0, 19072, 1664, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 12544, 20224, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 1664, 384, 128, 128, 128, 0, 29568, 1280, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 0, 0, 1664, 31232, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 30848, 2048, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 0, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 0, 128, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True + [2026-09-21 08:25:38] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 265.7 | throughput per GPU (TFLOP/s/GPU): 121.3 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.657319E+00 | loss scale: 1.0 | grad norm: 3.587 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[284, 447, 1129, 3, 8, 972, 756, 625, 1, 734, 21, 0, 11, 15, 643, 254, 6, 722, 786, 1432, 51, 223, 199, 11, 141, 16, 129, 184, 1542, 553, 1142, 942, 540, 550, 640, 935, 233, 875, 1022, 565, 1506, 347, 1652, 33, 814, 985, 47, 45, 802, 789, 804, 35, 1096, 940, 1181, 89, 1118, 50, 298, 3, 20, 50, 668, 54] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[94, 204, 1044, 0, 0, 599, 692, 402, 14, 384, 22, 3, 4, 38, 364, 135, 8, 1216, 965, 1924, 33, 240, 201, 12, 57, 4, 125, 424, 834, 404, 1236, 1775, 1086, 507, 378, 931, 114, 643, 676, 682, 1339, 521, 1469, 165, 812, 761, 39, 96, 963, 511, 721, 20, 916, 1720, 1474, 110, 1077, 13, 237, 7, 15, 29, 1234, 45] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[90, 301, 1455, 0, 0, 456, 678, 434, 0, 456, 119, 1, 8, 7, 741, 119, 1, 770, 1082, 1729, 25, 189, 226, 0, 74, 19, 56, 366, 877, 691, 1174, 1345, 973, 356, 450, 970, 206, 1030, 647, 856, 1813, 644, 1478, 95, 806, 378, 9, 45, 911, 937, 390, 35, 508, 1574, 1607, 28, 1188, 11, 362, 5, 6, 24, 902, 35] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[223, 135, 1630, 0, 0, 824, 631, 336, 0, 679, 67, 0, 14, 6, 485, 109, 15, 703, 913, 1560, 49, 96, 316, 0, 70, 30, 90, 233, 766, 865, 974, 1716, 1538, 487, 440, 1045, 155, 874, 211, 601, 2097, 328, 2251, 55, 619, 575, 5, 23, 897, 541, 629, 43, 635, 1275, 1831, 11, 1109, 7, 270, 1, 4, 31, 628, 17] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[141, 160, 1013, 1, 0, 649, 1127, 342, 1, 459, 16, 3, 21, 9, 278, 192, 2, 884, 757, 1478, 70, 134, 298, 13, 68, 42, 260, 235, 1004, 846, 1154, 1427, 782, 636, 517, 1276, 206, 1470, 521, 662, 1512, 419, 1605, 71, 730, 1089, 23, 33, 887, 674, 669, 43, 768, 1078, 1863, 59, 1173, 17, 320, 5, 31, 30, 485, 30] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[151, 236, 1328, 1, 2, 522, 698, 472, 1, 508, 32, 0, 30, 14, 495, 97, 9, 742, 713, 1457, 17, 197, 297, 0, 52, 11, 103, 417, 1021, 746, 1219, 1996, 896, 589, 706, 910, 193, 1065, 420, 782, 1154, 667, 1780, 203, 919, 639, 16, 100, 931, 547, 606, 59, 683, 1069, 1160, 22, 1392, 18, 334, 12, 9, 22, 1269, 12] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[75, 153, 1424, 4, 11, 660, 672, 633, 2, 548, 28, 5, 2, 8, 426, 120, 7, 919, 951, 1467, 26, 130, 285, 1, 83, 18, 223, 318, 756, 374, 994, 1602, 1215, 534, 620, 1147, 102, 859, 367, 549, 2081, 363, 1861, 58, 895, 1269, 9, 96, 1031, 545, 587, 72, 614, 1234, 1606, 25, 1072, 10, 267, 7, 24, 11, 676, 37] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[176, 283, 1042, 0, 2, 548, 940, 207, 2, 466, 25, 4, 14, 42, 441, 250, 18, 1162, 796, 1868, 22, 311, 223, 6, 67, 41, 98, 435, 1199, 699, 820, 1503, 774, 241, 422, 1494, 153, 835, 635, 743, 1268, 409, 1074, 173, 808, 860, 13, 25, 769, 863, 597, 84, 976, 1152, 1686, 84, 1503, 21, 266, 7, 31, 26, 1022, 44] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[90, 301, 1455, 0, 0, 456, 678, 434, 272, 256, 13, 0, 0, 0, 0, 0, 0, 456, 119, 1, 8, 7, 741, 119, 1607, 1163, 0, 0, 0, 0, 0, 0, 1, 770, 1082, 1729, 25, 189, 226, 0, 282, 0, 0, 0, 0, 0, 0, 0, 74, 19, 56, 366, 877, 691, 1174, 700, 0, 0, 0, 0, 0, 0, 0, 0, 717, 356, 450, 0, 206, 1030, 647, 856, 0, 0, 0, 0, 0, 0, 0, 0, 1813, 644, 210, 95, 806, 378, 9, 45, 0, 0, 0, 0, 0, 0, 0, 0, 911, 937, 390, 35, 508, 1279, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 1188, 11, 362, 5, 6, 24, 902, 35, 970, 373, 105, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3955, 4221, 4304, 3957, 4262, 4000, 4088, 3981] avg=4096.0000 max=4304 max_rank=2 min=3955 min_rank=0 imbalance_ratio=1.050781 max_load_reduction=28464 max_load_reduction_pct=0.868652 moved_to_echo=16331 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[284, 447, 1129, 3, 8, 972, 756, 625, 191, 142, 7, 0, 0, 0, 0, 0, 1, 734, 21, 0, 11, 15, 643, 254, 1181, 1300, 0, 0, 0, 0, 0, 0, 6, 722, 786, 1432, 51, 223, 199, 11, 168, 0, 0, 0, 0, 0, 0, 0, 141, 16, 129, 184, 1542, 553, 1142, 490, 0, 0, 0, 0, 0, 0, 0, 0, 398, 550, 640, 0, 233, 875, 1022, 565, 0, 0, 0, 0, 0, 0, 0, 0, 1506, 347, 235, 33, 814, 985, 47, 45, 0, 0, 0, 0, 0, 0, 0, 0, 802, 789, 804, 35, 1096, 765, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 1118, 50, 298, 3, 20, 50, 668, 54, 935, 261, 117, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4564, 4160, 3598, 4197, 4283, 4012, 4380, 3574] avg=4096.0000 max=4564 max_rank=0 min=3574 min_rank=7 imbalance_ratio=1.114258 max_load_reduction=28204 max_load_reduction_pct=0.860718 moved_to_echo=16249 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[94, 204, 1044, 0, 0, 599, 692, 402, 360, 286, 14, 0, 0, 0, 0, 0, 14, 384, 22, 3, 4, 38, 364, 135, 1474, 1156, 0, 0, 0, 0, 0, 0, 8, 1216, 965, 1924, 33, 240, 201, 12, 309, 0, 0, 0, 0, 0, 0, 0, 57, 4, 125, 424, 834, 404, 1236, 922, 0, 0, 0, 0, 0, 0, 0, 0, 800, 507, 378, 0, 114, 643, 676, 682, 0, 0, 0, 0, 0, 0, 0, 0, 1339, 521, 209, 165, 812, 761, 39, 96, 0, 0, 0, 0, 0, 0, 0, 0, 963, 511, 721, 20, 916, 1397, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 1077, 13, 237, 7, 15, 29, 1234, 45, 931, 493, 104, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3695, 3594, 4908, 4006, 3800, 3942, 4638, 4185] avg=4096.0000 max=4908 max_rank=2 min=3594 min_rank=1 imbalance_ratio=1.198242 max_load_reduction=27860 max_load_reduction_pct=0.850220 moved_to_echo=16565 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[151, 236, 1328, 1, 2, 522, 698, 472, 405, 236, 9, 0, 0, 0, 0, 0, 1, 508, 32, 0, 30, 14, 495, 97, 1160, 1401, 0, 0, 0, 0, 0, 0, 9, 742, 713, 1457, 17, 197, 297, 0, 192, 0, 0, 0, 0, 0, 0, 0, 52, 11, 103, 417, 1021, 746, 1219, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 660, 589, 706, 0, 193, 1065, 420, 782, 0, 0, 0, 0, 0, 0, 0, 0, 1154, 667, 253, 203, 919, 639, 16, 100, 0, 0, 0, 0, 0, 0, 0, 0, 931, 547, 606, 59, 683, 868, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 1392, 18, 334, 12, 9, 22, 1269, 12, 910, 554, 126, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 3738, 3624, 4606, 4415, 3951, 3716, 4658] avg=4096.0000 max=4658 max_rank=7 min=3624 min_rank=2 imbalance_ratio=1.137207 max_load_reduction=28110 max_load_reduction_pct=0.857849 moved_to_echo=16740 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[223, 135, 1630, 0, 0, 824, 631, 336, 348, 406, 10, 0, 0, 0, 0, 0, 0, 679, 67, 0, 14, 6, 485, 109, 1831, 1771, 0, 0, 0, 0, 0, 0, 15, 703, 913, 1560, 49, 96, 316, 0, 229, 0, 0, 0, 0, 0, 0, 0, 70, 30, 90, 233, 766, 865, 974, 892, 0, 0, 0, 0, 0, 0, 0, 0, 1132, 487, 440, 0, 155, 874, 211, 601, 0, 0, 0, 0, 0, 0, 0, 0, 2097, 328, 320, 55, 619, 575, 5, 23, 0, 0, 0, 0, 0, 0, 0, 0, 897, 541, 629, 43, 635, 1036, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 1109, 7, 270, 1, 4, 31, 628, 17, 1045, 476, 160, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4543, 4962, 3881, 3920, 3900, 4022, 3792, 3748] avg=4096.0000 max=4962 max_rank=1 min=3748 min_rank=7 imbalance_ratio=1.211426 max_load_reduction=27806 max_load_reduction_pct=0.848572 moved_to_echo=15462 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[141, 160, 1013, 1, 0, 649, 1127, 342, 289, 206, 9, 0, 0, 0, 0, 0, 1, 459, 16, 3, 21, 9, 278, 192, 1863, 1263, 0, 0, 0, 0, 0, 0, 2, 884, 757, 1478, 70, 134, 298, 13, 193, 0, 0, 0, 0, 0, 0, 0, 68, 42, 260, 235, 1004, 846, 1154, 742, 0, 0, 0, 0, 0, 0, 0, 0, 576, 636, 517, 0, 206, 1470, 521, 662, 0, 0, 0, 0, 0, 0, 0, 0, 1512, 419, 228, 71, 730, 1089, 23, 33, 0, 0, 0, 0, 0, 0, 0, 0, 887, 674, 669, 43, 768, 876, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 1173, 17, 320, 5, 31, 30, 485, 30, 1276, 396, 114, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3937, 4105, 3829, 4351, 4588, 4105, 3976, 3877] avg=4096.0000 max=4588 max_rank=4 min=3829 min_rank=2 imbalance_ratio=1.120117 max_load_reduction=28180 max_load_reduction_pct=0.859985 moved_to_echo=16546 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[176, 283, 1042, 0, 2, 548, 940, 207, 304, 204, 9, 0, 0, 0, 0, 0, 2, 466, 25, 4, 14, 42, 441, 250, 1686, 845, 0, 0, 0, 0, 0, 0, 18, 1162, 796, 1868, 22, 311, 223, 6, 207, 0, 0, 0, 0, 0, 0, 0, 67, 41, 98, 435, 1199, 699, 820, 782, 0, 0, 0, 0, 0, 0, 0, 0, 570, 241, 422, 0, 153, 835, 635, 743, 0, 0, 0, 0, 0, 0, 0, 0, 1268, 409, 153, 173, 808, 860, 13, 25, 0, 0, 0, 0, 0, 0, 0, 0, 769, 863, 597, 84, 976, 936, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 1503, 21, 266, 7, 31, 26, 1022, 44, 1494, 417, 76, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3715, 3775, 4613, 4141, 3599, 3709, 4309, 4907] avg=4096.0000 max=4907 max_rank=7 min=3599 min_rank=4 imbalance_ratio=1.197998 max_load_reduction=27861 max_load_reduction_pct=0.850250 moved_to_echo=16524 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[75, 153, 1424, 4, 11, 660, 672, 633, 328, 325, 14, 0, 0, 0, 0, 0, 2, 548, 28, 5, 2, 8, 426, 120, 1606, 1468, 0, 0, 0, 0, 0, 0, 7, 919, 951, 1467, 26, 130, 285, 1, 225, 0, 0, 0, 0, 0, 0, 0, 83, 18, 223, 318, 756, 374, 994, 824, 0, 0, 0, 0, 0, 0, 0, 0, 890, 534, 620, 0, 102, 859, 367, 549, 0, 0, 0, 0, 0, 0, 0, 0, 2081, 363, 256, 58, 895, 1269, 9, 96, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 545, 587, 72, 614, 995, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 1072, 10, 267, 7, 24, 11, 676, 37, 1147, 450, 137, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4299, 4213, 4011, 3590, 3921, 5027, 3869, 3838] avg=4096.0000 max=5027 max_rank=5 min=3590 min_rank=3 imbalance_ratio=1.227295 max_load_reduction=27741 max_load_reduction_pct=0.846588 moved_to_echo=16655 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1280, 1920, 10112, 128, 128, 5248, 6272, 3456, 2560, 2176, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[7296, 5504, 5120, 512, 6272, 8192, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[12800, 3712, 1920, 896, 6528, 6656, 256, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[640, 256, 1152, 2688, 8064, 5248, 8832, 6400, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[9728, 256, 2432, 128, 256, 256, 6912, 384, 8832, 3456, 1024, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 7168, 7040, 12928, 384, 1536, 2048, 128, 1920, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 4352, 384, 128, 128, 256, 3968, 1280, 12416, 10368, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[5760, 3968, 4224, 0, 1408, 7680, 4608, 5504, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[2149, 0, 0, 0, 11, 770, 1, 1752, 77, 1645, 849, 3709, 0, 0, 1072, 0, 6, 0, 314, 85, 70, 10, 325, 3, 0, 576, 0, 0, 505, 2504, 645, 46, 0, 3, 1, 0, 1640, 1577, 665, 0, 131, 0, 750, 0, 0, 0, 1785, 0, 1, 0, 36, 1566, 3545, 1, 0, 0, 1032, 1, 99, 2302, 21, 0, 488, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1381, 0, 15, 0, 4, 122, 0, 2825, 12, 141, 1056, 1846, 0, 0, 3076, 0, 0, 0, 2, 403, 981, 43, 66, 0, 0, 380, 0, 4, 45, 2006, 870, 0, 0, 1, 0, 0, 566, 243, 658, 0, 182, 0, 294, 0, 0, 0, 1586, 2, 0, 0, 478, 3958, 3063, 0, 0, 0, 2112, 0, 1226, 2966, 3, 0, 152, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[2861, 0, 17, 0, 0, 422, 5, 764, 108, 362, 773, 2966, 4, 0, 1126, 11, 1, 0, 33, 0, 429, 1113, 310, 41, 1, 54, 0, 0, 203, 2003, 1834, 265, 2, 0, 0, 2, 943, 1505, 918, 6, 1195, 0, 1176, 0, 0, 0, 380, 0, 0, 0, 155, 3448, 2234, 0, 21, 1, 299, 0, 769, 3890, 22, 0, 96, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1916, 5, 0, 0, 0, 195, 0, 933, 155, 1073, 901, 2775, 2, 0, 2180, 0, 3, 0, 33, 1048, 336, 5, 112, 1, 3, 1733, 0, 0, 1057, 1407, 390, 149, 3, 0, 0, 0, 114, 775, 746, 0, 172, 0, 1823, 3, 0, 0, 1465, 0, 0, 1, 50, 3464, 3390, 0, 0, 0, 2127, 0, 1227, 874, 97, 0, 25, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[705, 0, 0, 0, 1, 1232, 3, 1708, 543, 61, 1367, 1151, 0, 0, 2854, 0, 0, 0, 383, 105, 1459, 29, 248, 0, 0, 1040, 0, 0, 972, 1959, 774, 25, 0, 0, 0, 0, 654, 635, 3794, 0, 1397, 1, 28, 0, 0, 0, 1246, 0, 2, 1, 442, 2745, 2688, 0, 2, 1, 184, 1, 130, 1932, 224, 0, 42, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1742, 0, 15, 0, 4, 11, 0, 1912, 9, 487, 1830, 1938, 0, 0, 756, 15, 0, 0, 95, 182, 221, 34, 378, 0, 0, 2901, 0, 0, 696, 3625, 348, 0, 0, 0, 4, 0, 1055, 920, 861, 0, 5, 0, 64, 2, 0, 0, 1264, 0, 0, 33, 37, 3934, 3547, 0, 0, 0, 628, 0, 3028, 148, 22, 0, 17, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[89, 4, 35, 0, 0, 263, 0, 580, 3, 860, 1345, 3553, 1, 0, 2042, 0, 7, 1, 24, 1215, 1560, 23, 126, 0, 0, 2273, 0, 0, 31, 3352, 461, 1, 0, 0, 10, 0, 1776, 1363, 970, 0, 18, 0, 583, 5, 0, 15, 489, 0, 3, 0, 76, 2645, 1829, 24, 2, 0, 2436, 0, 1409, 173, 199, 0, 894, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1017, 4, 91, 0, 0, 181, 1, 528, 258, 993, 2513, 2644, 3, 2, 1797, 1, 0, 0, 338, 71, 2980, 158, 62, 0, 0, 824, 0, 0, 95, 1246, 1688, 0, 0, 0, 0, 0, 2265, 789, 299, 0, 210, 0, 428, 4, 1, 0, 593, 0, 33, 2, 391, 3807, 2345, 2, 7, 5, 2426, 1, 759, 519, 1, 3, 379, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2861, 0, 17, 0, 0, 422, 5, 764, 432, 799, 0, 0, 0, 0, 0, 0, 108, 362, 773, 52, 4, 0, 1126, 11, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 33, 0, 429, 1113, 310, 41, 2497, 0, 0, 0, 0, 0, 0, 0, 1, 54, 0, 0, 203, 1308, 1834, 265, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 943, 1505, 918, 6, 633, 263, 0, 0, 0, 0, 0, 0, 1195, 0, 1176, 0, 0, 0, 380, 0, 1701, 417, 0, 0, 0, 0, 0, 0, 0, 0, 155, 1114, 2234, 0, 21, 1, 0, 0, 0, 0, 0, 0, 0, 0, 299, 0, 769, 3091, 22, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5300, 2436, 4424, 3665, 4272, 4869, 3525, 4277] avg=4096.0000 max=5300 max_rank=0 min=2436 min_rank=1 imbalance_ratio=1.293945 max_load_reduction=27468 max_load_reduction_pct=0.838257 moved_to_echo=16943 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[89, 4, 35, 0, 0, 263, 0, 580, 717, 35, 0, 0, 0, 0, 0, 0, 3, 860, 1345, 71, 1, 0, 2042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1, 24, 1215, 1560, 23, 126, 0, 2985, 0, 0, 0, 0, 0, 0, 0, 0, 2273, 0, 0, 31, 2201, 461, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 1776, 1363, 970, 0, 481, 434, 0, 0, 0, 0, 0, 0, 18, 0, 583, 5, 0, 15, 489, 0, 1302, 497, 0, 0, 0, 0, 0, 0, 3, 0, 76, 862, 1829, 24, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2436, 0, 1409, 138, 199, 0, 894, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1723, 4322, 5941, 4967, 5034, 2909, 2796, 5076] avg=4096.0000 max=5941 max_rank=2 min=1723 min_rank=0 imbalance_ratio=1.450439 max_load_reduction=26827 max_load_reduction_pct=0.818695 moved_to_echo=15815 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1017, 4, 91, 0, 0, 181, 1, 528, 266, 106, 0, 0, 0, 0, 0, 0, 258, 993, 2513, 53, 3, 2, 1797, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 338, 71, 2980, 158, 62, 0, 2221, 0, 0, 0, 0, 0, 0, 0, 0, 824, 0, 0, 95, 819, 1688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2265, 789, 299, 0, 693, 161, 0, 0, 0, 0, 0, 0, 210, 0, 428, 4, 1, 0, 593, 0, 1874, 370, 0, 0, 0, 0, 0, 0, 33, 2, 391, 1240, 2345, 2, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 2426, 1, 759, 413, 1, 3, 379, 4, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2194, 5620, 5830, 3426, 4207, 3480, 4025, 3986] avg=4096.0000 max=5830 max_rank=2 min=2194 min_rank=0 imbalance_ratio=1.423340 max_load_reduction=26938 max_load_reduction_pct=0.822083 moved_to_echo=15698 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1381, 0, 15, 0, 4, 122, 0, 2825, 429, 606, 0, 0, 0, 0, 0, 0, 12, 141, 1056, 37, 0, 0, 3076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 403, 981, 43, 66, 0, 1551, 0, 0, 0, 0, 0, 0, 0, 0, 380, 0, 4, 45, 1317, 870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 566, 243, 658, 0, 720, 260, 0, 0, 0, 0, 0, 0, 182, 0, 294, 0, 0, 0, 1586, 2, 1949, 258, 0, 0, 0, 0, 0, 0, 0, 0, 478, 1289, 3063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2112, 0, 1226, 2360, 3, 0, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5382, 4322, 3046, 2616, 2448, 4271, 4830, 5853] avg=4096.0000 max=5853 max_rank=7 min=2448 min_rank=4 imbalance_ratio=1.428955 max_load_reduction=26915 max_load_reduction_pct=0.821381 moved_to_echo=17402 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[705, 0, 0, 0, 1, 1232, 3, 1708, 419, 395, 0, 0, 0, 0, 0, 0, 543, 61, 1367, 23, 0, 0, 2854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 383, 105, 1459, 29, 248, 0, 967, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 0, 0, 972, 1286, 774, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 654, 635, 3794, 0, 499, 254, 0, 0, 0, 0, 0, 0, 1397, 1, 28, 0, 0, 0, 1246, 0, 1351, 161, 0, 0, 0, 0, 0, 0, 2, 1, 442, 895, 2688, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 184, 1, 130, 1537, 224, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4463, 4848, 3191, 4097, 5836, 4184, 4031, 2118] avg=4096.0000 max=5836 max_rank=4 min=2118 min_rank=7 imbalance_ratio=1.424805 max_load_reduction=26932 max_load_reduction_pct=0.821899 moved_to_echo=16169 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1742, 0, 15, 0, 4, 11, 0, 1912, 775, 30, 0, 0, 0, 0, 0, 0, 9, 487, 1830, 39, 0, 0, 756, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 182, 221, 34, 378, 0, 1628, 0, 0, 0, 0, 0, 0, 0, 0, 2901, 0, 0, 696, 2380, 348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1055, 920, 861, 0, 716, 470, 0, 0, 0, 0, 0, 0, 5, 0, 64, 2, 0, 0, 1264, 0, 1937, 271, 0, 0, 0, 0, 0, 0, 0, 33, 37, 1281, 3547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 628, 0, 3028, 118, 22, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4489, 3136, 2538, 6325, 4026, 3543, 4898, 3813] avg=4096.0000 max=6325 max_rank=3 min=2538 min_rank=2 imbalance_ratio=1.544189 max_load_reduction=26443 max_load_reduction_pct=0.806976 moved_to_echo=16280 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2149, 0, 0, 0, 11, 770, 1, 1752, 536, 470, 0, 0, 0, 0, 0, 0, 77, 1645, 849, 74, 0, 0, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 314, 85, 70, 10, 325, 3, 3116, 0, 0, 0, 0, 0, 0, 0, 0, 576, 0, 0, 505, 1644, 645, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 1640, 1577, 665, 0, 285, 324, 0, 0, 0, 0, 0, 0, 131, 0, 750, 0, 0, 0, 1785, 0, 771, 519, 0, 0, 0, 0, 0, 0, 1, 0, 36, 510, 3545, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1, 99, 1832, 21, 0, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5689, 3717, 3929, 3416, 4495, 3956, 4093, 3473] avg=4096.0000 max=5689 max_rank=0 min=3416 min_rank=3 imbalance_ratio=1.388916 max_load_reduction=27079 max_load_reduction_pct=0.826385 moved_to_echo=16017 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1916, 5, 0, 0, 0, 195, 0, 933, 301, 178, 0, 0, 0, 0, 0, 0, 155, 1073, 901, 56, 2, 0, 2180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 33, 1048, 336, 5, 112, 1, 2331, 0, 0, 0, 0, 0, 0, 0, 3, 1733, 0, 0, 1057, 924, 390, 149, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 114, 775, 746, 0, 630, 182, 0, 0, 0, 0, 0, 0, 172, 0, 1823, 3, 0, 0, 1465, 0, 1705, 388, 0, 0, 0, 0, 0, 0, 0, 1, 50, 1129, 3390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2127, 0, 1227, 696, 97, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3528, 4367, 3869, 4256, 2450, 5556, 4570, 4172] avg=4096.0000 max=5556 max_rank=5 min=2450 min_rank=4 imbalance_ratio=1.356445 max_load_reduction=27212 max_load_reduction_pct=0.830444 moved_to_echo=16748 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[11904, 128, 256, 0, 128, 3200, 128, 11008, 3968, 2688, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[3328, 128, 5248, 128, 128, 128, 8832, 128, 12672, 2944, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[11264, 128, 8704, 10240, 640, 128, 2176, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 9856, 0, 128, 3712, 11904, 7040, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[1280, 5632, 10752, 512, 128, 128, 14976, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 9088, 7808, 8960, 128, 4736, 2432, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 1792, 8320, 22656, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 1280, 3200, 8064, 1536, 1664, 128, 17408, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 1784, 0, 892, 0, 2486, 0, 0, 21, 0, 57, 0, 12, 1805, 3, 2100, 28, 2, 0, 0, 0, 0, 0, 0, 1756, 157, 0, 0, 0, 0, 1749, 1697, 0, 118, 2749, 737, 0, 0, 0, 0, 0, 1449, 0, 0, 0, 78, 9, 3986, 1190, 0, 104, 2977, 0, 1, 1368, 20, 0, 14, 79, 3338, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 199, 0, 28, 0, 250, 0, 3866, 5, 0, 6, 0, 811, 0, 112, 1009, 171, 3945, 34, 0, 1, 0, 0, 0, 2, 0, 2119, 0, 359, 0, 0, 0, 1067, 48, 0, 8, 4070, 2541, 5, 0, 0, 0, 0, 2673, 11, 2, 0, 1526, 0, 4074, 1925, 1, 18, 571, 1, 0, 42, 0, 0, 0, 914, 353, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 44, 3, 1, 237, 0, 61, 0, 1276, 0, 17, 30, 0, 1859, 0, 20, 451, 0, 4038, 296, 144, 1, 0, 0, 0, 0, 0, 2785, 0, 41, 0, 0, 222, 1375, 16, 5, 26, 4047, 2075, 0, 0, 0, 7, 0, 3943, 1, 0, 0, 156, 4, 4080, 3590, 0, 1, 57, 0, 0, 375, 7, 2, 35, 133, 1305, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 12, 10, 0, 341, 0, 261, 0, 2768, 0, 1, 4, 0, 445, 0, 0, 588, 9, 3267, 22, 0, 0, 0, 0, 0, 0, 0, 1331, 5, 6, 0, 0, 0, 4042, 43, 5, 18, 3413, 1650, 0, 0, 0, 0, 5, 2395, 0, 0, 0, 227, 1, 4094, 2188, 0, 50, 1163, 0, 0, 1333, 3, 0, 1, 176, 2890, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 20, 0, 14, 118, 0, 113, 1, 1877, 0, 1, 6, 2, 1418, 0, 1162, 520, 4, 3921, 1, 0, 0, 0, 3, 0, 3, 0, 2953, 1, 0, 0, 0, 0, 2220, 7, 6, 0, 1585, 1261, 0, 3, 0, 0, 89, 2030, 0, 0, 0, 1513, 0, 4044, 3040, 0, 27, 1326, 0, 1, 1609, 0, 0, 2, 77, 1789, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 634, 0, 745, 16, 655, 0, 0, 3, 0, 2749, 0, 11, 1274, 23, 3904, 146, 74, 0, 0, 0, 1, 0, 0, 1733, 173, 0, 0, 0, 10, 1091, 60, 0, 1, 3018, 2372, 0, 0, 0, 1, 0, 2255, 8, 1, 0, 22, 75, 4096, 3409, 0, 67, 307, 0, 2, 315, 6, 0, 240, 373, 2896, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1246, 0, 1686, 0, 786, 0, 1, 448, 0, 65, 0, 19, 1379, 1, 3697, 1, 1, 0, 0, 0, 0, 0, 0, 1096, 43, 13, 0, 0, 7, 1007, 61, 0, 45, 3920, 241, 0, 0, 0, 0, 0, 2029, 0, 1, 0, 1386, 64, 4074, 1276, 0, 549, 1291, 0, 0, 2015, 30, 0, 5, 1042, 3243, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 27, 3, 0, 25, 2, 13, 0, 3812, 4, 1, 186, 4, 176, 0, 25, 759, 69, 3258, 23, 7, 8, 0, 0, 0, 5, 0, 2566, 0, 4, 0, 2, 50, 1271, 2, 0, 4, 1763, 3478, 0, 0, 0, 0, 1, 2381, 0, 1, 0, 218, 0, 3892, 1797, 0, 568, 1417, 12, 0, 3428, 4, 1, 1, 25, 1475, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 1784, 0, 892, 0, 2955, 0, 0, 0, 0, 0, 0, 0, 2486, 0, 0, 21, 0, 57, 0, 12, 287, 190, 0, 0, 0, 0, 0, 0, 1805, 3, 1786, 28, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1756, 157, 0, 0, 0, 1031, 811, 0, 0, 0, 0, 0, 0, 0, 1749, 1697, 0, 118, 240, 737, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1449, 0, 0, 0, 1508, 0, 0, 0, 0, 0, 0, 0, 78, 9, 0, 1098, 0, 104, 2977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1368, 20, 0, 14, 79, 3338, 0, 92, 27, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5633, 3053, 3624, 3755, 4541, 2957, 4266, 4939] avg=4096.0000 max=5633 max_rank=0 min=2957 min_rank=5 imbalance_ratio=1.375244 max_load_reduction=27135 max_load_reduction_pct=0.828094 moved_to_echo=16703 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 44, 3, 1, 237, 0, 61, 0, 3025, 0, 0, 0, 0, 0, 0, 0, 1276, 0, 17, 30, 0, 1859, 0, 20, 553, 280, 0, 0, 0, 0, 0, 0, 451, 0, 3433, 296, 144, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2785, 0, 41, 0, 0, 1055, 1194, 0, 0, 0, 0, 0, 0, 222, 1375, 16, 5, 26, 352, 2075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 3943, 1, 0, 0, 2221, 0, 0, 0, 0, 0, 0, 0, 156, 4, 0, 3311, 0, 1, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 375, 7, 2, 35, 133, 1305, 2, 279, 52, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3371, 4035, 4325, 5075, 4071, 6172, 3529, 2190] avg=4096.0000 max=6172 max_rank=5 min=2190 min_rank=7 imbalance_ratio=1.506836 max_load_reduction=26596 max_load_reduction_pct=0.811646 moved_to_echo=15962 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 20, 0, 14, 118, 0, 113, 1, 2998, 0, 0, 0, 0, 0, 0, 0, 1877, 0, 1, 6, 2, 1418, 0, 1162, 537, 110, 0, 0, 0, 0, 0, 0, 520, 4, 3334, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2953, 1, 0, 0, 0, 1046, 467, 0, 0, 0, 0, 0, 0, 0, 2220, 7, 6, 0, 139, 1261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 89, 2030, 0, 0, 0, 869, 0, 0, 0, 0, 0, 0, 0, 1513, 0, 0, 2804, 0, 27, 1326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1609, 0, 0, 2, 77, 1789, 1, 236, 50, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3264, 5113, 3862, 4470, 3633, 2991, 5670, 3765] avg=4096.0000 max=5670 max_rank=6 min=2991 min_rank=5 imbalance_ratio=1.384277 max_load_reduction=27098 max_load_reduction_pct=0.826965 moved_to_echo=16059 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 199, 0, 28, 0, 250, 0, 3021, 0, 0, 0, 0, 0, 0, 0, 3866, 5, 0, 6, 0, 811, 0, 112, 540, 282, 0, 0, 0, 0, 0, 0, 1009, 171, 3355, 34, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2119, 0, 359, 0, 0, 1053, 1201, 0, 0, 0, 0, 0, 0, 0, 1067, 48, 0, 8, 354, 2541, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2673, 11, 2, 0, 2233, 0, 0, 0, 0, 0, 0, 0, 1526, 0, 0, 1776, 1, 18, 571, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 914, 353, 0, 149, 50, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3499, 5622, 4570, 4734, 4023, 4919, 3893, 1508] avg=4096.0000 max=5622 max_rank=1 min=1508 min_rank=7 imbalance_ratio=1.372559 max_load_reduction=27146 max_load_reduction_pct=0.828430 moved_to_echo=14343 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 12, 10, 0, 341, 0, 261, 0, 3036, 0, 0, 0, 0, 0, 0, 0, 2768, 0, 1, 4, 0, 445, 0, 0, 447, 236, 0, 0, 0, 0, 0, 0, 588, 9, 2778, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1331, 5, 6, 0, 0, 1058, 1007, 0, 0, 0, 0, 0, 0, 0, 4042, 43, 5, 18, 297, 1650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2395, 0, 0, 0, 1873, 0, 0, 0, 0, 0, 0, 0, 227, 1, 0, 2018, 0, 50, 1163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1333, 3, 0, 1, 176, 2890, 1, 170, 42, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3660, 3901, 3397, 3407, 6055, 4273, 3459, 4616] avg=4096.0000 max=6055 max_rank=4 min=3397 min_rank=2 imbalance_ratio=1.478271 max_load_reduction=26713 max_load_reduction_pct=0.815216 moved_to_echo=18403 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 634, 0, 745, 16, 3036, 0, 0, 0, 0, 0, 0, 0, 655, 0, 0, 3, 0, 2749, 0, 11, 535, 209, 0, 0, 0, 0, 0, 0, 1274, 23, 3319, 146, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1733, 173, 0, 0, 0, 1060, 890, 0, 0, 0, 0, 0, 0, 10, 1091, 60, 0, 1, 263, 2372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2255, 8, 1, 0, 1656, 0, 0, 0, 0, 0, 0, 0, 22, 75, 0, 3144, 0, 67, 307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 315, 6, 0, 240, 373, 2896, 0, 265, 50, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4433, 4162, 4836, 3857, 3797, 3921, 3615, 4147] avg=4096.0000 max=4836 max_rank=2 min=3615 min_rank=6 imbalance_ratio=1.180664 max_load_reduction=27932 max_load_reduction_pct=0.852417 moved_to_echo=15480 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1246, 0, 1686, 0, 3020, 0, 0, 0, 0, 0, 0, 0, 786, 0, 1, 448, 0, 65, 0, 19, 506, 272, 0, 0, 0, 0, 0, 0, 1379, 1, 3144, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1096, 43, 13, 0, 0, 1054, 1157, 0, 0, 0, 0, 0, 0, 7, 1007, 61, 0, 45, 340, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2029, 0, 1, 0, 2151, 0, 0, 0, 0, 0, 0, 0, 1386, 64, 0, 1177, 0, 549, 1291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2015, 30, 0, 5, 1042, 3243, 0, 99, 47, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5952, 2097, 4526, 3363, 1701, 4181, 4467, 6481] avg=4096.0000 max=6481 max_rank=7 min=1701 min_rank=4 imbalance_ratio=1.582275 max_load_reduction=26287 max_load_reduction_pct=0.802216 moved_to_echo=16830 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 27, 3, 0, 25, 2, 13, 0, 2886, 0, 0, 0, 0, 0, 0, 0, 3812, 4, 1, 186, 4, 176, 0, 25, 451, 126, 0, 0, 0, 0, 0, 0, 759, 69, 2762, 23, 7, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2566, 0, 4, 0, 2, 1006, 524, 0, 0, 0, 0, 0, 0, 50, 1271, 2, 0, 4, 142, 3478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2381, 0, 1, 0, 971, 0, 0, 0, 0, 0, 0, 0, 218, 0, 0, 1654, 0, 568, 1417, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3428, 4, 1, 1, 25, 1475, 0, 143, 45, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2956, 4785, 3628, 4107, 4947, 3354, 3869, 5122] avg=4096.0000 max=5122 max_rank=7 min=2956 min_rank=0 imbalance_ratio=1.250488 max_load_reduction=27646 max_load_reduction_pct=0.843689 moved_to_echo=17292 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[17536, 128, 128, 768, 128, 7680, 0, 1408, 3968, 1792, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 0, 16384, 384, 512, 0, 128, 8448, 7296, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[384, 13824, 2048, 128, 256, 2176, 14464, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 19200, 128, 128, 0, 13568, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[7808, 384, 23936, 640, 256, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[5248, 256, 0, 17024, 128, 1408, 9216, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 10496, 128, 128, 384, 2944, 17408, 128, 1536, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 256, 128, 4480, 128, 4096, 128, 24064, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 117, 0, 0, 0, 2475, 3327, 0, 0, 1727, 0, 862, 2, 0, 2281, 375, 0, 0, 9, 0, 0, 0, 0, 42, 2, 674, 0, 4050, 0, 2, 1618, 0, 1, 0, 0, 0, 0, 0, 3644, 1734, 0, 0, 0, 19, 0, 1998, 0, 0, 4015, 0, 13, 0, 0, 0, 1, 2, 3777, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 2, 0, 14, 0, 254, 4076, 2, 0, 1627, 4, 2745, 0, 25, 2409, 2147, 0, 0, 123, 0, 0, 2, 0, 1, 4, 3324, 5, 4088, 0, 7, 1188, 0, 0, 0, 2, 0, 0, 6, 194, 15, 0, 0, 0, 172, 0, 1936, 0, 10, 3407, 0, 913, 0, 3, 1, 6, 0, 4051, 3, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 2, 1, 3, 13, 7, 9, 0, 1437, 1490, 0, 0, 2171, 2, 103, 0, 0, 3053, 1299, 0, 0, 1, 0, 0, 0, 0, 3, 39, 3846, 0, 4088, 0, 0, 3137, 0, 29, 7, 0, 0, 14, 1, 3301, 658, 0, 1, 0, 36, 0, 2633, 0, 0, 1148, 0, 187, 0, 0, 0, 16, 1, 4032, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 2, 0, 0, 565, 3507, 0, 0, 2826, 0, 2470, 0, 0, 2285, 437, 5, 0, 0, 0, 0, 0, 0, 32, 0, 287, 0, 3991, 0, 0, 3367, 0, 0, 0, 0, 0, 0, 0, 2197, 793, 1, 0, 0, 0, 0, 2216, 0, 0, 3981, 0, 10, 0, 0, 0, 0, 0, 3790, 5, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 12, 0, 0, 0, 15, 1620, 0, 0, 3892, 0, 573, 0, 0, 2294, 2278, 0, 1, 0, 0, 0, 0, 0, 20, 0, 2963, 0, 4081, 0, 0, 2799, 0, 0, 0, 0, 0, 0, 0, 3354, 830, 0, 0, 0, 356, 0, 628, 0, 0, 2931, 0, 572, 0, 0, 0, 0, 0, 3542, 0, 0, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 0, 620, 0, 4, 0, 805, 3871, 0, 0, 9, 0, 1732, 0, 20, 2642, 2496, 0, 0, 61, 7, 0, 6, 1, 27, 8, 778, 4, 3613, 0, 7, 3838, 0, 1, 14, 6, 0, 0, 0, 1926, 22, 17, 0, 0, 0, 0, 3677, 0, 0, 3712, 10, 261, 0, 1, 0, 0, 10, 2557, 0, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 18, 0, 333, 2310, 0, 0, 1024, 0, 45, 0, 20, 439, 764, 0, 0, 43, 0, 0, 0, 0, 1, 0, 3759, 0, 4052, 0, 0, 2111, 0, 0, 0, 0, 0, 0, 0, 4033, 1915, 0, 0, 0, 25, 0, 3936, 0, 0, 4022, 0, 207, 0, 0, 0, 0, 0, 3710, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[7, 4, 0, 0, 0, 460, 0, 0, 0, 2083, 698, 1, 0, 51, 1, 1626, 5, 0, 3635, 578, 3, 33, 0, 0, 0, 1, 0, 37, 38, 3496, 2, 4090, 2, 0, 1294, 1, 0, 0, 0, 1, 4, 1, 4048, 1591, 2, 0, 0, 0, 0, 1707, 0, 5, 3912, 6, 0, 0, 8, 5, 0, 0, 3295, 0, 3, 34] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 117, 0, 0, 3118, 1500, 0, 0, 0, 0, 0, 0, 0, 2475, 209, 0, 0, 1727, 0, 862, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2281, 375, 0, 0, 9, 0, 443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 2, 674, 0, 1690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1618, 0, 1, 0, 0, 0, 860, 965, 0, 0, 0, 0, 0, 0, 0, 0, 3644, 1734, 0, 0, 0, 19, 274, 0, 0, 0, 0, 0, 0, 0, 0, 1998, 0, 0, 1756, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3777, 0, 0, 1, 577, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4735, 5273, 3110, 2408, 3446, 5671, 3767, 4358] avg=4096.0000 max=5671 max_rank=5 min=2408 min_rank=3 imbalance_ratio=1.384521 max_load_reduction=27097 max_load_reduction_pct=0.826935 moved_to_echo=17242 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 2, 0, 14, 3820, 1514, 0, 0, 0, 0, 0, 0, 0, 254, 256, 2, 0, 1627, 4, 2745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 2409, 2147, 0, 0, 123, 0, 376, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 4, 3324, 5, 1706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1188, 0, 0, 0, 2, 0, 868, 819, 0, 0, 0, 0, 0, 0, 0, 6, 194, 15, 0, 0, 0, 172, 233, 0, 0, 0, 0, 0, 0, 0, 0, 1936, 0, 10, 1489, 0, 913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 6, 0, 4051, 3, 0, 0, 490, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5352, 4888, 5080, 5042, 2884, 620, 4348, 4554] avg=4096.0000 max=5352 max_rank=0 min=620 min_rank=5 imbalance_ratio=1.306641 max_load_reduction=27416 max_load_reduction_pct=0.836670 moved_to_echo=12406 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 1, 3, 13, 7, 9, 1396, 1514, 0, 0, 0, 0, 0, 0, 0, 1437, 94, 0, 0, 2171, 2, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3053, 1299, 0, 0, 1, 0, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 39, 3846, 0, 1706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3137, 0, 29, 7, 0, 0, 868, 276, 0, 0, 0, 0, 0, 0, 14, 1, 3301, 658, 0, 1, 0, 36, 78, 0, 0, 0, 0, 0, 0, 0, 0, 2633, 0, 0, 503, 0, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 1, 4032, 0, 0, 0, 165, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2945, 3807, 4479, 5594, 4317, 4089, 3323, 4214] avg=4096.0000 max=5594 max_rank=3 min=2945 min_rank=0 imbalance_ratio=1.365723 max_load_reduction=27174 max_load_reduction_pct=0.829285 moved_to_echo=15943 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 12, 0, 0, 1518, 1511, 0, 0, 0, 0, 0, 0, 0, 15, 102, 0, 0, 3892, 0, 573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2294, 2278, 0, 1, 0, 0, 323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 2963, 0, 1703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2799, 0, 0, 0, 0, 0, 867, 704, 0, 0, 0, 0, 0, 0, 0, 0, 3354, 830, 0, 0, 0, 356, 200, 0, 0, 0, 0, 0, 0, 0, 0, 628, 0, 0, 1283, 0, 572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3542, 0, 0, 6, 421, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3042, 4582, 4896, 4686, 4370, 4740, 2483, 3969] avg=4096.0000 max=4896 max_rank=2 min=2483 min_rank=6 imbalance_ratio=1.195312 max_load_reduction=27872 max_load_reduction_pct=0.850586 moved_to_echo=15562 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 0, 620, 0, 4, 3628, 1338, 0, 0, 0, 0, 0, 0, 0, 805, 243, 0, 0, 9, 0, 1732, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 2642, 2496, 0, 0, 61, 7, 410, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 27, 8, 778, 4, 1508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3838, 0, 1, 14, 6, 0, 767, 892, 0, 0, 0, 0, 0, 0, 0, 0, 1926, 22, 17, 0, 0, 0, 254, 0, 0, 0, 0, 0, 0, 0, 0, 3677, 0, 0, 1622, 10, 261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 10, 2557, 0, 2, 0, 534, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5593, 2789, 5636, 2332, 5525, 2219, 5570, 3104] avg=4096.0000 max=5636 max_rank=2 min=2219 min_rank=5 imbalance_ratio=1.375977 max_load_reduction=27132 max_load_reduction_pct=0.828003 moved_to_echo=16418 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 2, 0, 3287, 1478, 0, 0, 0, 0, 0, 0, 0, 565, 220, 0, 0, 2826, 0, 2470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2285, 437, 5, 0, 0, 0, 439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 287, 0, 1665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3367, 0, 0, 0, 0, 0, 848, 957, 0, 0, 0, 0, 0, 0, 0, 0, 2197, 793, 1, 0, 0, 0, 272, 0, 0, 0, 0, 0, 0, 0, 0, 2216, 0, 0, 1740, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3790, 5, 0, 0, 573, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4768, 6081, 3166, 1984, 5172, 3263, 3966, 4368] avg=4096.0000 max=6081 max_rank=1 min=1984 min_rank=3 imbalance_ratio=1.484619 max_load_reduction=26687 max_load_reduction_pct=0.814423 moved_to_echo=16769 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 18, 2165, 1501, 0, 0, 0, 0, 0, 0, 0, 333, 145, 0, 0, 1024, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 439, 764, 0, 0, 43, 0, 444, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3759, 0, 1690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2111, 0, 0, 0, 0, 0, 861, 967, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 1915, 0, 0, 0, 25, 275, 0, 0, 0, 0, 0, 0, 0, 0, 3936, 0, 0, 1758, 0, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3710, 0, 0, 1, 578, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3684, 1547, 1710, 5450, 3939, 6248, 5901, 4289] avg=4096.0000 max=6248 max_rank=5 min=1547 min_rank=1 imbalance_ratio=1.525391 max_load_reduction=26520 max_load_reduction_pct=0.809326 moved_to_echo=20377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[7, 4, 0, 0, 0, 460, 0, 0, 659, 1519, 0, 0, 0, 0, 0, 0, 0, 2083, 39, 1, 0, 51, 1, 1626, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3635, 578, 3, 33, 0, 0, 437, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 37, 38, 3496, 2, 1698, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1294, 1, 0, 0, 0, 1, 873, 944, 0, 0, 0, 0, 0, 0, 4, 1, 4048, 1591, 2, 0, 0, 0, 272, 0, 0, 0, 0, 0, 0, 0, 0, 1707, 0, 5, 1692, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 5, 0, 0, 3295, 0, 3, 34, 567, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2649, 3801, 4691, 5272, 3115, 5918, 3410, 3912] avg=4096.0000 max=5918 max_rank=5 min=2649 min_rank=0 imbalance_ratio=1.444824 max_load_reduction=26850 max_load_reduction_pct=0.819397 moved_to_echo=16355 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 128, 1280, 128, 128, 19712, 11904, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 18816, 0, 128, 11904, 128, 2176, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 22784, 7680, 128, 128, 0, 640, 1920, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 8064, 1408, 128, 0, 13440, 128, 10240, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[128, 128, 19456, 128, 128, 128, 128, 128, 6912, 6528, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 19072, 10496, 128, 128, 256, 128, 3072, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 256, 128, 19200, 128, 13440, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 28800, 128, 128, 128, 3968, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33792 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33792 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 3505, 0, 0, 4006, 0, 0, 2, 4057, 0, 2, 0, 0, 0, 0, 0, 0, 2, 4, 0, 0, 0, 0, 215, 0, 0, 0, 0, 651, 0, 0, 205, 0, 0, 8, 0, 0, 0, 1, 0, 1, 4012, 508, 4021, 5, 3142, 552, 0, 2, 0, 0, 0, 0, 0, 25, 0, 3756, 0, 4, 0, 0, 0, 1, 4081] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 2084, 0, 0, 2683, 2, 0, 64, 4096, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 18, 0, 0, 3387, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3165, 3343, 3588, 0, 966, 354, 0, 0, 0, 0, 0, 9, 0, 998, 0, 3844, 0, 0, 0, 0, 0, 0, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 3285, 0, 0, 1339, 0, 0, 2, 4082, 39, 0, 0, 90, 2, 10, 0, 0, 0, 0, 3, 9, 0, 2, 50, 15, 0, 0, 0, 1, 0, 0, 2217, 0, 3, 0, 3, 10, 0, 234, 0, 0, 4082, 2113, 3973, 0, 3731, 166, 0, 0, 0, 0, 0, 12, 0, 548, 4, 2636, 7, 25, 0, 6, 0, 0, 4069] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 2731, 0, 19, 2294, 0, 0, 213, 4057, 0, 0, 0, 348, 1, 5, 0, 35, 0, 0, 0, 8, 22, 0, 393, 2, 6, 2, 0, 205, 0, 0, 2822, 0, 1, 0, 0, 0, 0, 295, 0, 0, 3613, 1126, 4053, 0, 3333, 393, 0, 0, 1, 0, 0, 80, 8, 13, 13, 2462, 1, 115, 0, 7, 0, 0, 4091] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 2054, 0, 0, 1467, 0, 0, 9, 4096, 0, 0, 0, 125, 0, 0, 5, 0, 0, 0, 0, 0, 0, 1, 848, 0, 0, 0, 2, 7, 0, 0, 606, 0, 0, 0, 0, 0, 4, 46, 0, 0, 3894, 3658, 4045, 2, 2000, 2260, 0, 0, 0, 0, 0, 28, 5, 20, 0, 3533, 0, 0, 0, 0, 0, 0, 4053] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 2681, 0, 0, 1530, 7, 0, 23, 4091, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0, 3, 0, 0, 3609, 0, 0, 0, 0, 0, 0, 17, 0, 0, 3380, 937, 4052, 0, 2881, 1167, 0, 0, 0, 0, 0, 93, 0, 6, 0, 4063, 0, 0, 0, 0, 0, 0, 3971] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[2, 2596, 4, 0, 2780, 0, 0, 87, 4094, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 15, 0, 421, 0, 0, 0, 0, 586, 14, 0, 1245, 0, 0, 0, 0, 0, 0, 13, 0, 1, 3759, 1877, 3081, 1, 3588, 334, 0, 2, 0, 28, 0, 256, 0, 234, 0, 3646, 0, 13, 0, 0, 0, 1, 4086] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 137, 0, 50, 3407, 0, 2, 293, 4059, 1, 0, 0, 2, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 790, 0, 0, 2, 0, 45, 0, 8, 107, 1, 6, 0, 0, 3, 0, 58, 0, 15, 4058, 3943, 3833, 0, 2928, 838, 0, 0, 0, 1, 64, 13, 0, 37, 4, 3853, 9, 63, 1, 4, 0, 0, 4088] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 3505, 0, 0, 2653, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3996, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 0, 0, 0, 0, 215, 3273, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 651, 0, 0, 205, 1380, 1353, 61, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 1, 0, 4021, 190, 0, 0, 0, 0, 0, 0, 1, 0, 508, 0, 5, 1278, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 25, 0, 3822, 215, 0, 0, 0, 0, 0, 0, 3756, 0, 4, 0, 0, 0, 1, 593, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6160, 3998, 3978, 3650, 4220, 2344, 4064, 4354] avg=4096.0000 max=6160 max_rank=0 min=2344 min_rank=5 imbalance_ratio=1.503906 max_load_reduction=26608 max_load_reduction_pct=0.812012 moved_to_echo=14982 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2681, 0, 0, 1014, 7, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 3185, 444, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3609, 1265, 516, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 4052, 159, 0, 0, 0, 0, 0, 0, 0, 0, 937, 0, 0, 1172, 1167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 6, 0, 3221, 209, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 0, 0, 0, 0, 577, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3725, 4030, 3885, 5455, 4228, 3276, 3529, 4640] avg=4096.0000 max=5455 max_rank=3 min=3276 min_rank=5 imbalance_ratio=1.331787 max_load_reduction=27313 max_load_reduction_pct=0.833527 moved_to_echo=15673 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2054, 0, 0, 972, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 0, 0, 125, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 848, 3251, 308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7, 0, 0, 606, 878, 495, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 46, 0, 4045, 185, 0, 0, 0, 0, 0, 0, 0, 0, 3658, 0, 2, 814, 2260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 5, 20, 0, 3709, 214, 0, 0, 0, 0, 0, 0, 3533, 0, 0, 0, 0, 0, 0, 588, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3035, 4164, 4408, 2050, 4280, 6734, 3976, 4121] avg=4096.0000 max=6734 max_rank=5 min=2050 min_rank=3 imbalance_ratio=1.644043 max_load_reduction=26034 max_load_reduction_pct=0.794495 moved_to_echo=19111 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2731, 0, 19, 1519, 0, 0, 213, 0, 0, 0, 0, 0, 0, 0, 0, 3996, 0, 0, 0, 348, 1, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 8, 22, 0, 393, 3281, 513, 0, 0, 0, 0, 0, 0, 2, 6, 2, 0, 205, 0, 0, 2822, 1464, 775, 61, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 295, 0, 4053, 170, 0, 0, 0, 0, 0, 0, 0, 0, 1126, 0, 0, 1356, 393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 80, 8, 13, 13, 3443, 216, 0, 0, 0, 0, 0, 0, 2462, 1, 115, 0, 7, 0, 0, 594, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4482, 4350, 4252, 5337, 4519, 2875, 3774, 3179] avg=4096.0000 max=5337 max_rank=3 min=2875 min_rank=5 imbalance_ratio=1.302979 max_load_reduction=27431 max_load_reduction_pct=0.837128 moved_to_echo=14347 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 3285, 0, 0, 887, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4020, 39, 0, 0, 90, 2, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 9, 0, 2, 50, 3263, 575, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 1, 0, 0, 2217, 1639, 452, 62, 0, 0, 0, 0, 0, 0, 3, 0, 3, 10, 0, 234, 0, 3973, 194, 0, 0, 0, 0, 0, 0, 0, 0, 2113, 0, 0, 1517, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 548, 4, 3888, 215, 0, 0, 0, 0, 0, 0, 2636, 7, 25, 0, 6, 0, 0, 591, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4174, 4161, 3902, 4386, 4417, 3796, 4667, 3265] avg=4096.0000 max=4667 max_rank=6 min=3265 min_rank=7 imbalance_ratio=1.139404 max_load_reduction=28101 max_load_reduction_pct=0.857574 moved_to_echo=16145 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2084, 0, 0, 1777, 2, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 3283, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 3387, 424, 906, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3588, 149, 0, 0, 0, 0, 0, 0, 0, 0, 3343, 0, 0, 394, 354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 998, 0, 3016, 216, 0, 0, 0, 0, 0, 0, 3844, 0, 0, 0, 0, 0, 0, 595, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3927, 4036, 3501, 4797, 3738, 4091, 4239, 4439] avg=4096.0000 max=4797 max_rank=3 min=3501 min_rank=2 imbalance_ratio=1.171143 max_load_reduction=27971 max_load_reduction_pct=0.853607 moved_to_echo=16507 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2, 2596, 4, 0, 1841, 0, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 15, 0, 421, 3277, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 586, 14, 0, 1245, 1576, 939, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 3081, 178, 0, 0, 0, 0, 0, 0, 1, 0, 1877, 0, 1, 1459, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 28, 0, 256, 0, 234, 0, 3581, 215, 0, 0, 0, 0, 0, 0, 3646, 0, 13, 0, 0, 0, 1, 594, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4530, 4032, 4270, 4422, 3272, 3672, 4316, 4254] avg=4096.0000 max=4530 max_rank=0 min=3272 min_rank=4 imbalance_ratio=1.105957 max_load_reduction=28238 max_load_reduction_pct=0.861755 moved_to_echo=15514 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 137, 0, 50, 2253, 0, 2, 293, 0, 0, 0, 0, 0, 0, 0, 0, 3994, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 790, 3283, 454, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 45, 0, 8, 107, 1290, 1154, 65, 0, 0, 0, 0, 0, 1, 6, 0, 0, 3, 0, 58, 0, 3833, 193, 0, 0, 0, 0, 0, 0, 15, 0, 3943, 0, 0, 1184, 838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 64, 13, 0, 37, 4, 3865, 219, 0, 0, 0, 0, 0, 0, 3853, 9, 63, 1, 4, 0, 0, 586, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2735, 3997, 4572, 2671, 4094, 5980, 4203, 4516] avg=4096.0000 max=5980 max_rank=5 min=2671 min_rank=3 imbalance_ratio=1.459961 max_load_reduction=26788 max_load_reduction_pct=0.817505 moved_to_echo=18793 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 1536, 128, 128, 14208, 9984, 6656, 512, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 17536, 0, 128, 9216, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 19200, 128, 128, 12928, 128, 128, 768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[27904, 128, 256, 128, 128, 0, 128, 4736, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 3072, 26112, 3584, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 512, 128, 1920, 128, 28672, 1792, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 128, 128, 768, 0, 30720, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[32256, 128, 128, 0, 640, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33792 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 3827, 0, 619, 0, 0, 0, 0, 0, 737, 0, 1756, 0, 0, 0, 3737, 161, 66, 0, 0, 0, 0, 2087, 10, 0, 0, 0, 483, 0, 0, 3475, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 226, 1211, 0, 3513, 0, 0, 4096, 0, 0, 0, 0, 0, 2734, 0, 0, 3988, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 1034, 0, 328, 0, 0, 0, 1, 0, 3681, 0, 1157, 6, 0, 9, 1918, 1810, 0, 1, 0, 296, 0, 108, 410, 0, 1, 5, 1034, 7, 0, 4083, 3, 9, 8, 0, 34, 271, 0, 0, 0, 0, 0, 2683, 3148, 7, 1944, 0, 0, 4089, 0, 0, 0, 15, 10, 885, 0, 0, 3762, 0, 8, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 2336, 0, 3513, 0, 0, 0, 0, 0, 2297, 0, 3547, 6, 0, 0, 2180, 188, 13, 0, 0, 2, 0, 8, 12, 0, 0, 0, 3107, 0, 0, 3672, 0, 0, 0, 0, 3, 1, 0, 0, 1, 0, 0, 631, 1419, 0, 562, 0, 0, 4096, 0, 0, 0, 0, 0, 3754, 10, 0, 1410, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 1409, 0, 3821, 0, 0, 0, 0, 0, 3882, 0, 1897, 0, 2, 6, 701, 625, 0, 4, 0, 257, 1, 210, 136, 0, 0, 0, 4053, 0, 0, 2476, 0, 0, 0, 40, 0, 79, 0, 0, 0, 0, 0, 908, 1859, 5, 2991, 0, 1, 3853, 8, 8, 0, 21, 8, 168, 0, 0, 3325, 0, 3, 0, 11] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 3348, 0, 1476, 0, 0, 0, 0, 0, 2150, 0, 2460, 0, 0, 0, 1130, 101, 4, 0, 0, 1, 0, 1637, 2, 0, 0, 0, 3896, 0, 0, 3865, 0, 0, 0, 0, 7, 13, 0, 0, 0, 0, 0, 882, 1103, 0, 2136, 0, 0, 4096, 0, 0, 0, 0, 0, 3049, 0, 0, 1412, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 17, 0, 3446, 0, 81, 164, 0, 0, 2, 0, 2385, 0, 3703, 1, 0, 5, 419, 120, 0, 4, 0, 0, 0, 2824, 151, 18, 0, 10, 1099, 5, 0, 3967, 1, 0, 0, 0, 49, 1931, 0, 0, 0, 0, 0, 7, 3945, 13, 1073, 0, 0, 3694, 4, 4, 0, 0, 10, 3200, 0, 0, 408, 0, 8, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[9, 0, 24, 1041, 0, 3487, 13, 0, 10, 0, 0, 1451, 0, 25, 0, 0, 0, 1162, 2832, 8, 0, 0, 0, 0, 60, 17, 2, 0, 0, 394, 0, 0, 3202, 0, 6, 0, 23, 1, 5, 12, 0, 0, 0, 0, 3194, 3751, 0, 2524, 1, 0, 4091, 0, 0, 0, 0, 0, 2269, 0, 3, 3150, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 1, 1099, 0, 247, 0, 0, 0, 0, 0, 732, 0, 446, 0, 0, 0, 3818, 1554, 60, 0, 1, 0, 0, 1464, 404, 6, 0, 0, 3973, 0, 0, 3354, 0, 0, 0, 0, 14, 2, 0, 0, 0, 0, 3, 380, 3872, 0, 2111, 0, 0, 4047, 0, 0, 0, 0, 0, 2911, 7, 0, 2262, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3348, 0, 1476, 0, 0, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2150, 0, 2460, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 1130, 101, 4, 0, 0, 1, 0, 519, 0, 0, 0, 0, 0, 0, 0, 1637, 2, 0, 0, 0, 3896, 0, 0, 204, 98, 0, 0, 0, 0, 0, 0, 3865, 0, 0, 0, 0, 7, 13, 0, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 882, 380, 0, 2136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 3049, 0, 0, 986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4926, 4641, 1755, 5837, 4036, 3398, 4140, 4035] avg=4096.0000 max=5837 max_rank=3 min=1755 min_rank=2 imbalance_ratio=1.425049 max_load_reduction=26931 max_load_reduction_pct=0.821869 moved_to_echo=15609 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3827, 0, 619, 0, 0, 288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 737, 0, 1756, 0, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 3737, 161, 66, 0, 0, 0, 0, 570, 0, 0, 0, 0, 0, 0, 0, 2087, 10, 0, 0, 0, 483, 0, 0, 224, 278, 0, 0, 0, 0, 0, 0, 3475, 0, 0, 0, 0, 0, 42, 0, 428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 226, 417, 0, 3513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 124, 0, 0, 0, 0, 0, 0, 0, 2734, 0, 0, 2783, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4734, 2580, 4534, 3082, 3945, 4156, 4220, 5517] avg=4096.0000 max=5517 max_rank=7 min=2580 min_rank=1 imbalance_ratio=1.346924 max_load_reduction=27251 max_load_reduction_pct=0.831635 moved_to_echo=17838 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1034, 0, 328, 0, 0, 272, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3681, 0, 1157, 6, 0, 82, 0, 0, 0, 0, 0, 0, 0, 9, 1918, 1810, 0, 1, 0, 296, 0, 1484, 0, 0, 0, 0, 0, 0, 0, 108, 410, 0, 1, 5, 1034, 7, 0, 582, 262, 0, 0, 0, 0, 0, 0, 4083, 3, 9, 8, 0, 34, 271, 0, 404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2683, 1082, 7, 1944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 15, 10, 117, 0, 0, 0, 0, 0, 0, 0, 885, 0, 0, 2625, 0, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1634, 4927, 5518, 2409, 4812, 5716, 4231, 3521] avg=4096.0000 max=5716 max_rank=5 min=1634 min_rank=0 imbalance_ratio=1.395508 max_load_reduction=27052 max_load_reduction_pct=0.825562 moved_to_echo=18280 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2336, 0, 3513, 0, 0, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2297, 0, 3547, 6, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 2180, 188, 13, 0, 0, 2, 0, 668, 0, 0, 0, 0, 0, 0, 0, 8, 12, 0, 0, 0, 3107, 0, 0, 262, 98, 0, 0, 0, 0, 0, 0, 3672, 0, 0, 0, 0, 3, 1, 0, 151, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 631, 489, 0, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 3754, 10, 0, 984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5951, 5881, 3051, 3487, 3827, 1683, 4140, 4748] avg=4096.0000 max=5951 max_rank=0 min=1683 min_rank=5 imbalance_ratio=1.452881 max_load_reduction=26817 max_load_reduction_pct=0.818390 moved_to_echo=14398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1409, 0, 3821, 0, 0, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3882, 0, 1897, 0, 2, 73, 0, 0, 0, 0, 0, 0, 0, 6, 701, 625, 0, 4, 0, 257, 1, 876, 0, 0, 0, 0, 0, 0, 0, 210, 136, 0, 0, 0, 4053, 0, 0, 344, 231, 0, 0, 0, 0, 0, 0, 2476, 0, 0, 0, 40, 0, 79, 0, 357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 908, 639, 5, 2991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3853, 8, 8, 0, 21, 8, 103, 0, 0, 0, 0, 0, 0, 0, 168, 0, 0, 2321, 0, 3, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5470, 5854, 2470, 4974, 2952, 4543, 4002, 2503] avg=4096.0000 max=5854 max_rank=1 min=2470 min_rank=2 imbalance_ratio=1.429199 max_load_reduction=26914 max_load_reduction_pct=0.821350 moved_to_echo=14000 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[9, 0, 24, 1041, 0, 3487, 13, 0, 228, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 1451, 0, 25, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 1162, 2832, 8, 0, 0, 0, 0, 1768, 0, 0, 0, 0, 0, 0, 0, 60, 17, 2, 0, 0, 394, 0, 0, 694, 219, 0, 0, 0, 0, 0, 0, 3202, 0, 6, 0, 23, 1, 5, 12, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3194, 1289, 0, 2524, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4091, 0, 0, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 2269, 0, 3, 2198, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4802, 1555, 5770, 1386, 3587, 7007, 4190, 4471] avg=4096.0000 max=7007 max_rank=5 min=1386 min_rank=3 imbalance_ratio=1.710693 max_load_reduction=25761 max_load_reduction_pct=0.786163 moved_to_echo=19255 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 1099, 0, 247, 0, 0, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 732, 0, 446, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 3818, 1554, 60, 0, 1, 0, 0, 1825, 0, 0, 0, 0, 0, 0, 0, 1464, 404, 6, 0, 0, 3973, 0, 0, 716, 157, 0, 0, 0, 0, 0, 0, 3354, 0, 0, 0, 0, 14, 2, 0, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 380, 1331, 0, 2111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 2911, 7, 0, 1580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1510, 1227, 7258, 6720, 3613, 3825, 4117, 4498] avg=4096.0000 max=7258 max_rank=2 min=1227 min_rank=1 imbalance_ratio=1.771973 max_load_reduction=25510 max_load_reduction_pct=0.778503 moved_to_echo=16053 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 17, 0, 3446, 0, 81, 164, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2385, 0, 3703, 1, 0, 12, 0, 0, 0, 0, 0, 0, 0, 5, 419, 120, 0, 4, 0, 0, 0, 1864, 0, 0, 0, 0, 0, 0, 0, 2824, 151, 18, 0, 10, 1099, 5, 0, 734, 32, 0, 0, 0, 0, 0, 0, 3967, 1, 0, 0, 0, 49, 1931, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1347, 13, 1073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3694, 4, 4, 0, 0, 10, 16, 0, 0, 0, 0, 0, 0, 0, 3200, 0, 0, 267, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3741, 6103, 2412, 4873, 5996, 2440, 3728, 3475] avg=4096.0000 max=6103 max_rank=1 min=2412 min_rank=2 imbalance_ratio=1.489990 max_load_reduction=26665 max_load_reduction_pct=0.813751 moved_to_echo=15639 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 15104, 7424, 256, 128, 128, 640, 128, 9600, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 0, 128, 8960, 7040, 128, 16896, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 17664, 0, 13696, 256, 0, 1536, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 0, 17408, 0, 15104, 128, 128, 512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[19072, 128, 128, 13824, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8448, 1152, 128, 128, 128, 18048, 128, 0, 3840, 1408, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[28160, 128, 128, 128, 128, 128, 2432, 128, 2176, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 32128, 128, 128, 0, 128, 128, 640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[11, 0, 360, 0, 32, 0, 0, 7, 16, 0, 0, 0, 850, 3180, 0, 43, 0, 0, 0, 0, 0, 159, 37, 2, 4, 0, 1415, 4043, 116, 3950, 0, 4085, 0, 0, 2, 0, 4050, 0, 192, 0, 0, 0, 517, 7, 0, 0, 1560, 0, 0, 14, 65, 0, 0, 2990, 991, 7, 0, 2, 5, 0, 0, 4032, 13, 11] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 483, 0, 0, 6, 0, 0, 0, 0, 0, 0, 1019, 3615, 0, 4, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 1532, 4094, 0, 4088, 0, 4077, 20, 0, 0, 0, 4096, 0, 473, 0, 0, 0, 273, 0, 0, 0, 375, 0, 0, 0, 0, 0, 0, 2921, 1800, 0, 0, 0, 0, 0, 0, 3888, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[51, 0, 12, 0, 8, 0, 0, 4, 14, 0, 0, 4, 1790, 3384, 0, 131, 0, 2, 0, 0, 0, 465, 38, 1535, 0, 0, 263, 4075, 4, 3917, 0, 3897, 119, 0, 12, 0, 4049, 2, 6, 0, 0, 0, 300, 11, 0, 9, 162, 0, 0, 0, 0, 0, 0, 2757, 1652, 4, 2, 0, 7, 0, 0, 4068, 13, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 435, 3967, 0, 26, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3163, 4095, 1, 4057, 0, 4066, 8, 0, 0, 0, 4078, 0, 0, 0, 0, 1, 3, 0, 0, 0, 1861, 24, 0, 0, 2, 2, 0, 3036, 146, 0, 0, 0, 0, 0, 0, 3759, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 396, 0, 1, 54, 0, 0, 0, 0, 0, 0, 767, 4095, 0, 0, 0, 0, 0, 0, 0, 542, 0, 88, 0, 0, 1932, 4096, 0, 3749, 0, 4093, 0, 0, 0, 0, 4096, 0, 126, 0, 0, 0, 200, 0, 0, 0, 405, 1, 0, 0, 1, 0, 0, 3253, 1204, 0, 0, 0, 0, 0, 0, 3669, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[26, 0, 397, 0, 1, 183, 0, 0, 0, 0, 0, 6, 2229, 2180, 0, 1, 1, 0, 0, 0, 0, 11, 0, 0, 0, 0, 804, 4091, 0, 4084, 0, 4086, 21, 0, 0, 0, 4071, 2, 7, 1, 1, 1, 68, 0, 0, 0, 3381, 34, 0, 0, 9, 0, 0, 2966, 189, 0, 2, 0, 0, 0, 0, 3915, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 552, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1980, 3938, 0, 2, 1, 0, 0, 0, 0, 304, 5, 400, 0, 0, 1123, 4082, 0, 4093, 0, 4023, 24, 0, 0, 0, 4096, 0, 8, 0, 0, 0, 20, 0, 0, 0, 1900, 0, 1, 0, 0, 0, 0, 2846, 82, 0, 0, 0, 0, 0, 0, 3285, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 2541, 0, 1, 25, 43, 0, 11, 7, 0, 0, 26, 3778, 0, 129, 0, 10, 0, 21, 1, 10, 0, 3, 0, 34, 1003, 4067, 20, 4035, 0, 3965, 9, 0, 0, 0, 4053, 0, 19, 0, 0, 0, 588, 17, 0, 1, 158, 668, 0, 0, 0, 18, 37, 3948, 679, 0, 0, 0, 5, 0, 0, 2822, 16, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 483, 0, 0, 6, 0, 0, 3032, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 2993, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 3654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1532, 0, 0, 2731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 3987, 0, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 273, 0, 0, 0, 375, 0, 1357, 1045, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2921, 1800, 0, 109, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3888, 0, 0, 417, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3961, 4016, 3658, 4263, 4480, 3220, 4865, 4305] avg=4096.0000 max=4865 max_rank=6 min=3220 min_rank=5 imbalance_ratio=1.187744 max_load_reduction=27903 max_load_reduction_pct=0.851532 moved_to_echo=16870 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[11, 0, 360, 0, 32, 0, 0, 7, 3038, 435, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 850, 2632, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 159, 37, 2, 3608, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1415, 0, 116, 2639, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3942, 0, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 7, 0, 0, 1560, 0, 1311, 1047, 150, 0, 0, 0, 0, 0, 0, 14, 65, 0, 0, 2990, 991, 7, 108, 31, 0, 0, 0, 0, 0, 0, 0, 2, 5, 0, 0, 4032, 13, 11, 367, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3883, 3541, 3806, 4174, 4136, 4592, 4206, 4430] avg=4096.0000 max=4592 max_rank=5 min=3541 min_rank=1 imbalance_ratio=1.121094 max_load_reduction=28176 max_load_reduction_pct=0.859863 moved_to_echo=17364 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 396, 0, 1, 54, 0, 0, 3044, 441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 767, 3389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 0, 88, 3655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1932, 0, 0, 2504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3987, 0, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 0, 0, 0, 405, 1, 1245, 1049, 193, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3253, 1204, 0, 109, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3669, 0, 0, 473, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3936, 4156, 4285, 4436, 4113, 3093, 4607, 4142] avg=4096.0000 max=4607 max_rank=6 min=3093 min_rank=5 imbalance_ratio=1.124756 max_load_reduction=28161 max_load_reduction_pct=0.859406 moved_to_echo=15955 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 37, 0, 0, 0, 0, 0, 3024, 441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 435, 3284, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3163, 0, 1, 2710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 3969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 1861, 24, 1347, 1042, 187, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 3036, 146, 0, 109, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3759, 0, 0, 458, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3502, 3745, 3655, 5874, 3977, 4465, 3333, 4217] avg=4096.0000 max=5874 max_rank=3 min=3333 min_rank=6 imbalance_ratio=1.434082 max_load_reduction=26894 max_load_reduction_pct=0.820740 moved_to_echo=15992 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[26, 0, 397, 0, 1, 183, 0, 0, 3040, 441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2229, 1804, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 11, 0, 0, 3650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 804, 0, 0, 2728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 3963, 2, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 68, 0, 0, 0, 3381, 34, 1356, 1046, 103, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 2966, 189, 0, 108, 21, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3915, 0, 0, 252, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4040, 3662, 3532, 3994, 5990, 3293, 4169] avg=4096.0000 max=5990 max_rank=5 min=3293 min_rank=6 imbalance_ratio=1.462402 max_load_reduction=26778 max_load_reduction_pct=0.817200 moved_to_echo=17446 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 552, 0, 2, 0, 0, 0, 2992, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1980, 3259, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 304, 5, 400, 3642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1123, 0, 0, 2734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 3987, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 1900, 0, 1359, 1031, 186, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2846, 82, 0, 109, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3285, 0, 0, 455, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3987, 5241, 4352, 3857, 4019, 4496, 3076, 3740] avg=4096.0000 max=5241 max_rank=1 min=3076 min_rank=6 imbalance_ratio=1.279541 max_load_reduction=27527 max_load_reduction_pct=0.840057 moved_to_echo=15331 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2541, 0, 1, 25, 43, 0, 2950, 438, 0, 0, 0, 0, 0, 0, 11, 7, 0, 0, 26, 3116, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 21, 1, 10, 0, 3, 3629, 0, 0, 0, 0, 0, 0, 0, 0, 34, 1003, 0, 20, 2691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 3941, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 588, 17, 0, 1, 158, 668, 1344, 1015, 182, 0, 0, 0, 0, 0, 0, 0, 0, 18, 37, 3948, 679, 0, 112, 40, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 2822, 16, 0, 440, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5998, 3289, 3674, 3748, 3969, 3973, 4834, 3283] avg=4096.0000 max=5998 max_rank=0 min=3283 min_rank=7 imbalance_ratio=1.464355 max_load_reduction=26770 max_load_reduction_pct=0.816956 moved_to_echo=16059 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[51, 0, 12, 0, 8, 0, 0, 4, 2899, 439, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4, 1790, 2801, 0, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 465, 38, 1535, 3636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 263, 0, 4, 2617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 0, 12, 0, 3941, 2, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 11, 0, 9, 162, 0, 1300, 998, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2757, 1652, 4, 108, 33, 0, 0, 0, 0, 0, 0, 2, 0, 7, 0, 0, 4068, 13, 1, 391, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3413, 4740, 5676, 2884, 4080, 2939, 4554, 4482] avg=4096.0000 max=5676 max_rank=2 min=2884 min_rank=3 imbalance_ratio=1.385742 max_load_reduction=27092 max_load_reduction_pct=0.826782 moved_to_echo=16055 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 2048, 128, 0, 128, 9856, 768, 10624, 8320, 1408, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 11264, 0, 256, 21376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 4864, 0, 128, 384, 128, 128, 24064, 3584, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 24832, 6784, 128, 896, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 9216, 23296, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 128, 1536, 128, 2048, 29184, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 0, 29440, 128, 128, 3328, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 0, 128, 0, 31744, 128, 896, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 7, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4, 3, 0, 0, 0, 0, 0, 0, 4067, 4095, 4084, 70, 0, 0, 0, 3003, 0, 0, 0, 1064, 3849, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 4096, 0, 0, 0, 213, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[5, 3, 16, 551, 0, 0, 174, 0, 0, 12, 0, 0, 1, 0, 3646, 141, 0, 0, 12, 0, 0, 0, 0, 4085, 0, 3, 0, 0, 0, 1, 4, 10, 0, 0, 16, 3865, 3786, 2719, 0, 0, 22, 0, 4028, 4, 53, 0, 2734, 2842, 0, 0, 0, 4, 0, 0, 0, 27, 0, 1, 3892, 0, 0, 0, 0, 111] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 132, 0, 0, 0, 10, 0, 0, 155, 0, 0, 0, 3934, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 3, 44, 0, 0, 0, 0, 0, 0, 3690, 4096, 3705, 14, 0, 0, 0, 2525, 0, 0, 0, 2620, 3473, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 4095, 0, 0, 0, 75, 42] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 0, 0, 72, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 19, 48, 0, 0, 0, 0, 0, 0, 3393, 4096, 4057, 208, 0, 0, 2, 1813, 0, 1, 0, 1989, 3507, 0, 77, 0, 0, 0, 0, 0, 66, 0, 0, 4072, 0, 0, 0, 1146, 19] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1, 18, 0, 0, 0, 0, 76, 0, 0, 13, 148, 0, 6, 44, 4054, 17, 0, 14, 0, 13, 0, 7, 69, 4079, 0, 2, 0, 0, 0, 2, 83, 0, 0, 0, 9, 3975, 3894, 4001, 0, 0, 15, 0, 456, 5, 0, 11, 3179, 2937, 0, 851, 0, 11, 0, 0, 0, 10, 0, 0, 3929, 1, 0, 0, 833, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4077, 4091, 4039, 1, 0, 0, 0, 3175, 0, 0, 0, 282, 3995, 0, 0, 0, 0, 0, 0, 0, 365, 0, 0, 4095, 0, 0, 0, 433, 21] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 1, 0, 3, 23, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 175, 0, 0, 3992, 0, 0, 0, 0, 1099, 0, 17, 0, 5, 0, 0, 2914, 4086, 4082, 82, 0, 0, 0, 3763, 0, 1, 0, 967, 2346, 0, 10, 0, 0, 0, 1, 28, 26, 0, 0, 4041, 0, 0, 0, 1025, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[3, 7, 8, 1, 0, 0, 28, 0, 0, 10, 227, 0, 2, 4, 4064, 75, 0, 0, 15, 0, 0, 0, 0, 4086, 15, 1, 0, 0, 10, 5, 228, 1, 0, 1, 21, 3634, 3887, 3507, 12, 0, 5, 0, 2864, 13, 2, 4, 1804, 3848, 0, 2, 0, 4, 0, 0, 3, 0, 1, 0, 4078, 1, 0, 0, 287, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 1, 0, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 0, 0, 0, 3376, 474, 504, 18, 16, 0, 0, 0, 0, 0, 0, 4067, 0, 368, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2499, 0, 0, 0, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 3716, 473, 52, 0, 0, 0, 0, 0, 0, 0, 3622, 0, 0, 0, 213, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4045, 4064, 4077, 4395, 4505, 3563, 4284, 3835] avg=4096.0000 max=4505 max_rank=4 min=3563 min_rank=5 imbalance_ratio=1.099854 max_load_reduction=28263 max_load_reduction_pct=0.862518 moved_to_echo=16187 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[5, 3, 16, 551, 0, 0, 174, 0, 3738, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 1, 0, 3631, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 4, 10, 2492, 451, 677, 18, 15, 0, 0, 0, 0, 0, 16, 3865, 0, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 3351, 4, 53, 0, 2734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 27, 2474, 350, 48, 0, 0, 0, 0, 0, 0, 1, 3441, 0, 0, 0, 0, 111, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4487, 3785, 4079, 3671, 4126, 6164, 2903, 3553] avg=4096.0000 max=6164 max_rank=5 min=2903 min_rank=6 imbalance_ratio=1.504883 max_load_reduction=26604 max_load_reduction_pct=0.811890 moved_to_echo=16746 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 0, 0, 72, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 48, 0, 0, 0, 3076, 472, 304, 18, 16, 0, 0, 0, 0, 0, 0, 3393, 0, 365, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1509, 0, 1, 0, 1989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 0, 66, 3692, 431, 52, 0, 0, 0, 0, 0, 0, 0, 3600, 0, 0, 0, 1146, 19, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4122, 4076, 4067, 3953, 3966, 3501, 4318, 4765] avg=4096.0000 max=4765 max_rank=7 min=3501 min_rank=5 imbalance_ratio=1.163330 max_load_reduction=28003 max_load_reduction_pct=0.854584 moved_to_echo=16550 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 1, 0, 3, 23, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 175, 0, 0, 3974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1099, 0, 17, 0, 2058, 468, 632, 18, 16, 0, 0, 0, 5, 0, 0, 2914, 0, 367, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3131, 0, 1, 0, 967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 1, 28, 26, 3715, 288, 51, 0, 0, 0, 0, 0, 0, 0, 3573, 0, 0, 0, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4064, 4149, 4308, 3368, 4099, 4119, 4598] avg=4096.0000 max=4598 max_rank=7 min=3368 min_rank=4 imbalance_ratio=1.122559 max_load_reduction=28170 max_load_reduction_pct=0.859680 moved_to_echo=16184 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 132, 0, 0, 0, 10, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 0, 0, 0, 3918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 44, 0, 0, 0, 3046, 474, 424, 18, 16, 0, 0, 0, 0, 0, 0, 3690, 0, 334, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2101, 0, 0, 0, 2620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 3371, 427, 52, 0, 0, 0, 0, 0, 0, 0, 3621, 0, 0, 0, 75, 42, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4186, 4073, 4077, 4025, 4038, 4721, 3910, 3738] avg=4096.0000 max=4721 max_rank=5 min=3738 min_rank=7 imbalance_ratio=1.152588 max_load_reduction=28047 max_load_reduction_pct=0.855927 moved_to_echo=16407 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 0, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3503, 474, 533, 18, 17, 0, 0, 0, 0, 0, 0, 4077, 0, 364, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2642, 0, 0, 0, 282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 365, 3675, 492, 52, 0, 0, 0, 0, 0, 0, 0, 3621, 0, 0, 0, 433, 21, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4040, 4079, 4078, 4546, 4442, 2924, 4584, 4075] avg=4096.0000 max=4584 max_rank=6 min=2924 min_rank=5 imbalance_ratio=1.119141 max_load_reduction=28184 max_load_reduction_pct=0.860107 moved_to_echo=16025 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3, 7, 8, 1, 0, 0, 28, 0, 3838, 0, 0, 0, 0, 0, 0, 0, 0, 10, 227, 0, 2, 4, 4048, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 15, 1, 0, 0, 10, 5, 228, 1, 3375, 472, 481, 18, 16, 0, 0, 0, 0, 1, 21, 3634, 0, 316, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2383, 13, 2, 4, 1804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4, 0, 0, 3, 0, 3191, 473, 49, 0, 0, 0, 0, 0, 1, 0, 3606, 1, 0, 0, 287, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3885, 4366, 4083, 4622, 3984, 4211, 3722, 3895] avg=4096.0000 max=4622 max_rank=3 min=3722 min_rank=6 imbalance_ratio=1.128418 max_load_reduction=28146 max_load_reduction_pct=0.858948 moved_to_echo=15812 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 18, 0, 0, 0, 0, 76, 0, 3845, 0, 0, 0, 0, 0, 0, 0, 0, 13, 148, 0, 6, 44, 4033, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 13, 0, 7, 69, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 83, 0, 2576, 459, 81, 24, 21, 0, 0, 0, 0, 0, 9, 3975, 0, 355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 375, 5, 0, 11, 3179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 851, 0, 11, 0, 0, 0, 10, 3646, 361, 49, 0, 0, 0, 0, 0, 0, 0, 3470, 1, 0, 0, 833, 5, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3940, 4261, 4158, 3248, 4339, 3585, 4928, 4309] avg=4096.0000 max=4928 max_rank=6 min=3248 min_rank=3 imbalance_ratio=1.203125 max_load_reduction=27840 max_load_reduction_pct=0.849609 moved_to_echo=17161 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33920 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 128, 1280, 128, 384, 128, 23552, 3840, 3712, 256, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 18048, 128, 128, 128, 14720, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 29696, 0, 2816, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 28672, 128, 0, 0, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 1024, 0, 128, 0, 128, 128, 640, 27520, 3328, 512, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 256, 128, 128, 32512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 768, 128, 0, 384, 128, 31744, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 640, 0, 128, 128, 32000, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33920 slack_slots=1792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 3924, 0, 1872, 0, 0, 4096, 2, 3822, 107, 0, 0, 3743, 0, 0, 0, 11, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 2183, 0, 568, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 73, 0, 0, 0, 69, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 4081, 0, 2955, 0, 0, 4096, 0, 4067, 76, 0, 0, 2405, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 1073, 0, 1339, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 363, 0, 0, 0, 13, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1, 3798, 0, 593, 0, 0, 4056, 0, 4081, 514, 0, 0, 3410, 0, 0, 0, 27, 0, 0, 4092, 0, 0, 0, 4088, 0, 0, 241, 6, 257, 0, 0, 0, 0, 5, 0, 4090, 140, 0, 0, 3071, 0, 0, 0, 247, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 4022, 0, 2738, 0, 0, 4095, 0, 4076, 1312, 0, 0, 2594, 0, 0, 0, 7, 0, 0, 4095, 0, 1, 0, 4096, 0, 0, 19, 1, 802, 0, 0, 0, 0, 0, 0, 4094, 191, 0, 0, 556, 0, 0, 0, 53, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 3623, 17, 1988, 39, 0, 4078, 0, 3726, 11, 0, 0, 2527, 0, 36, 0, 0, 19, 0, 4081, 0, 0, 0, 3797, 0, 0, 629, 0, 2194, 10, 0, 10, 0, 0, 0, 4075, 352, 25, 4, 111, 0, 0, 0, 1327, 0, 0, 8, 0, 0, 5, 0, 47, 0, 10, 0, 0, 0, 0, 5, 0, 6, 0, 8, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 2552, 0, 333, 0, 0, 4096, 0, 3993, 616, 0, 0, 2866, 0, 0, 0, 77, 0, 0, 4058, 0, 0, 0, 4096, 0, 0, 1948, 0, 425, 0, 0, 0, 0, 0, 0, 3965, 8, 0, 0, 3686, 0, 0, 0, 2, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 4057, 17, 2290, 48, 0, 4068, 0, 3504, 31, 74, 0, 307, 0, 25, 0, 14, 0, 0, 4079, 0, 0, 0, 4044, 0, 0, 200, 0, 2026, 32, 0, 14, 0, 0, 0, 4021, 16, 18, 17, 2689, 0, 10, 0, 1149, 0, 0, 5, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[7, 3920, 1, 3379, 39, 0, 4075, 0, 3509, 505, 0, 0, 3350, 8, 50, 0, 7, 5, 1, 4072, 0, 0, 0, 4052, 1, 0, 517, 0, 480, 14, 0, 15, 0, 0, 0, 4074, 2, 37, 10, 109, 5, 0, 0, 336, 148, 0, 8, 0, 7, 0, 0, 0, 0, 13, 0, 0, 0, 2, 5, 0, 0, 5, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2155, 0, 1872, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 107, 0, 0, 3743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 31, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2183, 0, 568, 0, 0, 0, 1413, 855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2681, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 2780, 914, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 4056, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4029, 4870, 4136, 5019, 2754, 3763, 4092, 4105] avg=4096.0000 max=5019 max_rank=3 min=2754 min_rank=4 imbalance_ratio=1.225342 max_load_reduction=27749 max_load_reduction_pct=0.846832 moved_to_echo=14714 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1, 2085, 0, 593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1088, 514, 0, 0, 3410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 31, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 241, 6, 257, 0, 0, 0, 1412, 828, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2678, 140, 0, 0, 3071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 0, 0, 0, 0, 2969, 885, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 43, 4052, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2679, 5012, 4146, 2744, 5894, 4101, 4127, 4065] avg=4096.0000 max=5894 max_rank=4 min=2679 min_rank=0 imbalance_ratio=1.438965 max_load_reduction=26874 max_load_reduction_pct=0.820129 moved_to_echo=18187 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2208, 0, 2738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1087, 1312, 0, 0, 2594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 31, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 1, 802, 0, 0, 0, 1413, 877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2681, 191, 0, 0, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 1, 0, 0, 0, 2965, 937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4055, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4946, 4993, 4135, 3112, 3428, 3956, 4094, 4104] avg=4096.0000 max=4993 max_rank=1 min=3112 min_rank=3 imbalance_ratio=1.218994 max_load_reduction=27775 max_load_reduction_pct=0.847626 moved_to_echo=15582 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2240, 0, 2955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1084, 76, 0, 0, 2405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1073, 0, 1339, 0, 0, 0, 1414, 890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2682, 2, 0, 0, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 3, 0, 0, 0, 2959, 951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4056, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5195, 3565, 4127, 4716, 3047, 3926, 4087, 4105] avg=4096.0000 max=5195 max_rank=0 min=3047 min_rank=4 imbalance_ratio=1.268311 max_load_reduction=27573 max_load_reduction_pct=0.841461 moved_to_echo=15165 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1989, 17, 1988, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 993, 11, 0, 0, 2527, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 31, 0, 0, 0, 3797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 629, 0, 2194, 10, 0, 10, 1407, 790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2668, 352, 25, 4, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1327, 0, 0, 8, 0, 2711, 844, 0, 0, 0, 0, 0, 0, 0, 5, 0, 47, 0, 10, 0, 0, 4041, 22, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 6, 0, 8, 0, 4078, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4033, 3567, 3847, 5040, 3160, 4890, 4125, 4106] avg=4096.0000 max=5040 max_rank=3 min=3160 min_rank=4 imbalance_ratio=1.230469 max_load_reduction=27728 max_load_reduction_pct=0.846191 moved_to_echo=16281 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1401, 0, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1065, 616, 0, 0, 2866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 30, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1948, 0, 425, 0, 0, 0, 1369, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2596, 8, 0, 0, 3686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2905, 595, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 24, 4019, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1734, 4547, 4203, 4298, 6290, 3502, 4089, 4105] avg=4096.0000 max=6290 max_rank=4 min=1734 min_rank=0 imbalance_ratio=1.535645 max_load_reduction=26478 max_load_reduction_pct=0.808044 moved_to_echo=17986 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[7, 2152, 1, 3379, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 936, 505, 0, 0, 3350, 8, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 5, 1, 31, 0, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 517, 0, 480, 14, 0, 15, 1406, 854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2668, 2, 37, 10, 109, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 336, 148, 0, 8, 0, 2553, 914, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 13, 0, 0, 4032, 20, 0, 0, 0, 0, 0, 0, 0, 2, 5, 0, 0, 5, 0, 0, 4075, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5578, 4849, 4096, 3287, 2826, 3964, 4072, 4096] avg=4096.0000 max=5578 max_rank=0 min=2826 min_rank=4 imbalance_ratio=1.361816 max_load_reduction=27190 max_load_reduction_pct=0.829773 moved_to_echo=14958 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2219, 17, 2290, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 928, 31, 74, 0, 307, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 20, 0, 0, 0, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 0, 2026, 32, 0, 14, 1392, 888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2629, 16, 18, 17, 2689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 1149, 0, 0, 5, 0, 2552, 950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 4045, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 14, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4574, 1365, 4078, 4552, 5369, 4666, 4082, 4082] avg=4096.0000 max=5369 max_rank=4 min=1365 min_rank=1 imbalance_ratio=1.310791 max_load_reduction=27399 max_load_reduction_pct=0.836151 moved_to_echo=18199 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 0, 6912, 128, 8192, 128, 0, 128, 11264, 6656, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 16512, 128, 16256, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 3200, 256, 0, 128, 0, 22400, 7040, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 0, 21376, 768, 128, 128, 10752, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 3200, 128, 0, 21248, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 128, 128, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 0, 128, 0, 128, 32384, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 128, 128, 256, 0, 128, 0, 32384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 310, 0, 0, 3244, 0, 0, 74, 4008, 0, 836, 0, 0, 0, 1, 4092, 0, 0, 564, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4095, 0, 0, 0, 0, 4092, 0, 4091, 0, 0, 0, 0, 3271, 4081, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 228, 0, 0, 2052, 0, 0, 631, 3359, 0, 1402, 0, 0, 0, 0, 4095, 0, 0, 1969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 5, 0, 4096, 0, 0, 0, 0, 4092, 0, 4096, 0, 0, 0, 0, 2620, 4081, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 35, 0, 0, 0, 83, 0, 0, 481, 0, 0, 1623, 4059, 0, 1500, 0, 0, 0, 13, 4075, 0, 0, 1689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 519, 0, 4092, 0, 0, 0, 0, 4085, 0, 4072, 0, 0, 0, 0, 2344, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1005, 0, 3, 2135, 0, 0, 876, 3275, 0, 593, 3, 0, 0, 0, 4088, 0, 0, 1319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 4031, 0, 4093, 0, 0, 0, 0, 3150, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[17, 0, 10, 0, 0, 0, 0, 77, 0, 0, 1901, 0, 0, 28, 0, 0, 1927, 2103, 10, 933, 0, 23, 0, 0, 4028, 0, 0, 1458, 0, 0, 0, 0, 0, 10, 0, 45, 8, 0, 6, 49, 316, 48, 0, 3953, 0, 0, 7, 0, 4075, 0, 4070, 4, 18, 3, 0, 3643, 3940, 0, 0, 4, 10, 18, 16, 10] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1073, 4, 0, 1, 0, 37, 0, 22, 266, 0, 0, 1777, 2776, 0, 2587, 0, 0, 0, 14, 4077, 0, 0, 175, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 3700, 0, 4062, 0, 0, 0, 0, 3993, 4078, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 5, 0, 20, 0, 0, 0, 0, 0, 9, 178, 0, 37, 0, 1, 476, 2756, 12, 746, 0, 0, 0, 0, 4020, 31, 0, 4057, 5, 0, 0, 0, 95, 25, 6, 111, 3, 0, 5, 0, 0, 38, 0, 4055, 0, 0, 0, 0, 4066, 0, 4077, 28, 24, 18, 0, 3836, 3939, 0, 22, 4, 0, 29, 17, 17] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 40, 1, 0, 2, 2, 0, 52, 0, 0, 2459, 0, 0, 1720, 3954, 10, 2132, 0, 0, 0, 0, 4066, 35, 5, 154, 8, 0, 0, 5, 26, 65, 0, 125, 11, 0, 4, 0, 14, 362, 0, 4002, 0, 0, 14, 0, 4076, 0, 4080, 3, 16, 3, 0, 1248, 4008, 0, 0, 1, 9, 19, 16, 17] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 3006, 1111, 63, 0, 0, 0, 0, 0, 0, 0, 310, 0, 0, 3244, 0, 0, 1414, 938, 105, 0, 0, 0, 0, 0, 74, 1959, 0, 836, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2678, 0, 0, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4028, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1086, 0, 0, 0, 0, 0, 0, 3271, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4180, 6011, 2870, 3242, 4034, 3963, 4357, 4111] avg=4096.0000 max=6011 max_rank=1 min=2870 min_rank=2 imbalance_ratio=1.467529 max_load_reduction=26757 max_load_reduction_pct=0.816559 moved_to_echo=16465 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 35, 0, 3001, 1125, 63, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 481, 0, 0, 1408, 949, 105, 0, 0, 0, 0, 0, 1623, 1985, 0, 1500, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 2667, 0, 0, 1689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 0, 0, 0, 0, 0, 0, 0, 1, 519, 0, 3957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1084, 0, 0, 0, 0, 0, 0, 2344, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4225, 3026, 5121, 4356, 4009, 4477, 3428, 4126] avg=4096.0000 max=5121 max_rank=2 min=3026 min_rank=1 imbalance_ratio=1.250244 max_load_reduction=27647 max_load_reduction_pct=0.843719 moved_to_echo=16040 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 16, 0, 0, 3006, 931, 63, 0, 0, 0, 0, 0, 0, 0, 228, 0, 0, 2052, 0, 0, 1415, 786, 105, 0, 0, 0, 0, 0, 631, 1642, 0, 1402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2680, 0, 0, 1969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 0, 0, 0, 0, 0, 0, 26, 5, 0, 3961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1086, 0, 0, 0, 0, 0, 0, 2620, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4016, 4586, 3675, 4649, 4033, 3992, 3706, 4111] avg=4096.0000 max=4649 max_rank=3 min=3675 min_rank=2 imbalance_ratio=1.135010 max_load_reduction=28119 max_load_reduction_pct=0.858124 moved_to_echo=15842 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 2961, 907, 63, 0, 0, 0, 0, 0, 0, 0, 1005, 0, 3, 2135, 0, 0, 1412, 766, 105, 0, 0, 0, 0, 0, 876, 1602, 0, 593, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2676, 0, 0, 1319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1070, 0, 0, 0, 0, 0, 0, 3150, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3932, 5426, 3074, 3995, 4030, 3965, 4220, 4126] avg=4096.0000 max=5426 max_rank=1 min=3074 min_rank=2 imbalance_ratio=1.324707 max_load_reduction=27342 max_load_reduction_pct=0.834412 moved_to_echo=16341 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1073, 4, 0, 2718, 769, 63, 0, 0, 0, 0, 0, 1, 0, 37, 0, 22, 266, 0, 0, 1409, 649, 105, 0, 0, 0, 0, 0, 1777, 1358, 0, 2587, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 2668, 0, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 3999, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 982, 0, 0, 0, 0, 0, 0, 3993, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4627, 2489, 5736, 2843, 4029, 3961, 4975, 4108] avg=4096.0000 max=5736 max_rank=2 min=2489 min_rank=1 imbalance_ratio=1.400391 max_load_reduction=27032 max_load_reduction_pct=0.824951 moved_to_echo=17073 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[17, 0, 10, 0, 0, 0, 0, 77, 2994, 582, 62, 0, 0, 0, 0, 0, 0, 0, 1901, 0, 0, 28, 0, 0, 1392, 492, 101, 0, 0, 0, 0, 0, 1927, 1029, 10, 933, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2636, 0, 0, 1458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 45, 8, 0, 6, 49, 4008, 0, 0, 0, 0, 0, 0, 0, 316, 48, 0, 3823, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1081, 0, 0, 4, 18, 3, 0, 3643, 0, 0, 0, 0, 0, 0, 0, 0, 3940, 0, 0, 4, 10, 18, 16, 10, 29, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3742, 3914, 3922, 4094, 4126, 4194, 4749, 4027] avg=4096.0000 max=4749 max_rank=6 min=3742 min_rank=0 imbalance_ratio=1.159424 max_load_reduction=28019 max_load_reduction_pct=0.855072 moved_to_echo=17096 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 40, 1, 0, 2, 2994, 1096, 63, 0, 0, 0, 0, 0, 2, 0, 52, 0, 0, 2459, 0, 0, 1405, 925, 102, 0, 0, 0, 0, 0, 1720, 1933, 10, 2132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2661, 35, 5, 154, 8, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 26, 65, 0, 125, 11, 0, 4, 0, 4017, 0, 0, 0, 0, 0, 0, 0, 14, 362, 0, 3871, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1082, 0, 0, 3, 16, 3, 0, 1248, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 0, 0, 1, 9, 19, 16, 17, 29, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4200, 4945, 5795, 2868, 4248, 4261, 2352, 4099] avg=4096.0000 max=5795 max_rank=2 min=2352 min_rank=6 imbalance_ratio=1.414795 max_load_reduction=26973 max_load_reduction_pct=0.823151 moved_to_echo=14960 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 5, 0, 20, 0, 0, 0, 2991, 767, 63, 0, 0, 0, 0, 0, 0, 0, 9, 178, 0, 37, 0, 1, 1392, 648, 106, 0, 0, 0, 0, 0, 476, 1341, 12, 746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2628, 31, 0, 4057, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 25, 6, 111, 3, 0, 5, 0, 4014, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 3917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1075, 0, 0, 28, 24, 18, 0, 3836, 0, 0, 0, 0, 0, 0, 0, 0, 3939, 0, 22, 4, 0, 29, 17, 17, 32, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3846, 2371, 2575, 6721, 4259, 3955, 4981, 4060] avg=4096.0000 max=6721 max_rank=3 min=2371 min_rank=1 imbalance_ratio=1.640869 max_load_reduction=26047 max_load_reduction_pct=0.794891 moved_to_echo=17255 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 0, 128, 1152, 128, 128, 23680, 7296, 512, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 0, 3712, 256, 128, 10752, 0, 128, 11264, 6272, 896, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8576, 0, 0, 128, 128, 128, 0, 24192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[9216, 12928, 128, 10752, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[21376, 128, 128, 11392, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[384, 1024, 0, 31488, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[256, 128, 128, 384, 128, 0, 128, 128, 32256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[32384, 0, 128, 128, 128, 128, 128, 128, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 1597, 45, 0, 0, 723, 28, 15, 4010, 0, 0, 2670, 0, 48, 0, 0, 0, 1, 0, 916, 11, 0, 88, 8, 4048, 3255, 874, 0, 0, 1, 35, 2445, 7, 0, 8, 0, 0, 0, 0, 0, 0, 15, 14, 21, 3929, 0, 23, 18, 4017, 0, 0, 701, 0, 0, 1603, 0, 0, 1479, 0, 0, 15, 15, 5, 80] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 1521, 0, 5, 0, 1545, 0, 0, 3965, 0, 0, 2959, 0, 0, 0, 0, 0, 34, 0, 49, 0, 0, 0, 0, 4029, 3839, 930, 0, 0, 0, 0, 3701, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 4046, 0, 0, 0, 4096, 0, 0, 218, 0, 0, 1736, 0, 0, 26, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 697, 0, 0, 0, 2597, 0, 0, 4088, 3, 0, 2152, 0, 0, 0, 0, 0, 12, 0, 245, 0, 0, 0, 0, 4096, 4062, 2397, 0, 0, 0, 0, 3206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4, 66, 0, 0, 873, 0, 0, 78, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 295, 0, 0, 0, 455, 0, 0, 4079, 0, 0, 2140, 0, 0, 0, 0, 0, 0, 0, 47, 0, 10, 0, 0, 4083, 3323, 2106, 0, 0, 0, 0, 2777, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 89, 0, 0, 3150, 0, 0, 1980, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[4, 1047, 0, 0, 0, 2327, 0, 21, 3895, 59, 0, 288, 0, 1, 0, 4, 0, 0, 0, 108, 0, 0, 20, 7, 3906, 3368, 3194, 0, 0, 4, 0, 2156, 0, 0, 0, 18, 0, 0, 0, 258, 0, 29, 25, 26, 4063, 0, 5, 7, 4055, 0, 357, 0, 0, 0, 2710, 0, 14, 599, 0, 0, 68, 25, 55, 45] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 31, 0, 0, 0, 1819, 0, 0, 4001, 18, 0, 2544, 0, 0, 0, 0, 0, 1, 0, 28, 0, 0, 0, 0, 4095, 3032, 1726, 0, 0, 0, 0, 3577, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4090, 0, 0, 0, 4096, 0, 1, 562, 0, 0, 1815, 0, 0, 1325, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 92, 0, 0, 519, 0, 21, 3700, 0, 0, 982, 0, 32, 0, 24, 2, 0, 0, 34, 0, 0, 1, 3, 3834, 3952, 3524, 0, 0, 0, 14, 3521, 6, 0, 27, 1, 0, 0, 0, 0, 0, 16, 19, 19, 3801, 0, 0, 6, 4064, 0, 521, 260, 1, 0, 3629, 0, 0, 34, 21, 0, 0, 14, 73, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 96, 4, 21, 0, 194, 0, 0, 3753, 0, 0, 3330, 0, 0, 1, 0, 0, 86, 0, 302, 0, 0, 0, 0, 3965, 3984, 1392, 0, 0, 0, 0, 4001, 0, 0, 0, 0, 0, 0, 0, 444, 0, 0, 0, 0, 3156, 0, 0, 0, 4096, 0, 0, 597, 0, 0, 2411, 0, 0, 935, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 697, 0, 0, 0, 2597, 0, 0, 1231, 955, 0, 0, 0, 0, 0, 0, 2014, 3, 0, 2152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 245, 0, 0, 0, 0, 4062, 221, 22, 0, 0, 0, 0, 0, 0, 0, 2397, 0, 0, 0, 0, 2094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 157, 0, 0, 0, 0, 0, 0, 0, 1441, 0, 4, 66, 0, 0, 873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 0, 0, 2434, 843, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5480, 4169, 4562, 4491, 4074, 4253, 2384, 3355] avg=4096.0000 max=5480 max_rank=0 min=2384 min_rank=6 imbalance_ratio=1.337891 max_load_reduction=27288 max_load_reduction_pct=0.832764 moved_to_echo=14066 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 295, 0, 0, 0, 455, 0, 0, 1228, 827, 0, 0, 0, 0, 0, 0, 2010, 0, 0, 2140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 10, 0, 0, 3323, 221, 22, 0, 0, 0, 0, 0, 0, 0, 2106, 0, 0, 0, 0, 1814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 1441, 0, 0, 89, 0, 0, 3150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1980, 0, 0, 0, 0, 0, 0, 2434, 841, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2805, 4150, 3623, 3920, 4104, 4231, 4680, 5255] avg=4096.0000 max=5255 max_rank=7 min=2805 min_rank=0 imbalance_ratio=1.282959 max_load_reduction=27513 max_load_reduction_pct=0.839630 moved_to_echo=18270 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1597, 45, 0, 0, 723, 28, 15, 1207, 728, 0, 0, 0, 0, 0, 0, 1977, 0, 0, 2670, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 916, 11, 0, 88, 8, 3255, 217, 23, 0, 0, 0, 0, 0, 0, 0, 874, 0, 0, 1, 35, 1597, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 8, 0, 0, 0, 0, 0, 4025, 0, 0, 0, 0, 0, 0, 0, 0, 15, 14, 21, 3929, 0, 23, 18, 120, 0, 0, 0, 0, 0, 0, 0, 1413, 0, 0, 701, 0, 0, 1603, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1479, 0, 0, 15, 15, 5, 80, 2387, 826, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4343, 4695, 4519, 2507, 4040, 4140, 3717, 4807] avg=4096.0000 max=4807 max_rank=7 min=2507 min_rank=3 imbalance_ratio=1.173584 max_load_reduction=27961 max_load_reduction_pct=0.853302 moved_to_echo=16704 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1521, 0, 5, 0, 1545, 0, 0, 1194, 1102, 0, 0, 0, 0, 0, 0, 1954, 0, 0, 2959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 49, 0, 0, 0, 0, 3839, 221, 22, 0, 0, 0, 0, 0, 0, 0, 930, 0, 0, 0, 0, 2417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 4007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 0, 0, 0, 182, 0, 0, 0, 0, 0, 0, 0, 1441, 0, 0, 218, 0, 0, 1736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 2434, 817, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5367, 4913, 4165, 3347, 4076, 4228, 3395, 3277] avg=4096.0000 max=5367 max_rank=0 min=3277 min_rank=7 imbalance_ratio=1.310303 max_load_reduction=27401 max_load_reduction_pct=0.836212 moved_to_echo=14976 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 92, 0, 0, 519, 0, 21, 1114, 1049, 0, 0, 0, 0, 0, 0, 1823, 0, 0, 982, 0, 32, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 34, 0, 0, 1, 3, 3952, 219, 21, 0, 0, 0, 0, 0, 0, 0, 3524, 0, 0, 0, 14, 2299, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 27, 1, 0, 0, 0, 0, 3813, 0, 0, 0, 0, 0, 0, 0, 0, 16, 19, 19, 3801, 0, 0, 6, 173, 0, 0, 0, 0, 0, 0, 0, 1430, 0, 521, 260, 1, 0, 3629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 21, 0, 0, 14, 73, 0, 2415, 763, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2796, 2861, 4232, 5837, 3847, 4034, 5841, 3320] avg=4096.0000 max=5841 max_rank=6 min=2796 min_rank=0 imbalance_ratio=1.426025 max_load_reduction=26927 max_load_reduction_pct=0.821747 moved_to_echo=17042 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 31, 0, 0, 0, 1819, 0, 0, 1204, 1065, 0, 0, 0, 0, 0, 0, 1972, 18, 0, 2544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 28, 0, 0, 0, 0, 3032, 221, 22, 0, 0, 0, 0, 0, 0, 0, 1726, 0, 0, 0, 0, 2336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 176, 0, 0, 0, 0, 0, 0, 0, 1441, 0, 1, 562, 0, 0, 1815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1325, 0, 0, 0, 0, 0, 0, 2434, 825, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4534, 3304, 4062, 4080, 4266, 3819, 4584] avg=4096.0000 max=4584 max_rank=7 min=3304 min_rank=2 imbalance_ratio=1.119141 max_load_reduction=28184 max_load_reduction_pct=0.860107 moved_to_echo=16749 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 96, 4, 21, 0, 194, 0, 0, 1130, 1192, 0, 0, 0, 0, 0, 0, 1850, 0, 0, 3330, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 302, 0, 0, 0, 0, 3984, 221, 22, 0, 0, 0, 0, 0, 0, 0, 1392, 0, 0, 0, 0, 2613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 444, 3943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3156, 0, 0, 0, 196, 0, 0, 0, 0, 0, 0, 0, 1441, 0, 0, 597, 0, 0, 2411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 935, 0, 0, 0, 0, 0, 0, 2434, 773, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2637, 5181, 4615, 4005, 4387, 3352, 4449, 4142] avg=4096.0000 max=5181 max_rank=1 min=2637 min_rank=0 imbalance_ratio=1.264893 max_load_reduction=27587 max_load_reduction_pct=0.841888 moved_to_echo=16330 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4, 1047, 0, 0, 0, 2327, 0, 21, 1176, 646, 0, 0, 0, 0, 0, 0, 1913, 59, 0, 288, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 0, 0, 20, 7, 3368, 223, 22, 0, 0, 0, 0, 0, 0, 0, 3194, 0, 0, 4, 0, 1401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 258, 3884, 0, 0, 0, 0, 0, 0, 0, 0, 29, 25, 26, 4063, 0, 5, 7, 109, 0, 0, 0, 0, 0, 0, 0, 1416, 0, 357, 0, 0, 0, 2710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 599, 0, 0, 68, 25, 55, 45, 2416, 806, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5221, 2265, 3748, 4599, 4160, 4264, 4483, 4028] avg=4096.0000 max=5221 max_rank=0 min=2265 min_rank=1 imbalance_ratio=1.274658 max_load_reduction=27547 max_load_reduction_pct=0.840668 moved_to_echo=16935 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 5376, 256, 128, 0, 10240, 128, 128, 9600, 7680, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 6528, 128, 0, 128, 128, 256, 128, 19456, 6528, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[11520, 0, 896, 2560, 128, 0, 18048, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 16256, 0, 0, 128, 128, 16640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[15616, 128, 0, 17152, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 256, 0, 1792, 128, 128, 128, 128, 28928, 1792, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 31360, 0, 128, 128, 1280, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 0, 0, 0, 896, 32000, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[38, 0, 0, 0, 4078, 0, 4096, 3232, 0, 0, 4096, 0, 3396, 0, 0, 0, 0, 3030, 0, 0, 4089, 2260, 0, 0, 0, 0, 0, 0, 0, 0, 120, 0, 647, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 4096, 1435, 0, 0, 4096, 0, 3671, 0, 1, 0, 0, 528, 0, 0, 4055, 499, 0, 0, 37, 0, 0, 0, 0, 0, 3956, 0, 342, 2046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3908, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 1, 4096, 3448, 0, 0, 4096, 0, 4071, 0, 0, 0, 0, 245, 0, 0, 4096, 189, 0, 0, 11, 0, 0, 0, 0, 0, 3452, 0, 11, 854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 14, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3995, 0, 4044, 317, 0, 0, 4066, 8, 2273, 16, 0, 0, 0, 2909, 0, 0, 4080, 2227, 0, 0, 199, 0, 212, 0, 71, 16, 795, 29, 367, 2845, 2, 0, 3, 0, 0, 84, 0, 0, 0, 3892, 0, 79, 20, 97, 0, 17, 22, 0, 21, 0, 0, 2, 0, 19, 8, 0, 0, 0, 33, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3831, 0, 4068, 2191, 0, 0, 4058, 26, 1853, 25, 0, 0, 0, 971, 27, 0, 4074, 3479, 0, 0, 2170, 0, 0, 0, 0, 14, 1685, 0, 6, 30, 1, 0, 28, 0, 0, 36, 0, 0, 0, 4049, 0, 21, 34, 5, 0, 15, 35, 0, 29, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 4096, 1478, 0, 0, 4096, 0, 3282, 0, 0, 0, 0, 2409, 0, 0, 4096, 1119, 0, 0, 113, 0, 0, 0, 0, 0, 3805, 0, 0, 427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3749, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 4006, 0, 4084, 837, 0, 0, 4096, 0, 3000, 0, 0, 0, 0, 3807, 0, 0, 4039, 465, 0, 0, 4, 0, 14, 0, 39, 0, 3350, 0, 92, 928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3904, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 97, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3955, 0, 4021, 1244, 0, 0, 4035, 20, 3082, 18, 0, 0, 0, 2333, 13, 0, 4078, 1953, 0, 0, 0, 0, 72, 0, 3, 12, 1015, 39, 765, 1552, 4, 9, 0, 0, 0, 61, 0, 0, 0, 3872, 0, 27, 23, 44, 0, 15, 26, 0, 22, 0, 28, 407, 5, 4, 2, 0, 0, 0, 9, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2365, 1, 0, 3448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, 0, 541, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 3452, 0, 1469, 0, 0, 0, 0, 0, 0, 0, 11, 854, 0, 0, 0, 0, 0, 0, 2613, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3555, 474, 5, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 14, 0, 4091, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5814, 5080, 975, 4932, 3541, 4286, 4034, 4106] avg=4096.0000 max=5814 max_rank=0 min=975 min_rank=2 imbalance_ratio=1.419434 max_load_reduction=26954 max_load_reduction_pct=0.822571 moved_to_echo=15967 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[38, 0, 0, 0, 2353, 0, 0, 3232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 0, 3396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3030, 0, 0, 540, 2260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 0, 1463, 0, 0, 0, 0, 0, 0, 0, 647, 297, 0, 0, 0, 0, 0, 0, 2613, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3384, 0, 0, 0, 0, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3549, 474, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4091, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5623, 4405, 5830, 1583, 3620, 3583, 4028, 4096] avg=4096.0000 max=5830 max_rank=2 min=1583 min_rank=3 imbalance_ratio=1.423340 max_load_reduction=26938 max_load_reduction_pct=0.822083 moved_to_echo=15327 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2306, 0, 0, 317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 8, 2273, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2909, 0, 0, 539, 2227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 199, 0, 212, 0, 71, 16, 795, 29, 1433, 0, 0, 0, 0, 0, 0, 0, 367, 2845, 2, 0, 3, 0, 0, 84, 2594, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3892, 0, 79, 20, 97, 194, 0, 0, 0, 0, 0, 0, 0, 0, 17, 22, 0, 21, 0, 0, 2, 3541, 471, 4, 0, 0, 0, 0, 0, 0, 19, 8, 0, 0, 0, 33, 0, 4040, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2623, 3298, 5675, 2755, 5957, 4282, 4078, 4100] avg=4096.0000 max=5957 max_rank=4 min=2623 min_rank=0 imbalance_ratio=1.454346 max_load_reduction=26811 max_load_reduction_pct=0.818207 moved_to_echo=18417 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2365, 0, 0, 1435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 0, 3671, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 0, 0, 536, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 3956, 0, 1469, 0, 0, 0, 0, 0, 0, 0, 342, 2046, 0, 0, 0, 0, 0, 0, 2613, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3908, 0, 2, 0, 0, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3519, 474, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3800, 4681, 1563, 5462, 5064, 4109, 3998, 4091] avg=4096.0000 max=5462 max_rank=3 min=1563 min_rank=2 imbalance_ratio=1.333496 max_load_reduction=27306 max_load_reduction_pct=0.833313 moved_to_echo=17262 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 2312, 0, 0, 837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 0, 3000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3807, 0, 0, 534, 465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 14, 0, 39, 0, 3350, 0, 1437, 0, 0, 0, 0, 0, 0, 0, 92, 928, 0, 0, 0, 0, 0, 0, 2613, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3904, 0, 0, 0, 1, 195, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3505, 474, 6, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 97, 0, 4078, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3151, 4009, 4806, 4844, 3695, 4100, 3986, 4177] avg=4096.0000 max=4844 max_rank=3 min=3151 min_rank=0 imbalance_ratio=1.182617 max_load_reduction=27924 max_load_reduction_pct=0.852173 moved_to_echo=15958 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2283, 0, 0, 1244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 994, 20, 3082, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2333, 13, 0, 539, 1953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 3, 12, 1015, 39, 1418, 0, 0, 0, 0, 0, 0, 0, 765, 1552, 4, 9, 0, 0, 0, 61, 2574, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3872, 0, 27, 23, 44, 193, 0, 0, 0, 0, 0, 0, 0, 0, 15, 26, 0, 22, 0, 28, 407, 3539, 467, 5, 0, 0, 0, 0, 0, 5, 4, 2, 0, 0, 0, 9, 0, 4016, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3527, 4114, 4838, 2559, 5026, 4159, 4509, 4036] avg=4096.0000 max=5026 max_rank=4 min=2559 min_rank=3 imbalance_ratio=1.227051 max_load_reduction=27742 max_load_reduction_pct=0.846619 moved_to_echo=17730 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2364, 0, 0, 1478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 0, 3282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2409, 0, 0, 541, 1119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 0, 0, 0, 0, 0, 3805, 0, 1468, 0, 0, 0, 0, 0, 0, 0, 0, 427, 0, 0, 0, 0, 0, 0, 2613, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3749, 0, 0, 0, 1, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3555, 474, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4091, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3842, 4291, 4069, 5386, 3103, 3949, 4036, 4092] avg=4096.0000 max=5386 max_rank=3 min=3103 min_rank=4 imbalance_ratio=1.314941 max_load_reduction=27382 max_load_reduction_pct=0.835632 moved_to_echo=15180 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2197, 0, 0, 2191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 986, 26, 1853, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 971, 27, 0, 535, 3479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2170, 0, 0, 0, 0, 14, 1685, 0, 1378, 0, 0, 0, 0, 0, 0, 0, 6, 30, 1, 0, 28, 0, 0, 36, 2596, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 21, 34, 5, 191, 0, 0, 0, 0, 0, 0, 0, 0, 15, 35, 0, 29, 0, 0, 0, 3539, 476, 5, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4388, 2890, 5012, 5247, 2762, 4300, 4099, 4070] avg=4096.0000 max=5247 max_rank=3 min=2762 min_rank=4 imbalance_ratio=1.281006 max_load_reduction=27521 max_load_reduction_pct=0.839874 moved_to_echo=15231 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 0, 18560, 128, 0, 14208, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 16256, 128, 0, 4352, 12288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[2560, 0, 384, 0, 128, 128, 18304, 128, 11648, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 8064, 128, 24704, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 128, 0, 128, 512, 28416, 3840, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 0, 30848, 0, 256, 128, 256, 1664, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 0, 0, 256, 0, 32640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[2304, 9088, 128, 128, 128, 0, 0, 256, 20864, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[31, 5, 0, 1701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 4001, 0, 0, 0, 0, 2213, 0, 0, 6, 0, 0, 0, 0, 3681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 623, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 24, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[33, 602, 0, 1689, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 189, 3984, 0, 0, 0, 0, 430, 0, 0, 14, 0, 7, 0, 0, 2803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3004, 0, 3533, 0, 0, 4096, 0, 0, 0, 0, 10, 0, 0, 0, 4081, 0, 0, 4096, 0, 65, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[40, 950, 0, 3666, 0, 0, 0, 0, 0, 227, 0, 0, 0, 2, 0, 0, 0, 4095, 0, 48, 3962, 0, 0, 0, 0, 3043, 0, 0, 24, 0, 0, 0, 0, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2637, 3, 1697, 0, 0, 4038, 0, 5, 0, 0, 0, 0, 0, 0, 4074, 4, 0, 4082, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 2504, 18, 3661, 2, 51, 0, 3, 21, 1, 9, 2, 3, 0, 0, 27, 0, 4013, 36, 0, 4053, 0, 158, 0, 3, 1897, 0, 19, 0, 0, 0, 0, 0, 1599, 12, 0, 0, 5, 0, 0, 3, 0, 0, 2541, 0, 1392, 37, 3, 4023, 0, 0, 0, 0, 0, 0, 9, 0, 4065, 0, 0, 2556, 0, 34, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 244, 14, 3164, 7, 12, 0, 0, 32, 119, 4, 0, 17, 0, 0, 23, 0, 4009, 36, 0, 4058, 0, 358, 0, 7, 1082, 0, 19, 32, 0, 234, 0, 0, 973, 0, 0, 0, 0, 0, 0, 39, 0, 2, 3287, 0, 3138, 26, 0, 3980, 0, 0, 0, 0, 0, 0, 21, 0, 4070, 0, 0, 3724, 0, 30, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 105, 15, 3298, 5, 17, 0, 0, 26, 269, 9, 7, 0, 0, 1, 25, 0, 4011, 8, 2, 4074, 0, 252, 0, 7, 1609, 0, 21, 109, 0, 179, 0, 12, 1002, 0, 0, 0, 149, 0, 0, 5, 0, 0, 1588, 0, 3829, 47, 4, 4048, 0, 0, 0, 0, 0, 0, 10, 0, 4072, 0, 0, 3711, 0, 241, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[14, 38, 0, 738, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 4077, 0, 0, 0, 0, 1145, 0, 0, 0, 0, 0, 0, 0, 4007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2494, 0, 3695, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4090, 0, 186, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[8, 680, 0, 1495, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 31, 3976, 0, 0, 0, 0, 457, 0, 0, 13, 0, 0, 0, 0, 3845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 0, 1856, 0, 0, 4096, 0, 1, 0, 0, 9, 0, 0, 0, 4080, 0, 0, 4094, 0, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[31, 5, 0, 1701, 0, 0, 0, 0, 1349, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2213, 0, 0, 6, 0, 0, 0, 2458, 77, 0, 0, 0, 0, 0, 0, 0, 3681, 0, 0, 0, 0, 0, 0, 1421, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 623, 0, 2187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 217, 0, 0, 4096, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3123, 4019, 3938, 4754, 5662, 2810, 4125, 4337] avg=4096.0000 max=5662 max_rank=4 min=2810 min_rank=5 imbalance_ratio=1.382324 max_load_reduction=27106 max_load_reduction_pct=0.827209 moved_to_echo=16934 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[33, 602, 0, 1689, 0, 0, 0, 0, 1164, 37, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 3918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 430, 0, 0, 14, 0, 7, 0, 2449, 76, 0, 0, 0, 0, 0, 0, 0, 2803, 0, 0, 0, 0, 0, 0, 1416, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3004, 0, 1886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 10, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 216, 0, 0, 4096, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3525, 4056, 4107, 2976, 4702, 4890, 4135, 4377] avg=4096.0000 max=4890 max_rank=5 min=2976 min_rank=3 imbalance_ratio=1.193848 max_load_reduction=27878 max_load_reduction_pct=0.850769 moved_to_echo=18104 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[40, 950, 0, 3666, 0, 0, 0, 0, 559, 37, 0, 0, 0, 0, 0, 0, 0, 227, 0, 0, 0, 2, 0, 0, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 3896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3043, 0, 0, 24, 0, 0, 0, 2445, 76, 0, 0, 0, 0, 0, 0, 0, 171, 0, 0, 0, 0, 0, 0, 1414, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2637, 3, 906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 0, 5, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 215, 4, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5252, 4248, 3944, 5588, 1817, 3546, 4072, 4301] avg=4096.0000 max=5588 max_rank=3 min=1817 min_rank=4 imbalance_ratio=1.364258 max_load_reduction=27180 max_load_reduction_pct=0.829468 moved_to_echo=13736 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 105, 15, 3298, 5, 17, 0, 0, 1261, 38, 0, 0, 0, 0, 0, 0, 26, 269, 9, 7, 0, 0, 1, 25, 3935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 2, 4006, 0, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1609, 0, 21, 109, 0, 179, 0, 2444, 76, 0, 0, 0, 0, 0, 0, 12, 1002, 0, 0, 0, 149, 0, 0, 1413, 524, 0, 0, 0, 0, 0, 0, 5, 0, 0, 1588, 0, 2044, 47, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 0, 0, 0, 0, 0, 10, 30, 0, 0, 0, 0, 0, 0, 0, 0, 215, 0, 0, 3711, 0, 241, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4739, 4272, 4268, 4445, 3100, 3688, 4088, 4168] avg=4096.0000 max=4739 max_rank=0 min=3100 min_rank=4 imbalance_ratio=1.156982 max_load_reduction=28029 max_load_reduction_pct=0.855377 moved_to_echo=15044 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 2504, 18, 3661, 2, 51, 0, 3, 458, 38, 0, 0, 0, 0, 0, 0, 21, 1, 9, 2, 3, 0, 0, 27, 3938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 3986, 0, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1897, 0, 19, 0, 0, 0, 0, 2440, 75, 0, 0, 0, 0, 0, 0, 0, 1599, 12, 0, 0, 5, 0, 0, 1410, 190, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2541, 0, 744, 37, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 0, 0, 0, 0, 0, 0, 9, 29, 0, 0, 0, 0, 0, 0, 0, 0, 215, 0, 0, 2556, 0, 34, 8, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6735, 4001, 4180, 4434, 3216, 3328, 4061, 2813] avg=4096.0000 max=6735 max_rank=0 min=2813 min_rank=7 imbalance_ratio=1.644287 max_load_reduction=26033 max_load_reduction_pct=0.794464 moved_to_echo=13418 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[14, 38, 0, 738, 0, 0, 0, 0, 1217, 38, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1145, 0, 0, 0, 0, 0, 0, 2458, 77, 0, 0, 0, 0, 0, 0, 0, 4007, 0, 0, 0, 0, 0, 0, 1421, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2494, 0, 1972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 217, 0, 0, 4090, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2045, 4021, 4011, 3680, 5934, 4466, 4118, 4493] avg=4096.0000 max=5934 max_rank=4 min=2045 min_rank=0 imbalance_ratio=1.448730 max_load_reduction=26834 max_load_reduction_pct=0.818909 moved_to_echo=19011 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[8, 680, 0, 1495, 0, 0, 0, 0, 611, 37, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 3910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 457, 0, 0, 13, 0, 0, 0, 2449, 77, 0, 0, 0, 0, 0, 0, 0, 3845, 0, 0, 0, 0, 0, 0, 1416, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 0, 991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 9, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 215, 0, 0, 4094, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2831, 4021, 3941, 2996, 5515, 5018, 4135, 4311] avg=4096.0000 max=5515 max_rank=4 min=2831 min_rank=0 imbalance_ratio=1.346436 max_load_reduction=27253 max_load_reduction_pct=0.831696 moved_to_echo=18979 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 244, 14, 3164, 7, 12, 0, 0, 1037, 40, 0, 0, 0, 0, 0, 0, 32, 119, 4, 0, 17, 0, 0, 23, 3935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 3985, 0, 358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1082, 0, 19, 32, 0, 234, 0, 2447, 74, 0, 0, 0, 0, 0, 0, 0, 973, 0, 0, 0, 0, 0, 0, 1416, 433, 0, 0, 0, 0, 0, 0, 39, 0, 2, 3287, 0, 1668, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3980, 0, 0, 0, 0, 0, 0, 21, 33, 0, 0, 0, 0, 0, 0, 0, 0, 207, 0, 0, 3724, 0, 30, 7, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4518, 4130, 4379, 3895, 2822, 5022, 4034, 3968] avg=4096.0000 max=5022 max_rank=5 min=2822 min_rank=4 imbalance_ratio=1.226074 max_load_reduction=27746 max_load_reduction_pct=0.846741 moved_to_echo=15846 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 5248, 128, 19456, 128, 128, 0, 128, 7680, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 18176, 128, 0, 0, 256, 0, 0, 11392, 3200, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 11904, 0, 128, 256, 0, 512, 0, 19712, 640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 20224, 128, 12416, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 1792, 128, 0, 30464, 0, 640, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 384, 31744, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 768, 128, 128, 128, 128, 128, 128, 32000, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32512, 0, 128, 0, 0, 128, 0, 128, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3955, 0, 4096, 0, 0, 4070, 2766, 0, 0, 0, 0, 0, 130, 0, 0, 0, 3823, 0, 0, 2, 53, 0, 49, 0, 0, 123, 0, 4096, 4096, 51, 0, 0, 0, 0, 1365, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4048, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2844, 0, 4095, 0, 0, 3906, 3859, 0, 0, 0, 0, 0, 544, 0, 0, 0, 3458, 0, 0, 0, 32, 0, 261, 0, 0, 265, 0, 4096, 4096, 140, 0, 0, 0, 0, 1108, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[16, 20, 0, 417, 0, 27, 4068, 0, 0, 0, 0, 0, 35, 25, 13, 0, 0, 0, 2, 12, 0, 3100, 0, 4071, 0, 4, 2172, 3363, 0, 0, 35, 16, 38, 47, 0, 1, 0, 4067, 20, 19, 78, 0, 0, 27, 0, 0, 654, 0, 3744, 4062, 29, 0, 0, 0, 0, 2582, 0, 0, 0, 0, 4, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3989, 0, 4096, 0, 0, 1406, 3826, 0, 0, 0, 0, 0, 129, 0, 0, 0, 3813, 0, 0, 10, 0, 0, 3, 0, 0, 922, 0, 4077, 4096, 208, 0, 0, 0, 0, 2115, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[24, 9, 0, 0, 0, 31, 4072, 30, 0, 0, 0, 0, 1, 37, 21, 0, 0, 0, 0, 12, 2, 3994, 0, 4061, 84, 4, 3993, 1197, 0, 0, 31, 20, 49, 119, 0, 9, 3, 4025, 15, 26, 6, 0, 0, 30, 0, 0, 273, 0, 3937, 4064, 1780, 0, 0, 0, 0, 793, 0, 0, 0, 0, 16, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3455, 0, 4096, 0, 0, 4084, 2328, 0, 0, 0, 0, 0, 35, 0, 0, 0, 3132, 0, 0, 2, 3, 0, 629, 0, 0, 35, 0, 4052, 4095, 48, 0, 0, 0, 0, 2708, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4062, 0, 4086, 0, 0, 3108, 1173, 9, 7, 0, 0, 0, 28, 0, 0, 0, 2862, 0, 0, 46, 1, 0, 2970, 0, 0, 159, 0, 4050, 4094, 430, 0, 0, 0, 0, 1600, 0, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[11, 5, 0, 0, 0, 25, 4070, 0, 0, 0, 0, 0, 23, 26, 16, 0, 0, 0, 0, 1, 3, 3334, 21, 4069, 16, 6, 3983, 2880, 0, 0, 24, 20, 0, 418, 4, 36, 16, 3704, 21, 18, 30, 6, 0, 364, 0, 0, 33, 0, 3821, 4071, 489, 0, 0, 0, 1, 1203, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3588, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3955, 0, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1692, 2766, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 0, 0, 3823, 0, 0, 262, 54, 0, 0, 0, 0, 0, 0, 2, 53, 0, 49, 0, 0, 123, 0, 2116, 1517, 0, 0, 0, 0, 0, 0, 2072, 0, 51, 0, 0, 0, 0, 1365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4039, 4095, 4454, 4458, 4269, 3860, 3488, 4105] avg=4096.0000 max=4458 max_rank=3 min=3488 min_rank=6 imbalance_ratio=1.088379 max_load_reduction=28310 max_load_reduction_pct=0.863953 moved_to_echo=15722 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3995, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3587, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2844, 0, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1623, 3859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 0, 0, 0, 3458, 0, 0, 252, 53, 0, 0, 0, 0, 0, 0, 0, 32, 0, 261, 0, 0, 265, 0, 2031, 1517, 0, 0, 0, 0, 0, 0, 2072, 0, 140, 0, 0, 0, 0, 1108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4011, 4094, 3343, 5482, 4307, 4106, 3320, 4105] avg=4096.0000 max=5482 max_rank=3 min=3320 min_rank=6 imbalance_ratio=1.338379 max_load_reduction=27286 max_load_reduction_pct=0.832703 moved_to_echo=15838 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[16, 20, 0, 417, 0, 27, 4014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 25, 13, 0, 3566, 463, 0, 0, 0, 0, 0, 0, 0, 0, 2, 12, 0, 3100, 0, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 903, 3363, 0, 0, 35, 16, 0, 0, 0, 0, 0, 0, 0, 0, 38, 47, 0, 1, 0, 4067, 20, 19, 140, 54, 0, 0, 0, 0, 0, 0, 78, 0, 0, 27, 0, 0, 654, 0, 1129, 1386, 0, 0, 0, 0, 0, 0, 1895, 0, 29, 0, 0, 0, 0, 2582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4062, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4494, 4102, 3610, 4321, 4386, 3274, 4506, 4075] avg=4096.0000 max=4506 max_rank=6 min=3274 min_rank=5 imbalance_ratio=1.100098 max_load_reduction=28262 max_load_reduction_pct=0.862488 moved_to_echo=16241 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3588, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3989, 0, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 585, 3826, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 0, 0, 0, 3813, 0, 0, 90, 54, 0, 0, 0, 0, 0, 0, 10, 0, 0, 3, 0, 0, 922, 0, 731, 1510, 0, 0, 0, 0, 0, 0, 2062, 0, 208, 0, 0, 0, 0, 2115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4024, 4093, 4488, 4411, 4086, 3176, 4385, 4105] avg=4096.0000 max=4488 max_rank=2 min=3176 min_rank=5 imbalance_ratio=1.095703 max_load_reduction=28280 max_load_reduction_pct=0.863037 moved_to_echo=15752 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3579, 501, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4062, 0, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1292, 1173, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 2862, 0, 0, 200, 54, 0, 0, 0, 0, 0, 0, 46, 1, 0, 2970, 0, 0, 159, 0, 1616, 1500, 0, 0, 0, 0, 0, 0, 2049, 0, 430, 0, 0, 0, 0, 1600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4021, 4080, 4567, 2481, 3144, 6292, 4079, 4104] avg=4096.0000 max=6292 max_rank=5 min=2481 min_rank=3 imbalance_ratio=1.536133 max_load_reduction=26476 max_load_reduction_pct=0.807983 moved_to_echo=17619 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3588, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3455, 0, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1698, 2328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 3132, 0, 0, 263, 54, 0, 0, 0, 0, 0, 0, 2, 3, 0, 629, 0, 0, 35, 0, 2123, 1500, 0, 0, 0, 0, 0, 0, 2050, 0, 48, 0, 0, 0, 0, 2708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4012, 4090, 3954, 4026, 3484, 4292, 4806, 4104] avg=4096.0000 max=4806 max_rank=6 min=3484 min_rank=4 imbalance_ratio=1.173340 max_load_reduction=27962 max_load_reduction_pct=0.853333 moved_to_echo=16686 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[11, 5, 0, 0, 0, 25, 4016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 26, 16, 0, 3564, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 3334, 21, 496, 0, 0, 0, 0, 0, 0, 0, 0, 16, 6, 1655, 2880, 0, 0, 24, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 418, 4, 36, 16, 3704, 21, 18, 257, 54, 0, 0, 0, 0, 0, 0, 30, 6, 0, 364, 0, 0, 33, 0, 2071, 1415, 0, 0, 0, 0, 0, 0, 1933, 0, 489, 0, 0, 0, 1, 1203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4057, 4102, 3855, 4601, 4528, 3919, 3626, 4080] avg=4096.0000 max=4601 max_rank=3 min=3626 min_rank=6 imbalance_ratio=1.123291 max_load_reduction=28167 max_load_reduction_pct=0.859589 moved_to_echo=16153 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[24, 9, 0, 0, 0, 31, 4016, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 37, 21, 0, 3562, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 2, 3994, 0, 489, 0, 0, 0, 0, 0, 0, 0, 0, 84, 4, 1652, 1197, 0, 0, 31, 20, 0, 0, 0, 0, 0, 0, 0, 0, 49, 119, 0, 9, 3, 4025, 15, 26, 262, 56, 0, 0, 0, 0, 0, 0, 6, 0, 0, 30, 0, 0, 273, 0, 2079, 1461, 0, 0, 0, 0, 0, 0, 1985, 0, 1780, 0, 0, 0, 0, 793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 4064, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4112, 4497, 2988, 4564, 3849, 4558, 4090] avg=4096.0000 max=4564 max_rank=4 min=2988 min_rank=3 imbalance_ratio=1.114258 max_load_reduction=28204 max_load_reduction_pct=0.860718 moved_to_echo=17061 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 128, 0, 4352, 0, 0, 2560, 0, 13952, 11904, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 11136, 21504, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[16128, 0, 3200, 0, 0, 0, 128, 13568, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 128, 128, 128, 0, 28672, 3968, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 128, 28800, 128, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 1536, 128, 128, 128, 28928, 128, 128, 1792, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 512, 0, 128, 32256, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 211, 3800, 0, 0, 653, 4, 0, 6, 0, 0, 0, 0, 0, 4081, 0, 0, 2130, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 234, 4090, 0, 0, 0, 3325, 0, 0, 0, 1, 0, 0, 2608, 0, 4095, 3436, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 9, 1628, 0, 0, 3025, 0, 0, 242, 0, 0, 0, 0, 0, 4096, 0, 0, 1452, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 915, 4096, 0, 0, 0, 2955, 0, 0, 0, 0, 0, 0, 2575, 0, 4096, 3584, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 23, 4013, 0, 0, 347, 0, 0, 18, 0, 0, 0, 0, 0, 4082, 0, 3, 1240, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 527, 4079, 0, 0, 0, 3563, 0, 0, 0, 0, 0, 0, 3528, 0, 4096, 3139, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 873, 4077, 0, 0, 395, 0, 0, 16, 0, 0, 0, 7, 0, 4081, 0, 21, 1458, 0, 0, 0, 14, 1, 0, 0, 0, 0, 0, 1544, 4068, 0, 0, 0, 2910, 0, 0, 0, 0, 0, 0, 4093, 0, 4082, 1019, 0, 0, 12, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7, 3466, 0, 0, 1425, 0, 0, 595, 0, 2, 0, 0, 0, 4096, 0, 0, 998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 806, 4096, 0, 0, 0, 1634, 0, 0, 0, 0, 0, 0, 3662, 0, 4094, 3804, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 539, 971, 0, 7, 38, 0, 12, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 4039, 18, 0, 0, 0, 0, 0, 0, 0, 29, 16, 2579, 4063, 0, 0, 20, 1660, 141, 0, 11, 0, 0, 21, 2828, 0, 4065, 3455, 25, 0, 7, 0, 0, 12, 37, 4018, 1, 31, 27, 3, 24, 0, 0, 5, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1374, 1148, 0, 9, 118, 0, 10, 29, 0, 25, 0, 0, 0, 4070, 0, 10, 3207, 16, 0, 0, 0, 0, 0, 0, 0, 25, 6, 1797, 4067, 0, 0, 32, 1253, 250, 0, 5, 0, 0, 12, 3256, 0, 4071, 4013, 11, 0, 19, 0, 0, 0, 2, 3832, 15, 25, 2, 19, 19, 0, 0, 18, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1363, 2762, 0, 18, 9, 0, 12, 8, 0, 0, 0, 0, 0, 4072, 0, 33, 243, 4, 0, 0, 0, 0, 0, 0, 0, 24, 5, 2311, 4057, 0, 0, 26, 3885, 30, 0, 9, 0, 0, 26, 2358, 0, 4072, 3413, 17, 0, 53, 0, 0, 0, 22, 3861, 0, 25, 0, 0, 20, 0, 0, 26, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7, 3466, 0, 0, 811, 0, 0, 0, 0, 0, 0, 0, 1425, 0, 0, 595, 0, 2, 0, 0, 2707, 597, 19, 0, 0, 0, 0, 0, 0, 2247, 0, 0, 998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 806, 2067, 986, 0, 0, 0, 0, 0, 0, 1389, 0, 0, 0, 1634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3662, 0, 0, 1140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4284, 5345, 3245, 3859, 3023, 4802, 4135, 4075] avg=4096.0000 max=5345 max_rank=1 min=3023 min_rank=4 imbalance_ratio=1.304932 max_load_reduction=27423 max_load_reduction_pct=0.836884 moved_to_echo=16035 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 23, 4013, 0, 0, 808, 0, 0, 0, 0, 0, 0, 0, 347, 0, 0, 18, 0, 0, 0, 0, 2695, 492, 19, 0, 0, 0, 0, 0, 0, 2240, 0, 3, 1240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 527, 1705, 982, 0, 0, 0, 0, 0, 0, 1384, 0, 0, 0, 3563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3528, 0, 0, 942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4844, 3571, 3483, 3231, 4947, 4470, 4145, 4077] avg=4096.0000 max=4947 max_rank=4 min=3231 min_rank=3 imbalance_ratio=1.207764 max_load_reduction=27821 max_load_reduction_pct=0.849030 moved_to_echo=17639 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 873, 4077, 0, 0, 808, 0, 0, 0, 0, 0, 0, 0, 395, 0, 0, 16, 0, 0, 0, 7, 2688, 160, 20, 0, 0, 0, 0, 0, 0, 2239, 0, 21, 1458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 1, 0, 0, 0, 0, 0, 1544, 553, 982, 0, 0, 0, 0, 0, 0, 1380, 0, 0, 0, 2910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 4096, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5759, 3286, 3718, 3094, 4290, 4399, 4160, 4062] avg=4096.0000 max=5759 max_rank=0 min=3094 min_rank=3 imbalance_ratio=1.406006 max_load_reduction=27009 max_load_reduction_pct=0.824249 moved_to_echo=16911 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 9, 1628, 0, 0, 811, 0, 0, 0, 0, 0, 0, 0, 3025, 0, 0, 242, 0, 0, 0, 0, 2707, 562, 19, 0, 0, 0, 0, 0, 0, 2247, 0, 0, 1452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 915, 1947, 986, 0, 0, 0, 0, 0, 0, 1389, 0, 0, 0, 2955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2575, 0, 0, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2448, 6555, 3699, 3849, 4344, 3650, 4146, 4077] avg=4096.0000 max=6555 max_rank=1 min=2448 min_rank=0 imbalance_ratio=1.600342 max_load_reduction=26213 max_load_reduction_pct=0.799957 moved_to_echo=16217 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1374, 1148, 0, 9, 806, 0, 0, 0, 0, 0, 0, 0, 118, 0, 10, 29, 0, 25, 0, 0, 2688, 630, 19, 0, 0, 0, 0, 0, 0, 2232, 0, 10, 3207, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 6, 1797, 2180, 980, 0, 0, 0, 0, 0, 0, 1379, 0, 0, 32, 1253, 250, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 3256, 0, 0, 1203, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 2, 3832, 15, 52, 0, 0, 0, 0, 0, 0, 0, 25, 2, 19, 19, 0, 0, 18, 3, 4052, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3337, 3519, 5465, 4988, 2919, 4482, 3920, 4138] avg=4096.0000 max=5465 max_rank=2 min=2919 min_rank=4 imbalance_ratio=1.334229 max_load_reduction=27303 max_load_reduction_pct=0.833221 moved_to_echo=15459 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1363, 2762, 0, 18, 806, 0, 0, 0, 0, 0, 0, 0, 9, 0, 12, 8, 0, 0, 0, 0, 2681, 535, 19, 0, 0, 0, 0, 0, 0, 2234, 0, 33, 243, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 5, 2311, 1854, 980, 0, 0, 0, 0, 0, 0, 1376, 0, 0, 26, 3885, 30, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 2358, 0, 0, 1024, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 22, 3861, 0, 52, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 20, 0, 0, 26, 4, 4053, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4949, 3264, 2514, 5174, 5326, 3425, 3988, 4128] avg=4096.0000 max=5326 max_rank=4 min=2514 min_rank=2 imbalance_ratio=1.300293 max_load_reduction=27442 max_load_reduction_pct=0.837463 moved_to_echo=16867 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 539, 971, 0, 7, 811, 0, 0, 0, 0, 0, 0, 0, 38, 0, 12, 0, 0, 0, 0, 0, 2688, 546, 20, 0, 0, 0, 0, 0, 0, 2218, 0, 0, 4039, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 16, 2579, 1882, 983, 0, 0, 0, 0, 0, 0, 1375, 0, 0, 20, 1660, 141, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 2828, 0, 0, 1027, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 12, 37, 4018, 1, 54, 0, 0, 0, 0, 0, 0, 0, 31, 27, 3, 24, 0, 0, 5, 0, 4045, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2328, 3304, 6275, 5489, 3207, 3901, 4129, 4135] avg=4096.0000 max=6275 max_rank=2 min=2328 min_rank=0 imbalance_ratio=1.531982 max_load_reduction=26493 max_load_reduction_pct=0.808502 moved_to_echo=15372 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 211, 3800, 0, 0, 808, 0, 0, 0, 0, 0, 0, 0, 653, 4, 0, 6, 0, 0, 0, 0, 2703, 539, 19, 0, 0, 0, 0, 0, 0, 2239, 0, 0, 2130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 234, 1867, 982, 0, 0, 0, 0, 0, 0, 1387, 0, 0, 0, 3325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2608, 0, 0, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4819, 3924, 4369, 3084, 4712, 3639, 4145, 4076] avg=4096.0000 max=4819 max_rank=0 min=3084 min_rank=3 imbalance_ratio=1.176514 max_load_reduction=27949 max_load_reduction_pct=0.852936 moved_to_echo=16572 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 0, 128, 128, 10752, 14080, 7936, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[6016, 128, 128, 1024, 0, 128, 0, 128, 21632, 4096, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 17920, 0, 128, 14848, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 24960, 0, 0, 7808, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 4480, 21888, 0, 128, 6528, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[11136, 0, 0, 128, 21248, 512, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 0, 0, 128, 128, 32640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 128, 128, 32256, 128, 512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 3975, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 6, 0, 0, 4096, 1, 0, 2, 0, 0, 0, 0, 126, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 3690, 4030, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 27, 0, 0, 4096, 1, 0, 61, 0, 0, 0, 0, 366, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 3808, 4051, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 42, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 285, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 4083, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 14, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 3473, 3916, 0, 4, 17, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 37, 0, 0, 4087, 4, 0, 120, 0, 0, 0, 0, 651, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 3845, 4072, 0, 0, 0, 0, 3, 4043, 0, 44, 0, 0, 4, 3996, 0, 0, 0, 0, 0, 36, 7, 17, 0, 1, 3, 23, 0, 0, 16, 0, 0, 15, 32, 0, 4050, 26, 12, 41, 0, 14, 0, 4070, 0, 24, 5, 0, 15, 0, 0, 557, 0, 2, 6, 8, 0, 0, 0, 0, 0, 0, 3722, 4059, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 2737, 4060, 0, 89, 0, 15, 1, 4014, 0, 31, 0, 0, 4, 3995, 0, 0, 2, 0, 0, 0, 11, 636, 0, 0, 20, 29, 0, 0, 10, 0, 0, 44, 0, 0, 4049, 58, 5, 46, 0, 3, 0, 4058, 0, 29, 0, 14, 0, 6, 0, 1047, 0, 65, 1, 26, 0, 0, 0, 0, 0, 0, 3612, 4051, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 3816, 4054, 0, 53, 0, 0, 21, 3992, 0, 21, 0, 0, 0, 4028, 0, 0, 6, 0, 0, 0, 9, 1, 0, 0, 0, 25, 0, 0, 20, 0, 0, 28, 0, 0, 4052, 23, 10, 36, 1, 35, 0, 4063, 0, 24, 0, 2, 0, 0, 0, 646, 0, 6, 4, 0, 20, 0, 0, 0, 0, 0, 3704, 4068, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3975, 397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3889, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 2, 0, 0, 0, 0, 3567, 42, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4372, 4117, 4019, 4107, 4060, 4068, 3768, 4257] avg=4096.0000 max=4372 max_rank=0 min=3768 min_rank=6 imbalance_ratio=1.067383 max_load_reduction=28396 max_load_reduction_pct=0.866577 moved_to_echo=16153 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3690, 391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3889, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 1, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 366, 47, 0, 0, 0, 0, 3511, 42, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4117, 4017, 4107, 4080, 4127, 3997, 4242] avg=4096.0000 max=4242 max_rank=7 min=3997 min_rank=6 imbalance_ratio=1.035645 max_load_reduction=28526 max_load_reduction_pct=0.870544 moved_to_echo=16446 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3808, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3889, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 285, 72, 0, 0, 0, 0, 3529, 42, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4202, 4117, 4022, 4107, 4096, 4065, 3959, 4200] avg=4096.0000 max=4202 max_rank=0 min=3959 min_rank=6 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16320 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3473, 381, 0, 4, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3889, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 4, 0, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 651, 21, 0, 0, 0, 0, 3411, 41, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3875, 4117, 4013, 4107, 4076, 4180, 4155, 4245] avg=4096.0000 max=4245 max_rank=7 min=3875 min_rank=0 imbalance_ratio=1.036377 max_load_reduction=28523 max_load_reduction_pct=0.870453 moved_to_echo=16656 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3816, 394, 0, 53, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 21, 0, 0, 0, 4028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 9, 1, 0, 3862, 128, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 20, 0, 0, 3971, 32, 0, 0, 0, 0, 0, 0, 28, 0, 0, 4011, 23, 10, 36, 1, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 4033, 0, 24, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 646, 0, 6, 4, 0, 20, 3532, 41, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3704, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4284, 4070, 4006, 4048, 4109, 4094, 4279, 3878] avg=4096.0000 max=4284 max_rank=0 min=3878 min_rank=7 imbalance_ratio=1.045898 max_load_reduction=28484 max_load_reduction_pct=0.869263 moved_to_echo=16360 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4083, 398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3889, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 1, 0, 0, 0, 0, 3568, 42, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4481, 4117, 4019, 4107, 4054, 4065, 3656, 4269] avg=4096.0000 max=4481 max_rank=0 min=3656 min_rank=6 imbalance_ratio=1.093994 max_load_reduction=28287 max_load_reduction_pct=0.863251 moved_to_echo=16044 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3845, 396, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 44, 0, 0, 4, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 7, 17, 0, 3854, 129, 0, 0, 0, 0, 0, 0, 1, 3, 23, 0, 0, 16, 0, 0, 4022, 31, 0, 0, 0, 0, 0, 0, 15, 32, 0, 4009, 26, 12, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 4040, 0, 24, 5, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 557, 0, 2, 6, 8, 0, 3547, 41, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3722, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4244, 4065, 4043, 4096, 4135, 4098, 4191, 3896] avg=4096.0000 max=4244 max_rank=0 min=3896 min_rank=7 imbalance_ratio=1.036133 max_load_reduction=28524 max_load_reduction_pct=0.870483 moved_to_echo=16320 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2737, 387, 0, 89, 0, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 31, 0, 0, 4, 3995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 11, 636, 0, 3848, 132, 0, 0, 0, 0, 0, 0, 0, 20, 29, 0, 0, 10, 0, 0, 3996, 34, 0, 0, 0, 0, 0, 0, 44, 0, 0, 4005, 58, 5, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4025, 0, 29, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1047, 0, 65, 1, 26, 0, 3541, 44, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3612, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3229, 4048, 4629, 4089, 4158, 4071, 4763, 3781] avg=4096.0000 max=4763 max_rank=6 min=3229 min_rank=0 imbalance_ratio=1.162842 max_load_reduction=28005 max_load_reduction_pct=0.854645 moved_to_echo=16773 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 29440, 3200, 0, 256, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 0, 3712, 256, 128, 128, 128, 128, 28288, 384, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 128, 128, 768, 0, 31104, 1152, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 31488, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 0, 0, 128, 0, 0, 32384, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 32512, 128, 128, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 32384, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[256, 0, 128, 0, 0, 128, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=32896 slack_slots=2816 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True + [2026-09-21 08:25:38] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 275.0 | throughput per GPU (TFLOP/s/GPU): 117.2 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.642425E+00 | loss scale: 1.0 | grad norm: 3.521 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[173, 251, 1254, 0, 0, 686, 923, 396, 3, 350, 52, 0, 11, 4, 493, 133, 12, 1501, 809, 1367, 88, 88, 285, 1, 144, 38, 135, 265, 761, 591, 1071, 1662, 901, 398, 483, 1239, 220, 897, 759, 516, 1529, 289, 1332, 70, 645, 569, 21, 98, 749, 963, 843, 32, 942, 1713, 1578, 116, 825, 41, 443, 23, 25, 33, 900, 29] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[134, 346, 1055, 2, 0, 596, 569, 417, 2, 477, 38, 1, 12, 31, 464, 170, 6, 993, 921, 1568, 37, 130, 345, 8, 123, 11, 111, 348, 929, 550, 1383, 1412, 728, 821, 820, 884, 216, 669, 708, 1017, 1565, 664, 1299, 104, 1073, 1072, 11, 31, 988, 820, 464, 36, 717, 1387, 1410, 42, 1002, 22, 259, 0, 8, 8, 710, 24] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[246, 338, 852, 0, 2, 686, 936, 538, 2, 476, 27, 1, 2, 50, 557, 281, 7, 445, 854, 1431, 43, 376, 530, 14, 171, 34, 129, 534, 1010, 888, 1131, 1443, 670, 729, 508, 1378, 148, 1110, 799, 669, 1187, 493, 1324, 31, 963, 683, 8, 64, 835, 680, 505, 107, 864, 666, 1920, 44, 962, 19, 286, 4, 93, 9, 933, 43] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[118, 292, 1660, 0, 1, 596, 687, 601, 4, 886, 94, 1, 13, 8, 486, 162, 2, 909, 617, 1483, 34, 122, 243, 7, 69, 18, 114, 398, 558, 707, 922, 1527, 1156, 689, 539, 855, 181, 512, 526, 635, 1620, 841, 1691, 101, 810, 745, 11, 69, 799, 898, 691, 33, 810, 1680, 1280, 25, 887, 16, 286, 1, 4, 21, 983, 34] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[154, 243, 972, 2, 0, 594, 735, 526, 3, 679, 34, 0, 12, 31, 397, 133, 4, 1004, 538, 1626, 27, 373, 335, 3, 50, 5, 63, 554, 1018, 487, 744, 1731, 1114, 957, 757, 1130, 176, 686, 780, 1067, 1189, 655, 1994, 144, 824, 906, 5, 17, 767, 830, 725, 34, 594, 1213, 1101, 18, 773, 11, 341, 14, 10, 33, 812, 14] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[302, 330, 1088, 1, 0, 928, 661, 317, 0, 1080, 41, 3, 4, 31, 559, 347, 1, 659, 1210, 1499, 32, 233, 297, 8, 129, 18, 129, 542, 1000, 899, 1009, 1489, 618, 502, 638, 1465, 192, 782, 1006, 785, 1507, 365, 1073, 171, 855, 572, 6, 17, 704, 619, 720, 72, 506, 1086, 1381, 138, 1105, 44, 234, 5, 11, 18, 701, 24] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[76, 327, 1488, 2, 2, 767, 660, 764, 0, 435, 54, 2, 37, 10, 791, 77, 0, 795, 680, 2108, 21, 294, 168, 11, 145, 34, 67, 204, 991, 757, 784, 1302, 757, 403, 578, 827, 144, 1100, 707, 582, 1560, 624, 2010, 85, 476, 633, 20, 101, 857, 953, 771, 34, 865, 1464, 1288, 77, 714, 44, 578, 5, 60, 52, 531, 15] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[158, 218, 1183, 3, 0, 507, 606, 451, 0, 412, 14, 2, 34, 8, 861, 129, 9, 1297, 1016, 1255, 46, 193, 220, 3, 111, 46, 74, 311, 978, 849, 1131, 1844, 972, 376, 563, 1063, 281, 829, 699, 1003, 1187, 562, 1353, 103, 874, 599, 16, 75, 750, 958, 684, 37, 983, 1155, 1098, 32, 1017, 35, 308, 48, 54, 111, 956, 18] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[173, 251, 1254, 0, 0, 686, 923, 396, 331, 297, 0, 0, 0, 0, 0, 0, 3, 350, 52, 0, 11, 4, 493, 133, 1578, 1239, 150, 0, 0, 0, 0, 0, 12, 1501, 809, 1367, 88, 88, 285, 1, 252, 0, 0, 0, 0, 0, 0, 0, 144, 38, 135, 265, 761, 591, 1071, 870, 0, 0, 0, 0, 0, 0, 0, 0, 341, 398, 483, 0, 220, 897, 759, 516, 0, 0, 0, 0, 0, 0, 0, 0, 1529, 289, 359, 70, 645, 569, 21, 98, 0, 0, 0, 0, 0, 0, 0, 0, 749, 963, 843, 32, 942, 1164, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 825, 41, 443, 23, 25, 33, 900, 29, 823, 792, 229, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4311, 4013, 4403, 3875, 3614, 3580, 4809, 4163] avg=4096.0000 max=4809 max_rank=6 min=3580 min_rank=5 imbalance_ratio=1.174072 max_load_reduction=27959 max_load_reduction_pct=0.853241 moved_to_echo=16166 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[246, 338, 852, 0, 2, 686, 936, 538, 246, 115, 0, 0, 0, 0, 0, 0, 2, 476, 27, 1, 2, 50, 557, 281, 1920, 1378, 149, 0, 0, 0, 0, 0, 7, 445, 854, 1431, 43, 376, 530, 14, 98, 0, 0, 0, 0, 0, 0, 0, 171, 34, 129, 534, 1010, 888, 1131, 756, 0, 0, 0, 0, 0, 0, 0, 0, 254, 729, 508, 0, 148, 1110, 799, 669, 0, 0, 0, 0, 0, 0, 0, 0, 1187, 493, 357, 31, 963, 683, 8, 64, 0, 0, 0, 0, 0, 0, 0, 0, 835, 680, 505, 107, 864, 453, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 962, 19, 286, 4, 93, 9, 933, 43, 818, 687, 170, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3959, 4843, 3798, 4653, 4217, 3786, 3488, 4024] avg=4096.0000 max=4843 max_rank=1 min=3488 min_rank=6 imbalance_ratio=1.182373 max_load_reduction=27925 max_load_reduction_pct=0.852203 moved_to_echo=15515 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[134, 346, 1055, 2, 0, 596, 569, 417, 268, 240, 0, 0, 0, 0, 0, 0, 2, 477, 38, 1, 12, 31, 464, 170, 1410, 884, 146, 0, 0, 0, 0, 0, 6, 993, 921, 1568, 37, 130, 345, 8, 204, 0, 0, 0, 0, 0, 0, 0, 123, 11, 111, 348, 929, 550, 1383, 740, 0, 0, 0, 0, 0, 0, 0, 0, 275, 821, 820, 0, 216, 669, 708, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1565, 664, 351, 104, 1073, 1072, 11, 31, 0, 0, 0, 0, 0, 0, 0, 0, 988, 820, 464, 36, 717, 943, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 22, 259, 0, 8, 8, 710, 24, 802, 672, 185, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3627, 3635, 4212, 4195, 4526, 4871, 4010, 3692] avg=4096.0000 max=4871 max_rank=5 min=3627 min_rank=0 imbalance_ratio=1.189209 max_load_reduction=27897 max_load_reduction_pct=0.851349 moved_to_echo=17099 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[118, 292, 1660, 0, 1, 596, 687, 601, 425, 291, 0, 0, 0, 0, 0, 0, 4, 886, 94, 1, 13, 8, 486, 162, 1280, 855, 190, 0, 0, 0, 0, 0, 2, 909, 617, 1483, 34, 122, 243, 7, 247, 0, 0, 0, 0, 0, 0, 0, 69, 18, 114, 398, 558, 707, 922, 800, 0, 0, 0, 0, 0, 0, 0, 0, 437, 689, 539, 0, 181, 512, 526, 635, 0, 0, 0, 0, 0, 0, 0, 0, 1620, 841, 456, 101, 810, 745, 11, 69, 0, 0, 0, 0, 0, 0, 0, 0, 799, 898, 691, 33, 810, 1142, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 887, 16, 286, 1, 4, 21, 983, 34, 1045, 727, 294, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4671, 3979, 3664, 3586, 3519, 4653, 4398, 4298] avg=4096.0000 max=4671 max_rank=0 min=3519 min_rank=4 imbalance_ratio=1.140381 max_load_reduction=28097 max_load_reduction_pct=0.857452 moved_to_echo=16868 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[302, 330, 1088, 1, 0, 928, 661, 317, 227, 188, 0, 0, 0, 0, 0, 0, 0, 1080, 41, 3, 4, 31, 559, 347, 1381, 1465, 120, 0, 0, 0, 0, 0, 1, 659, 1210, 1499, 32, 233, 297, 8, 160, 0, 0, 0, 0, 0, 0, 0, 129, 18, 129, 542, 1000, 899, 1009, 780, 0, 0, 0, 0, 0, 0, 0, 0, 234, 502, 638, 0, 192, 782, 1006, 785, 0, 0, 0, 0, 0, 0, 0, 0, 1507, 365, 290, 171, 855, 572, 6, 17, 0, 0, 0, 0, 0, 0, 0, 0, 704, 619, 720, 72, 506, 738, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 1105, 44, 234, 5, 11, 18, 701, 24, 663, 709, 157, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4042, 5031, 4099, 4506, 4139, 3783, 3497, 3671] avg=4096.0000 max=5031 max_rank=1 min=3497 min_rank=6 imbalance_ratio=1.228271 max_load_reduction=27737 max_load_reduction_pct=0.846466 moved_to_echo=15090 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[76, 327, 1488, 2, 2, 767, 660, 764, 278, 254, 0, 0, 0, 0, 0, 0, 0, 435, 54, 2, 37, 10, 791, 77, 1288, 827, 226, 0, 0, 0, 0, 0, 0, 795, 680, 2108, 21, 294, 168, 11, 215, 0, 0, 0, 0, 0, 0, 0, 145, 34, 67, 204, 991, 757, 784, 682, 0, 0, 0, 0, 0, 0, 0, 0, 287, 403, 578, 0, 144, 1100, 707, 582, 0, 0, 0, 0, 0, 0, 0, 0, 1560, 624, 542, 85, 476, 633, 20, 101, 0, 0, 0, 0, 0, 0, 0, 0, 857, 953, 771, 34, 865, 995, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 714, 44, 578, 5, 60, 52, 531, 15, 1242, 620, 192, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4618, 3747, 4292, 3664, 3801, 4041, 4552, 4053] avg=4096.0000 max=4618 max_rank=0 min=3664 min_rank=3 imbalance_ratio=1.127441 max_load_reduction=28150 max_load_reduction_pct=0.859070 moved_to_echo=16447 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[158, 218, 1183, 3, 0, 507, 606, 451, 358, 200, 0, 0, 0, 0, 0, 0, 0, 412, 14, 2, 34, 8, 861, 129, 1098, 1063, 152, 0, 0, 0, 0, 0, 9, 1297, 1016, 1255, 46, 193, 220, 3, 170, 0, 0, 0, 0, 0, 0, 0, 111, 46, 74, 311, 978, 849, 1131, 966, 0, 0, 0, 0, 0, 0, 0, 0, 367, 376, 563, 0, 281, 829, 699, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1187, 562, 365, 103, 874, 599, 16, 75, 0, 0, 0, 0, 0, 0, 0, 0, 750, 958, 684, 37, 983, 785, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 35, 308, 48, 54, 111, 956, 18, 836, 878, 247, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3684, 3773, 4209, 4466, 4118, 3781, 4229, 4508] avg=4096.0000 max=4508 max_rank=7 min=3684 min_rank=0 imbalance_ratio=1.100586 max_load_reduction=28260 max_load_reduction_pct=0.862427 moved_to_echo=16636 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[154, 243, 972, 2, 0, 594, 735, 526, 415, 215, 0, 0, 0, 0, 0, 0, 3, 679, 34, 0, 12, 31, 397, 133, 1101, 1130, 227, 0, 0, 0, 0, 0, 4, 1004, 538, 1626, 27, 373, 335, 3, 181, 0, 0, 0, 0, 0, 0, 0, 50, 5, 63, 554, 1018, 487, 744, 902, 0, 0, 0, 0, 0, 0, 0, 0, 411, 957, 757, 0, 176, 686, 780, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 1189, 655, 533, 144, 824, 906, 5, 17, 0, 0, 0, 0, 0, 0, 0, 0, 767, 830, 725, 34, 594, 817, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 773, 11, 341, 14, 10, 33, 812, 14, 1234, 829, 288, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3856, 3747, 4091, 3823, 4834, 4273, 3785, 4359] avg=4096.0000 max=4834 max_rank=4 min=3747 min_rank=1 imbalance_ratio=1.180176 max_load_reduction=27934 max_load_reduction_pct=0.852478 moved_to_echo=17251 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[1408, 2432, 9600, 128, 128, 5376, 5888, 4096, 2560, 1920, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[2688, 4992, 4992, 0, 1664, 6656, 6016, 6400, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[11392, 4608, 3328, 896, 6528, 5888, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1024, 256, 896, 3200, 7296, 5760, 8192, 6528, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[7296, 256, 2816, 128, 384, 384, 6528, 256, 7552, 6016, 1792, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 4864, 384, 128, 128, 256, 4608, 1536, 11136, 8960, 1408, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[6528, 6784, 5504, 512, 6400, 7040, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 7680, 6656, 12416, 384, 1920, 2432, 128, 1536, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1172, 0, 51, 0, 0, 599, 0, 1734, 109, 959, 13, 2843, 0, 0, 2806, 0, 2, 0, 45, 707, 657, 523, 317, 0, 0, 755, 0, 0, 1963, 1818, 218, 509, 0, 0, 0, 0, 406, 3, 1037, 0, 67, 3, 697, 0, 0, 0, 1041, 0, 115, 1, 958, 3559, 3376, 0, 8, 0, 3152, 0, 427, 111, 4, 0, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1023, 0, 7, 0, 0, 402, 2, 3609, 1770, 1876, 273, 2910, 0, 0, 1206, 3, 0, 0, 7, 2965, 483, 301, 65, 0, 0, 642, 0, 0, 569, 2386, 549, 371, 0, 0, 0, 0, 728, 24, 198, 3, 0, 5, 469, 1, 0, 0, 185, 0, 0, 2, 290, 3501, 2160, 0, 27, 0, 2316, 0, 1360, 21, 10, 0, 48, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1055, 0, 33, 0, 0, 70, 2, 1799, 2184, 2978, 73, 3235, 0, 0, 2399, 2, 0, 0, 2, 1643, 903, 50, 8, 0, 0, 989, 0, 0, 89, 3580, 265, 41, 1, 0, 0, 0, 1278, 134, 16, 1, 39, 1, 1016, 0, 0, 0, 822, 0, 6, 1, 40, 2959, 2013, 0, 26, 0, 1481, 2, 341, 1102, 34, 0, 55, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1238, 0, 40, 0, 2, 216, 0, 3347, 1589, 3886, 151, 3587, 0, 0, 1339, 30, 0, 7, 37, 1046, 3732, 361, 104, 0, 0, 427, 0, 0, 518, 1578, 280, 93, 0, 0, 2, 0, 255, 22, 21, 1, 99, 0, 184, 0, 0, 0, 17, 0, 0, 31, 906, 1433, 3460, 0, 0, 0, 1354, 0, 644, 216, 166, 0, 349, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1731, 1, 122, 0, 2, 3, 0, 1530, 24, 1782, 2, 2631, 0, 0, 3512, 0, 4, 0, 1, 591, 9, 723, 48, 0, 0, 1046, 0, 0, 251, 2177, 1, 272, 12, 0, 22, 0, 220, 722, 729, 0, 35, 10, 1909, 0, 0, 0, 722, 0, 74, 0, 1213, 3643, 609, 0, 4, 0, 3243, 0, 678, 2153, 16, 0, 291, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[833, 0, 787, 0, 0, 28, 3, 588, 425, 1078, 3, 2984, 0, 0, 3495, 4, 1, 0, 23, 2056, 627, 72, 525, 0, 1, 1549, 0, 0, 221, 3660, 114, 27, 0, 0, 3, 2, 1551, 34, 48, 4, 0, 1, 1241, 0, 0, 0, 196, 0, 32, 0, 986, 3516, 3245, 0, 1, 0, 2286, 1, 384, 85, 7, 0, 41, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[755, 1, 110, 0, 6, 89, 3, 3382, 1867, 990, 373, 1987, 0, 0, 1894, 0, 0, 0, 160, 82, 1868, 139, 31, 0, 0, 544, 0, 0, 631, 2762, 10, 1, 0, 0, 0, 0, 377, 9, 207, 0, 39, 0, 2084, 0, 0, 0, 3657, 1, 9, 0, 204, 3944, 1855, 0, 4, 0, 614, 0, 1835, 104, 41, 0, 99, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[2307, 2, 47, 0, 0, 159, 34, 2842, 1123, 3583, 12, 1251, 0, 0, 1907, 0, 5, 0, 204, 124, 446, 407, 969, 2, 0, 1096, 0, 0, 543, 1679, 560, 17, 0, 0, 1, 1, 620, 56, 640, 8, 16, 0, 831, 0, 0, 1, 627, 16, 47, 0, 30, 3933, 2681, 0, 0, 0, 2139, 3, 360, 1365, 34, 0, 40, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1172, 0, 51, 0, 0, 599, 0, 1734, 92, 0, 0, 0, 0, 0, 0, 0, 109, 959, 13, 0, 0, 0, 849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 45, 707, 657, 523, 317, 0, 1464, 0, 0, 0, 0, 0, 0, 0, 0, 755, 0, 0, 1963, 1631, 218, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 406, 3, 1037, 0, 2843, 261, 0, 0, 0, 0, 0, 0, 67, 3, 697, 0, 0, 0, 1041, 0, 2175, 82, 0, 0, 0, 0, 0, 0, 115, 1, 958, 1123, 3376, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3152, 0, 427, 111, 4, 0, 3, 0, 411, 95, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3648, 1930, 3715, 5076, 4550, 4065, 5581, 4203] avg=4096.0000 max=5581 max_rank=6 min=1930 min_rank=1 imbalance_ratio=1.362549 max_load_reduction=27187 max_load_reduction_pct=0.829681 moved_to_echo=18399 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[755, 1, 110, 0, 6, 89, 3, 3382, 145, 0, 0, 0, 0, 0, 0, 0, 1867, 990, 373, 0, 0, 0, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 82, 1868, 139, 31, 0, 991, 0, 0, 0, 0, 0, 0, 0, 0, 544, 0, 0, 631, 2469, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 377, 9, 207, 0, 1987, 292, 0, 0, 0, 0, 0, 0, 39, 0, 2084, 0, 0, 0, 3657, 1, 2415, 59, 0, 0, 0, 0, 0, 0, 9, 0, 204, 1237, 1855, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 614, 0, 1835, 104, 41, 0, 99, 0, 281, 148, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4491, 3793, 3271, 3655, 2872, 8255, 3309, 3122] avg=4096.0000 max=8255 max_rank=5 min=2872 min_rank=4 imbalance_ratio=2.015381 max_load_reduction=24513 max_load_reduction_pct=0.748077 moved_to_echo=17558 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2307, 2, 47, 0, 0, 159, 34, 2842, 85, 0, 0, 0, 0, 0, 0, 0, 1123, 3583, 12, 0, 0, 0, 577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 204, 124, 446, 407, 969, 2, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1096, 0, 0, 543, 1506, 560, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 620, 56, 640, 8, 1251, 288, 0, 0, 0, 0, 0, 0, 16, 0, 831, 0, 0, 1, 627, 16, 2403, 56, 0, 0, 0, 0, 0, 0, 47, 0, 30, 1242, 2681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2139, 3, 360, 1365, 34, 0, 40, 0, 279, 88, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5476, 5295, 3152, 3722, 2865, 3950, 4000, 4308] avg=4096.0000 max=5476 max_rank=0 min=2865 min_rank=4 imbalance_ratio=1.336914 max_load_reduction=27292 max_load_reduction_pct=0.832886 moved_to_echo=15123 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1731, 1, 122, 0, 2, 3, 0, 1530, 111, 0, 0, 0, 0, 0, 0, 0, 24, 1782, 2, 0, 0, 0, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 591, 9, 723, 48, 0, 1833, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 0, 0, 251, 1952, 1, 272, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 22, 0, 220, 722, 729, 0, 2631, 267, 0, 0, 0, 0, 0, 0, 35, 10, 1909, 0, 0, 0, 722, 0, 2226, 103, 0, 0, 0, 0, 0, 0, 74, 0, 1213, 1150, 609, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3243, 0, 678, 2153, 16, 0, 291, 0, 514, 114, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3500, 2870, 3209, 3522, 4603, 5005, 3050, 7009] avg=4096.0000 max=7009 max_rank=7 min=2870 min_rank=1 imbalance_ratio=1.711182 max_load_reduction=25759 max_load_reduction_pct=0.786102 moved_to_echo=19667 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1023, 0, 7, 0, 0, 402, 2, 3609, 121, 0, 0, 0, 0, 0, 0, 0, 1770, 1876, 273, 0, 0, 0, 366, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2965, 483, 301, 65, 0, 629, 0, 0, 0, 0, 0, 0, 0, 0, 642, 0, 0, 569, 2140, 549, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 728, 24, 198, 3, 2910, 256, 0, 0, 0, 0, 0, 0, 0, 5, 469, 1, 0, 0, 185, 0, 2139, 35, 0, 0, 0, 0, 0, 0, 0, 2, 290, 1106, 2160, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2316, 0, 1360, 21, 10, 0, 48, 1, 176, 125, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5164, 4288, 4450, 4271, 4119, 2834, 3585, 4057] avg=4096.0000 max=5164 max_rank=0 min=2834 min_rank=5 imbalance_ratio=1.260742 max_load_reduction=27604 max_load_reduction_pct=0.842407 moved_to_echo=14595 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[833, 0, 787, 0, 0, 28, 3, 588, 186, 0, 0, 0, 0, 0, 0, 0, 425, 1078, 3, 0, 0, 0, 1057, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 23, 2056, 627, 72, 525, 0, 1824, 0, 0, 0, 0, 0, 0, 0, 1, 1549, 0, 0, 221, 3282, 114, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 1551, 34, 48, 4, 2984, 258, 0, 0, 0, 0, 0, 0, 0, 1, 1241, 0, 0, 0, 196, 0, 2149, 102, 0, 0, 0, 0, 0, 0, 32, 0, 986, 1109, 3245, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2286, 1, 384, 85, 7, 0, 41, 0, 512, 192, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2425, 2567, 5128, 5194, 4884, 3689, 5373, 3508] avg=4096.0000 max=5373 max_rank=6 min=2425 min_rank=0 imbalance_ratio=1.311768 max_load_reduction=27395 max_load_reduction_pct=0.836029 moved_to_echo=17454 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1238, 0, 40, 0, 2, 216, 0, 3347, 80, 0, 0, 0, 0, 0, 0, 0, 1589, 3886, 151, 0, 0, 0, 406, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 37, 1046, 3732, 361, 104, 0, 698, 0, 0, 0, 0, 0, 0, 0, 0, 427, 0, 0, 518, 1416, 280, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 255, 22, 21, 1, 3587, 105, 0, 0, 0, 0, 0, 0, 99, 0, 184, 0, 0, 0, 17, 0, 875, 39, 0, 0, 0, 0, 0, 0, 0, 31, 906, 453, 3460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1354, 0, 644, 216, 166, 0, 349, 0, 196, 82, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4923, 6062, 5985, 2734, 3993, 1214, 4850, 3007] avg=4096.0000 max=6062 max_rank=1 min=1214 min_rank=5 imbalance_ratio=1.479980 max_load_reduction=26706 max_load_reduction_pct=0.815002 moved_to_echo=13064 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1055, 0, 33, 0, 0, 70, 2, 1799, 182, 0, 0, 0, 0, 0, 0, 0, 2184, 2978, 73, 0, 0, 0, 726, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1643, 903, 50, 8, 0, 1252, 0, 0, 0, 0, 0, 0, 0, 0, 989, 0, 0, 89, 3210, 265, 41, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1278, 134, 16, 1, 3235, 217, 0, 0, 0, 0, 0, 0, 39, 1, 1016, 0, 0, 0, 822, 0, 1808, 70, 0, 0, 0, 0, 0, 0, 6, 1, 40, 934, 2013, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1481, 2, 341, 1102, 34, 0, 55, 0, 351, 188, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3141, 5963, 3858, 4594, 4882, 3756, 3020, 3554] avg=4096.0000 max=5963 max_rank=1 min=3020 min_rank=6 imbalance_ratio=1.455811 max_load_reduction=26805 max_load_reduction_pct=0.818024 moved_to_echo=15212 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[384, 128, 8448, 128, 0, 128, 7296, 128, 16256, 640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 512, 9216, 8832, 2688, 2176, 128, 9728, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[9216, 17152, 1024, 0, 0, 0, 5632, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[384, 128, 4736, 8448, 19456, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[16640, 128, 6144, 5248, 384, 0, 1024, 128, 2816, 1152, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 7168, 0, 0, 4864, 17664, 2048, 1408, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[10240, 128, 1280, 0, 128, 1664, 128, 18944, 1024, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 5504, 1024, 2944, 128, 21504, 2048, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 1, 0, 44, 0, 2387, 0, 13, 2, 0, 586, 0, 200, 50, 20, 2360, 20, 32, 0, 0, 0, 0, 0, 0, 3830, 6, 2, 0, 0, 717, 1686, 654, 0, 398, 2953, 3519, 0, 0, 0, 0, 0, 180, 0, 0, 0, 302, 0, 2024, 199, 0, 77, 38, 0, 45, 4091, 4, 0, 12, 3178, 3137, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 1, 0, 23, 0, 3394, 0, 13, 17, 0, 1, 0, 86, 1, 5, 269, 330, 2, 0, 0, 0, 1, 0, 0, 3135, 701, 430, 0, 1, 0, 1883, 2101, 0, 651, 3664, 2841, 1, 0, 0, 0, 0, 439, 2, 0, 0, 1286, 34, 3035, 312, 6, 141, 5, 0, 0, 2974, 1, 0, 0, 2601, 2377, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 10, 2, 31, 0, 121, 0, 3945, 0, 1, 88, 0, 338, 1, 289, 2941, 636, 3273, 725, 5, 0, 0, 0, 3, 4, 9, 3845, 0, 17, 0, 0, 1, 107, 1275, 0, 52, 1936, 1417, 2, 0, 0, 0, 11, 100, 1, 0, 0, 598, 0, 3534, 30, 0, 0, 0, 0, 40, 3845, 1, 0, 1, 1558, 1975, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 63, 0, 73, 5, 22, 0, 3223, 0, 0, 0, 12, 54, 1, 1172, 644, 96, 2006, 87, 1, 0, 0, 0, 0, 0, 3, 4028, 0, 2980, 0, 0, 1, 780, 1525, 0, 218, 3597, 146, 0, 0, 0, 0, 3, 129, 0, 0, 0, 479, 148, 3670, 107, 0, 33, 275, 0, 17, 1926, 0, 0, 0, 1776, 3468, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 249, 1, 1, 0, 301, 0, 2007, 0, 48, 1, 29, 489, 0, 3, 82, 127, 1420, 4, 4, 0, 0, 0, 1, 0, 0, 3353, 41, 2, 0, 0, 11, 1135, 3355, 0, 149, 2552, 2907, 0, 0, 0, 0, 0, 433, 0, 0, 0, 2134, 17, 3510, 16, 0, 1, 3, 0, 6, 4005, 0, 0, 27, 2095, 2248, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 34, 0, 2254, 0, 0, 56, 1, 97, 0, 23, 0, 26, 3497, 131, 18, 0, 0, 0, 0, 0, 0, 3118, 0, 0, 0, 0, 14, 705, 1374, 0, 2552, 2058, 1690, 0, 0, 0, 0, 0, 1848, 0, 0, 0, 2570, 677, 1312, 212, 1, 0, 34, 0, 2, 4065, 0, 0, 5, 3035, 1359, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[5, 0, 1, 0, 3, 0, 867, 0, 2199, 0, 4, 1, 0, 14, 0, 584, 543, 458, 358, 19, 357, 0, 0, 0, 0, 0, 0, 2256, 1, 6, 0, 0, 17, 442, 1167, 0, 192, 2562, 2627, 0, 0, 0, 0, 3, 1670, 0, 0, 0, 2879, 6, 3098, 802, 10, 0, 18, 0, 46, 3977, 0, 0, 3, 1786, 3787, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 49, 0, 0, 92, 0, 1701, 0, 1736, 0, 8, 0, 0, 175, 0, 48, 55, 355, 886, 95, 63, 0, 0, 1, 2, 0, 0, 853, 11, 2, 0, 0, 50, 617, 3915, 0, 1150, 2871, 549, 0, 0, 4, 0, 22, 190, 0, 0, 0, 2456, 0, 3785, 719, 1, 3, 0, 0, 0, 4082, 0, 7, 11, 2331, 3873, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 1, 0, 44, 0, 4091, 13, 0, 0, 0, 0, 0, 0, 2387, 0, 13, 2, 0, 586, 0, 200, 312, 489, 0, 0, 0, 0, 0, 0, 50, 20, 2360, 20, 32, 0, 0, 0, 1387, 646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3830, 6, 2, 0, 0, 348, 0, 0, 0, 0, 0, 0, 0, 717, 1686, 654, 0, 398, 0, 868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 0, 0, 0, 2940, 1264, 0, 0, 0, 0, 0, 0, 302, 0, 1364, 199, 0, 77, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 4, 0, 12, 3178, 2002, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4150, 3989, 4515, 4186, 4323, 4384, 1980, 5241] avg=4096.0000 max=5241 max_rank=7 min=1980 min_rank=6 imbalance_ratio=1.279541 max_load_reduction=27527 max_load_reduction_pct=0.840057 moved_to_echo=15928 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 10, 2, 31, 0, 121, 0, 3845, 8, 0, 0, 0, 0, 0, 0, 3945, 0, 1, 88, 0, 338, 1, 289, 545, 308, 0, 0, 0, 0, 0, 0, 2941, 636, 3273, 725, 5, 0, 0, 0, 558, 406, 0, 0, 0, 0, 0, 0, 3, 4, 9, 3845, 0, 17, 0, 0, 608, 0, 0, 0, 0, 0, 0, 0, 1, 107, 1275, 0, 52, 0, 351, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 100, 1, 0, 0, 1928, 508, 0, 0, 0, 0, 0, 0, 598, 0, 2381, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 1, 0, 1, 1558, 1261, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4017, 5515, 8544, 4486, 1788, 2548, 3009, 2861] avg=4096.0000 max=8544 max_rank=2 min=1788 min_rank=4 imbalance_ratio=2.085938 max_load_reduction=24224 max_load_reduction_pct=0.739258 moved_to_echo=10206 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 1, 0, 23, 0, 2974, 16, 0, 0, 0, 0, 0, 0, 3394, 0, 13, 17, 0, 1, 0, 86, 468, 370, 0, 0, 0, 0, 0, 0, 1, 5, 269, 330, 2, 0, 0, 0, 1119, 489, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3135, 701, 430, 0, 1, 522, 0, 0, 0, 0, 0, 0, 0, 0, 1883, 2101, 0, 651, 0, 702, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 439, 2, 0, 0, 3648, 1020, 0, 0, 0, 0, 0, 0, 1286, 34, 2045, 312, 6, 141, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2601, 1518, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3018, 4349, 2215, 4790, 5338, 5109, 3829, 4120] avg=4096.0000 max=5338 max_rank=4 min=2215 min_rank=2 imbalance_ratio=1.303223 max_load_reduction=27430 max_load_reduction_pct=0.837097 moved_to_echo=18396 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 63, 0, 73, 5, 22, 0, 1926, 16, 0, 0, 0, 0, 0, 0, 3223, 0, 0, 0, 12, 54, 1, 1172, 566, 540, 0, 0, 0, 0, 0, 0, 644, 96, 2006, 87, 1, 0, 0, 0, 57, 714, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4028, 0, 2980, 0, 0, 631, 0, 0, 0, 0, 0, 0, 0, 1, 780, 1525, 0, 218, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 129, 0, 0, 0, 3581, 52, 0, 0, 0, 0, 0, 0, 479, 148, 2473, 107, 0, 33, 275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 1776, 2214, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2105, 5568, 3605, 7642, 2561, 3765, 3515, 4007] avg=4096.0000 max=7642 max_rank=3 min=2105 min_rank=0 imbalance_ratio=1.865723 max_load_reduction=25126 max_load_reduction_pct=0.766785 moved_to_echo=13848 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 34, 0, 4065, 9, 0, 0, 0, 0, 0, 0, 2254, 0, 0, 56, 1, 97, 0, 23, 202, 211, 0, 0, 0, 0, 0, 0, 0, 26, 3497, 131, 18, 0, 0, 0, 666, 280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3118, 0, 0, 0, 0, 225, 0, 0, 0, 0, 0, 0, 0, 14, 705, 1374, 0, 2552, 0, 417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1848, 0, 0, 0, 2049, 607, 0, 0, 0, 0, 0, 0, 2570, 677, 885, 212, 1, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 5, 3035, 868, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 2844, 4618, 3343, 5062, 4504, 4379, 3910] avg=4096.0000 max=5062 max_rank=4 min=2844 min_rank=1 imbalance_ratio=1.235840 max_load_reduction=27706 max_load_reduction_pct=0.845520 moved_to_echo=17855 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 249, 1, 1, 0, 301, 0, 4005, 12, 0, 0, 0, 0, 0, 0, 2007, 0, 48, 1, 29, 489, 0, 3, 542, 350, 0, 0, 0, 0, 0, 0, 82, 127, 1420, 4, 4, 0, 0, 0, 1145, 463, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3353, 41, 2, 0, 0, 604, 0, 0, 0, 0, 0, 0, 0, 11, 1135, 3355, 0, 149, 0, 718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 433, 0, 0, 0, 2540, 1044, 0, 0, 0, 0, 0, 0, 2134, 17, 2364, 16, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 27, 2095, 1435, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4569, 3469, 3245, 4001, 5368, 4017, 4535, 3564] avg=4096.0000 max=5368 max_rank=4 min=3245 min_rank=2 imbalance_ratio=1.310547 max_load_reduction=27400 max_load_reduction_pct=0.836182 moved_to_echo=17484 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 49, 0, 0, 92, 0, 1701, 0, 4082, 12, 0, 0, 0, 0, 0, 0, 1736, 0, 8, 0, 0, 175, 0, 48, 584, 604, 0, 0, 0, 0, 0, 0, 55, 355, 886, 95, 63, 0, 0, 1, 216, 797, 0, 0, 0, 0, 0, 0, 2, 0, 0, 853, 11, 2, 0, 0, 651, 0, 0, 0, 0, 0, 0, 0, 50, 617, 3915, 0, 1150, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 22, 190, 0, 0, 0, 2859, 197, 0, 0, 0, 0, 0, 0, 2456, 0, 2550, 719, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 11, 2331, 2472, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5936, 3155, 2468, 1519, 5868, 3272, 5729, 4821] avg=4096.0000 max=5936 max_rank=0 min=1519 min_rank=3 imbalance_ratio=1.449219 max_load_reduction=26832 max_load_reduction_pct=0.818848 moved_to_echo=19690 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 1, 0, 3, 0, 867, 0, 3977, 12, 0, 0, 0, 0, 0, 0, 2199, 0, 4, 1, 0, 14, 0, 584, 483, 594, 0, 0, 0, 0, 0, 0, 543, 458, 358, 19, 357, 0, 0, 0, 1039, 784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2256, 1, 6, 0, 0, 538, 0, 0, 0, 0, 0, 0, 0, 17, 442, 1167, 0, 192, 0, 642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1670, 0, 0, 0, 2550, 946, 0, 0, 0, 0, 0, 0, 2879, 6, 2077, 802, 10, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 3, 1786, 2409, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4865, 3879, 3558, 2801, 2460, 5169, 5792, 4244] avg=4096.0000 max=5792 max_rank=6 min=2460 min_rank=4 imbalance_ratio=1.414062 max_load_reduction=26976 max_load_reduction_pct=0.823242 moved_to_echo=17665 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4352, 1792, 14080, 1536, 512, 0, 0, 128, 6272, 4608, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[896, 7424, 15488, 0, 5376, 0, 3968, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[21248, 0, 128, 256, 128, 1792, 128, 2432, 3712, 3584, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[12800, 896, 16256, 2432, 128, 256, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 0, 128, 4992, 128, 0, 0, 22144, 5760, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[256, 0, 128, 128, 128, 18432, 14208, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 24448, 768, 3456, 0, 128, 4224, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 384, 128, 256, 128, 3200, 0, 29056, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 138, 0, 20, 0, 3157, 3648, 0, 0, 304, 0, 4096, 0, 0, 54, 61, 0, 0, 10, 0, 0, 0, 116, 3893, 0, 1450, 0, 904, 0, 0, 3718, 0, 0, 0, 0, 0, 0, 0, 1, 4059, 0, 0, 0, 104, 0, 2750, 0, 0, 131, 0, 89, 0, 0, 0, 0, 0, 4050, 0, 0, 15] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[60, 0, 0, 0, 0, 0, 0, 51, 0, 1546, 3906, 0, 0, 328, 0, 3050, 0, 0, 844, 269, 0, 13, 1, 0, 3, 0, 70, 4042, 0, 2917, 0, 682, 0, 0, 463, 0, 0, 0, 0, 0, 17, 0, 56, 3615, 22, 0, 0, 582, 0, 3919, 121, 0, 1690, 0, 39, 0, 0, 1, 0, 6, 4086, 353, 0, 16] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 22, 0, 177, 0, 936, 3914, 0, 0, 2224, 0, 3550, 0, 0, 1227, 925, 0, 1, 178, 2, 0, 0, 61, 3327, 11, 4063, 0, 101, 0, 0, 300, 0, 0, 4, 0, 1, 2, 0, 35, 3533, 2, 0, 0, 651, 0, 2254, 74, 0, 616, 0, 114, 0, 0, 0, 0, 0, 4096, 367, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[87, 5, 12, 0, 0, 0, 0, 49, 0, 256, 3222, 0, 0, 1510, 4, 4048, 0, 0, 1, 400, 4, 0, 9, 0, 0, 3, 0, 4056, 0, 1750, 0, 2451, 0, 0, 1278, 0, 0, 3, 3, 4, 48, 1, 580, 4066, 88, 1, 0, 1427, 0, 637, 0, 0, 1937, 0, 8, 1, 0, 0, 0, 0, 4078, 96, 3, 642] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[9, 0, 0, 0, 0, 0, 0, 0, 0, 594, 3093, 0, 0, 1491, 0, 4003, 0, 0, 225, 961, 0, 3, 4, 0, 0, 0, 0, 3870, 1, 1514, 0, 347, 0, 0, 170, 0, 0, 0, 0, 0, 11, 0, 47, 4085, 0, 0, 0, 3229, 0, 2795, 0, 0, 987, 0, 1, 0, 0, 0, 0, 0, 4091, 709, 0, 528] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[6, 0, 0, 0, 1, 4, 0, 7, 0, 1295, 3641, 0, 0, 3144, 0, 4009, 0, 0, 0, 293, 0, 2, 1, 0, 0, 0, 213, 3339, 0, 234, 0, 7, 0, 0, 1824, 0, 0, 3, 0, 0, 51, 0, 277, 4044, 0, 0, 0, 2925, 0, 2535, 3, 0, 67, 0, 303, 0, 0, 20, 0, 0, 4093, 1, 0, 426] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[4, 0, 0, 0, 0, 1, 0, 0, 0, 3326, 1373, 0, 0, 2540, 0, 3952, 0, 2, 388, 297, 0, 0, 0, 0, 0, 0, 72, 4093, 0, 387, 0, 1230, 0, 0, 155, 0, 4, 0, 0, 0, 0, 0, 306, 4046, 57, 0, 0, 95, 0, 2852, 21, 0, 2739, 0, 460, 0, 0, 0, 0, 0, 4096, 226, 0, 46] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 0, 16, 0, 7, 0, 3501, 4081, 0, 0, 722, 2, 3309, 0, 0, 1445, 1357, 4, 2, 100, 0, 0, 0, 315, 3271, 2, 289, 0, 0, 0, 0, 30, 0, 0, 0, 0, 3, 92, 0, 0, 3903, 23, 0, 0, 2619, 2, 2409, 18, 0, 813, 0, 6, 0, 0, 0, 0, 0, 4079, 343, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[9, 0, 0, 0, 0, 0, 0, 0, 4003, 238, 0, 0, 0, 0, 0, 0, 0, 594, 678, 0, 0, 1491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 225, 961, 0, 3, 4, 0, 1554, 1350, 165, 0, 0, 0, 0, 0, 0, 0, 0, 1758, 1, 1514, 0, 347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 170, 0, 0, 0, 0, 0, 2177, 762, 0, 0, 0, 0, 0, 0, 11, 0, 47, 2531, 0, 0, 0, 3229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2795, 0, 0, 987, 0, 1, 0, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3629, 709, 0, 528, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4250, 2763, 4262, 3620, 3109, 5818, 4080, 4866] avg=4096.0000 max=5818 max_rank=5 min=2763 min_rank=1 imbalance_ratio=1.420410 max_load_reduction=26950 max_load_reduction_pct=0.822449 moved_to_echo=17873 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 138, 0, 20, 4096, 281, 0, 0, 0, 0, 0, 0, 0, 3157, 800, 0, 0, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 61, 0, 0, 10, 0, 1544, 1358, 164, 0, 0, 0, 0, 0, 0, 0, 116, 1769, 0, 1450, 0, 904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3718, 0, 0, 0, 0, 0, 2567, 766, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2515, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2750, 0, 0, 131, 0, 89, 0, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3592, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4535, 4261, 3191, 4239, 7051, 2620, 3264, 3607] avg=4096.0000 max=7051 max_rank=4 min=2620 min_rank=5 imbalance_ratio=1.721436 max_load_reduction=25717 max_load_reduction_pct=0.784821 moved_to_echo=16542 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[60, 0, 0, 0, 0, 0, 0, 51, 3050, 300, 0, 0, 0, 0, 0, 0, 0, 1546, 857, 0, 0, 328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 844, 269, 0, 13, 1, 0, 1375, 1410, 165, 0, 0, 0, 0, 0, 3, 0, 70, 1837, 0, 2917, 0, 682, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 0, 0, 0, 0, 0, 2749, 795, 0, 0, 0, 0, 0, 0, 17, 0, 56, 2240, 22, 0, 0, 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3919, 121, 0, 1690, 0, 39, 0, 297, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 3624, 353, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3461, 2731, 4077, 5509, 4007, 2917, 6066, 4000] avg=4096.0000 max=6066 max_rank=6 min=2731 min_rank=1 imbalance_ratio=1.480957 max_load_reduction=26702 max_load_reduction_pct=0.814880 moved_to_echo=16990 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 22, 0, 177, 3550, 301, 0, 0, 0, 0, 0, 0, 0, 936, 858, 0, 0, 2224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1227, 925, 0, 1, 178, 2, 1344, 1160, 165, 0, 0, 0, 0, 0, 0, 0, 61, 1512, 11, 4063, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 0, 0, 4, 0, 1, 2755, 655, 0, 0, 0, 0, 0, 0, 2, 0, 35, 2189, 2, 0, 0, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2254, 74, 0, 616, 0, 114, 0, 298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3633, 367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4050, 4018, 5002, 5748, 3715, 2879, 3356, 4000] avg=4096.0000 max=5748 max_rank=3 min=2879 min_rank=5 imbalance_ratio=1.403320 max_load_reduction=27020 max_load_reduction_pct=0.824585 moved_to_echo=13950 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[87, 5, 12, 0, 0, 0, 0, 49, 4048, 248, 0, 0, 0, 0, 0, 0, 0, 256, 707, 0, 0, 1510, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 400, 4, 0, 9, 0, 1547, 1415, 165, 0, 0, 0, 0, 0, 0, 3, 0, 1843, 0, 1750, 0, 2451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1278, 0, 0, 3, 3, 4, 2267, 798, 0, 0, 0, 0, 0, 0, 48, 1, 580, 2519, 88, 1, 0, 1427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 637, 0, 0, 1937, 0, 8, 1, 296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3617, 96, 3, 642, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4449, 2477, 3541, 6047, 4353, 4664, 2879, 4358] avg=4096.0000 max=6047 max_rank=3 min=2477 min_rank=1 imbalance_ratio=1.476318 max_load_reduction=26721 max_load_reduction_pct=0.815460 moved_to_echo=16254 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 0, 0, 1, 4, 0, 7, 4009, 280, 0, 0, 0, 0, 0, 0, 0, 1295, 799, 0, 0, 3144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 0, 2, 1, 0, 1539, 1165, 165, 0, 0, 0, 0, 0, 0, 0, 213, 1517, 0, 234, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1824, 0, 0, 3, 0, 0, 2562, 657, 0, 0, 0, 0, 0, 0, 51, 0, 277, 2505, 0, 0, 0, 2925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2535, 3, 0, 67, 0, 303, 0, 297, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 3631, 1, 0, 426, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4307, 5238, 3165, 1971, 5046, 5758, 3205, 4078] avg=4096.0000 max=5758 max_rank=5 min=1971 min_rank=3 imbalance_ratio=1.405762 max_load_reduction=27010 max_load_reduction_pct=0.824280 moved_to_echo=18087 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 0, 16, 0, 7, 3309, 314, 0, 0, 0, 0, 0, 0, 0, 3501, 895, 0, 0, 722, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1445, 1357, 4, 2, 100, 0, 1485, 1141, 165, 0, 0, 0, 0, 0, 0, 0, 315, 1486, 2, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 3, 2872, 644, 0, 0, 0, 0, 0, 0, 92, 0, 0, 2418, 23, 0, 0, 2619, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2409, 18, 0, 813, 0, 6, 0, 296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3618, 343, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3650, 5120, 5699, 2092, 3549, 5152, 3544, 3962] avg=4096.0000 max=5699 max_rank=2 min=2092 min_rank=3 imbalance_ratio=1.391357 max_load_reduction=27069 max_load_reduction_pct=0.826080 moved_to_echo=16207 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 0, 0, 0, 1, 0, 0, 3952, 109, 0, 0, 0, 0, 0, 0, 0, 3326, 294, 0, 0, 2540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 388, 297, 0, 0, 0, 0, 1544, 1431, 169, 0, 0, 0, 0, 0, 0, 0, 72, 1853, 0, 387, 0, 1230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 0, 4, 0, 0, 0, 970, 809, 0, 0, 0, 0, 0, 0, 0, 0, 306, 2502, 57, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2852, 21, 0, 2739, 0, 460, 0, 302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3625, 226, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 6160, 3831, 3542, 1938, 2960, 6374, 3897] avg=4096.0000 max=6374 max_rank=6 min=1938 min_rank=4 imbalance_ratio=1.556152 max_load_reduction=26394 max_load_reduction_pct=0.805481 moved_to_echo=15169 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[0, 128, 4224, 4608, 128, 128, 384, 128, 12032, 10496, 1408, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 896, 13696, 128, 12672, 0, 5760, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 0, 8064, 0, 128, 128, 128, 128, 18944, 5888, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 14720, 5888, 0, 0, 12288, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 128, 1408, 19456, 256, 128, 0, 11648, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 20224, 256, 0, 9088, 0, 1024, 128, 2432, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 29056, 2176, 128, 1792, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[256, 128, 128, 0, 128, 256, 0, 384, 30080, 2176, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 4080, 0, 25, 569, 0, 0, 3, 3941, 11, 0, 0, 33, 1, 6, 0, 13, 0, 0, 4, 0, 9, 5, 3664, 1, 0, 0, 0, 3877, 15, 0, 1486, 0, 1, 0, 0, 0, 0, 145, 6, 0, 4058, 1, 1560, 0, 408, 1522, 0, 0, 0, 0, 1, 183, 0, 52, 0, 3042, 1, 0, 0, 0, 0, 6, 4039] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 4088, 0, 0, 2621, 0, 0, 0, 1963, 0, 6, 0, 305, 0, 0, 0, 0, 1, 0, 0, 0, 0, 5, 2453, 0, 0, 0, 0, 2095, 0, 0, 3563, 0, 0, 12, 0, 0, 0, 715, 0, 0, 3874, 12, 1088, 0, 3009, 648, 1, 0, 0, 0, 0, 52, 31, 2, 0, 2180, 0, 3, 0, 0, 0, 0, 4041] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 3676, 4, 27, 3238, 8, 0, 56, 1173, 0, 11, 0, 149, 1, 0, 0, 0, 0, 0, 0, 0, 0, 14, 3420, 0, 6, 0, 0, 4084, 0, 0, 1659, 4, 0, 25, 0, 0, 0, 541, 0, 0, 4088, 0, 2335, 0, 1594, 2434, 0, 0, 0, 1, 0, 204, 121, 21, 0, 46, 0, 0, 0, 0, 0, 0, 3828] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 4062, 0, 2, 1472, 0, 0, 0, 150, 0, 0, 0, 12, 0, 0, 0, 14, 0, 0, 0, 0, 0, 28, 2375, 0, 0, 0, 0, 4079, 0, 0, 178, 0, 0, 237, 0, 0, 0, 42, 0, 0, 4094, 0, 659, 0, 3347, 4085, 1, 0, 0, 0, 34, 24, 3, 189, 0, 3943, 0, 21, 0, 0, 0, 0, 3717] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 4080, 0, 0, 1148, 0, 0, 1, 1020, 0, 0, 0, 29, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 3701, 0, 0, 0, 0, 4004, 0, 0, 296, 0, 0, 4, 0, 0, 0, 1228, 0, 0, 4096, 0, 28, 0, 3501, 1474, 0, 0, 0, 0, 2, 63, 13, 0, 0, 3970, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 3613, 0, 0, 1497, 0, 0, 0, 2790, 0, 0, 0, 887, 0, 0, 0, 9, 0, 0, 0, 0, 0, 10, 3186, 0, 0, 0, 0, 3994, 0, 0, 356, 0, 0, 2, 0, 0, 0, 3, 0, 0, 4083, 0, 75, 0, 2108, 3151, 0, 0, 0, 1, 0, 2, 0, 0, 0, 2943, 0, 0, 0, 0, 0, 0, 4058] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 3845, 0, 0, 1545, 0, 0, 19, 229, 4, 0, 0, 2069, 0, 0, 0, 0, 32, 0, 0, 0, 0, 2, 2545, 0, 0, 0, 0, 3601, 0, 0, 2696, 0, 1, 0, 0, 0, 0, 650, 0, 0, 4096, 0, 930, 0, 462, 3637, 14, 0, 0, 0, 0, 1, 34, 50, 0, 2325, 0, 31, 0, 0, 0, 0, 3950] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 4082, 0, 0, 2639, 0, 0, 4, 826, 0, 0, 0, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2785, 0, 0, 0, 0, 3882, 0, 0, 985, 0, 0, 475, 0, 0, 7, 763, 0, 0, 4091, 0, 134, 0, 1616, 2254, 0, 0, 0, 0, 0, 0, 0, 65, 0, 3921, 0, 0, 0, 0, 0, 0, 4091] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2317, 0, 25, 569, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3941, 11, 0, 0, 33, 1, 6, 0, 707, 1044, 0, 0, 0, 0, 0, 0, 13, 0, 0, 4, 0, 9, 5, 3664, 1058, 30, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2819, 15, 0, 1486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 145, 6, 2727, 719, 108, 0, 0, 0, 0, 0, 0, 0, 1, 1560, 0, 408, 785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 183, 0, 52, 0, 3950, 0, 0, 0, 0, 0, 0, 0, 3042, 1, 0, 0, 0, 0, 6, 1312, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2914, 5743, 4783, 4321, 3706, 2754, 4186, 4361] avg=4096.0000 max=5743 max_rank=1 min=2754 min_rank=5 imbalance_ratio=1.402100 max_load_reduction=27025 max_load_reduction_pct=0.824738 moved_to_echo=15007 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2051, 0, 0, 1497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2790, 0, 0, 0, 887, 0, 0, 0, 1465, 925, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 10, 3186, 1090, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2904, 0, 0, 356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 0, 2739, 637, 108, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 2108, 1623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 3975, 0, 0, 0, 0, 0, 0, 0, 2943, 0, 0, 0, 0, 0, 0, 1319, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3548, 6067, 4358, 3260, 3489, 3806, 3978, 4262] avg=4096.0000 max=6067 max_rank=1 min=3260 min_rank=3 imbalance_ratio=1.481201 max_load_reduction=26701 max_load_reduction_pct=0.814850 moved_to_echo=15535 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2306, 0, 2, 1472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 0, 0, 0, 12, 0, 0, 0, 1899, 1040, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 28, 2375, 1114, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2965, 0, 0, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 237, 0, 0, 0, 42, 0, 2509, 716, 109, 0, 0, 0, 0, 0, 0, 0, 0, 659, 0, 3347, 2104, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 24, 3, 189, 0, 3985, 0, 0, 0, 0, 0, 0, 0, 3943, 0, 21, 0, 0, 0, 0, 1208, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3780, 3101, 3613, 3143, 3613, 6111, 4235, 5172] avg=4096.0000 max=6111 max_rank=5 min=3101 min_rank=1 imbalance_ratio=1.491943 max_load_reduction=26657 max_load_reduction_pct=0.813507 moved_to_echo=19131 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2087, 4, 27, 3238, 8, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 1173, 0, 11, 0, 149, 1, 0, 0, 1131, 941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 3420, 1115, 49, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 2969, 0, 0, 1659, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 25, 0, 0, 0, 541, 0, 2584, 648, 108, 0, 0, 0, 0, 0, 0, 0, 0, 2335, 0, 1594, 1254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 204, 121, 21, 0, 3980, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 1244, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5420, 3406, 4598, 4634, 3910, 5183, 4327, 1290] avg=4096.0000 max=5420 max_rank=0 min=1290 min_rank=7 imbalance_ratio=1.323242 max_load_reduction=27348 max_load_reduction_pct=0.834595 moved_to_echo=14710 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2321, 0, 0, 2621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1963, 0, 6, 0, 305, 0, 0, 0, 301, 1046, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 5, 2453, 572, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1523, 0, 0, 3563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 715, 0, 2728, 721, 102, 0, 0, 0, 0, 0, 0, 0, 12, 1088, 0, 3009, 334, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 31, 2, 0, 3772, 0, 0, 0, 0, 0, 0, 0, 2180, 0, 3, 0, 0, 0, 0, 1313, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4942, 3621, 3044, 5086, 4278, 4444, 3857, 3496] avg=4096.0000 max=5086 max_rank=3 min=3044 min_rank=2 imbalance_ratio=1.241699 max_load_reduction=27682 max_load_reduction_pct=0.844788 moved_to_echo=16075 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2317, 0, 0, 1148, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 0, 0, 0, 29, 0, 0, 0, 685, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 3701, 1093, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2911, 0, 0, 296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1228, 0, 2765, 719, 108, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 3501, 760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 63, 13, 0, 0, 3988, 0, 0, 0, 0, 0, 0, 0, 3970, 0, 0, 0, 0, 0, 0, 1331, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3466, 2778, 4837, 3207, 4824, 4289, 4066, 5301] avg=4096.0000 max=5301 max_rank=7 min=2778 min_rank=1 imbalance_ratio=1.294189 max_load_reduction=27467 max_load_reduction_pct=0.838226 moved_to_echo=18480 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2183, 0, 0, 1545, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 229, 4, 0, 0, 2069, 0, 0, 0, 1691, 984, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 2, 2545, 983, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2618, 0, 0, 2696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 650, 0, 2666, 678, 109, 0, 0, 0, 0, 0, 0, 0, 0, 930, 0, 462, 1873, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 34, 50, 0, 3987, 0, 0, 0, 0, 0, 0, 0, 2325, 0, 31, 0, 0, 0, 0, 1284, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3747, 4977, 3635, 5314, 4104, 3279, 4072, 3640] avg=4096.0000 max=5314 max_rank=3 min=3279 min_rank=5 imbalance_ratio=1.297363 max_load_reduction=27454 max_load_reduction_pct=0.837830 moved_to_echo=15095 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2308, 0, 0, 2639, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 826, 0, 0, 0, 148, 0, 0, 0, 1051, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2785, 1064, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2818, 0, 0, 985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 0, 0, 7, 763, 0, 2766, 724, 109, 0, 0, 0, 0, 0, 0, 0, 0, 134, 0, 1616, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 3982, 0, 0, 0, 0, 0, 0, 0, 3921, 0, 0, 0, 0, 0, 0, 1325, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4951, 3075, 3900, 3803, 4844, 2902, 4047, 5246] avg=4096.0000 max=5246 max_rank=7 min=2902 min_rank=5 imbalance_ratio=1.280762 max_load_reduction=27522 max_load_reduction_pct=0.839905 moved_to_echo=17039 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[12160, 128, 128, 0, 3712, 128, 128, 0, 8960, 8192, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 17920, 128, 128, 14848, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 768, 0, 0, 128, 4096, 128, 21504, 5632, 896, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[22400, 128, 128, 0, 0, 0, 128, 10368, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 128, 6912, 0, 16128, 9984, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 0, 21632, 128, 0, 11264, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 0, 128, 128, 24192, 8192, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 128, 128, 640, 256, 384, 0, 31744, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[14, 0, 0, 2427, 0, 3835, 0, 0, 0, 1, 6, 8, 0, 3567, 8, 0, 5, 2832, 1, 1126, 1, 0, 243, 0, 1980, 1797, 0, 0, 1, 3906, 0, 0, 2610, 8, 0, 0, 0, 104, 122, 0, 0, 0, 0, 0, 1405, 3374, 0, 14, 0, 3, 104, 0, 0, 0, 4, 3, 2285, 0, 0, 970, 0, 2, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 4018, 0, 2371, 0, 0, 0, 0, 0, 74, 0, 2752, 4, 0, 0, 3141, 4, 2891, 0, 0, 0, 0, 3076, 2566, 0, 0, 0, 3921, 0, 0, 112, 0, 0, 0, 0, 36, 5, 0, 0, 1, 0, 0, 1, 4092, 0, 619, 0, 0, 1487, 0, 0, 0, 0, 0, 1190, 0, 0, 407, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1, 0, 0, 36, 0, 3721, 5, 0, 0, 0, 0, 6, 0, 3384, 0, 0, 0, 1543, 16, 241, 0, 0, 50, 3, 802, 3796, 0, 0, 0, 4064, 0, 0, 65, 0, 0, 0, 0, 0, 46, 0, 4, 1, 0, 0, 235, 4095, 0, 2401, 0, 0, 3718, 0, 0, 0, 0, 0, 1746, 2, 0, 2787, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 944, 0, 3976, 2, 9, 1, 0, 0, 23, 0, 2682, 1, 1, 0, 3631, 38, 2128, 0, 6, 6, 13, 182, 2630, 26, 0, 0, 4090, 0, 0, 547, 0, 0, 0, 6, 244, 2, 14, 0, 27, 0, 0, 1272, 3915, 0, 762, 0, 0, 1246, 0, 0, 0, 0, 0, 2829, 0, 2, 1513, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 4058, 0, 3314, 0, 0, 4, 0, 0, 10, 0, 3651, 0, 0, 0, 1770, 154, 179, 0, 0, 2, 10, 272, 2221, 0, 0, 0, 4076, 0, 0, 177, 0, 0, 0, 0, 5, 1408, 0, 0, 0, 0, 0, 71, 4049, 0, 641, 0, 0, 26, 0, 0, 0, 0, 0, 2966, 0, 0, 3704, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 133, 0, 2439, 21, 0, 2, 0, 0, 178, 3, 238, 0, 0, 0, 3301, 28, 2804, 0, 0, 334, 0, 0, 3934, 2, 0, 0, 4085, 0, 0, 2571, 0, 0, 0, 0, 150, 4, 0, 0, 1, 0, 0, 647, 4041, 0, 2964, 0, 0, 265, 0, 0, 0, 0, 0, 541, 0, 0, 4082, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 3520, 0, 2338, 1, 0, 0, 0, 0, 86, 0, 3381, 1, 0, 0, 3756, 8, 1971, 0, 0, 118, 0, 80, 2194, 13, 0, 0, 4084, 0, 0, 2404, 0, 0, 0, 3, 1, 10, 0, 0, 15, 0, 16, 4, 4096, 0, 2290, 0, 0, 327, 0, 0, 0, 0, 0, 8, 0, 0, 2043, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 3950, 0, 0, 5, 0, 0, 63, 0, 3347, 1, 0, 0, 3804, 103, 2950, 0, 0, 0, 0, 146, 23, 37, 0, 0, 2972, 0, 0, 859, 0, 0, 0, 0, 0, 4, 0, 0, 30, 0, 0, 222, 4093, 0, 1469, 0, 0, 117, 0, 0, 0, 0, 0, 940, 0, 0, 3538, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4018, 0, 1234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 2752, 4, 0, 691, 369, 0, 0, 0, 0, 0, 0, 0, 2291, 4, 2891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3076, 2566, 0, 0, 0, 879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 0, 0, 0, 36, 5, 0, 1654, 768, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 2276, 0, 619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1487, 0, 0, 0, 0, 0, 3042, 162, 0, 0, 0, 0, 0, 0, 1190, 0, 0, 407, 0, 0, 0, 0, 159, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5252, 3890, 5186, 6521, 2575, 2897, 4691, 1756] avg=4096.0000 max=6521 max_rank=3 min=1756 min_rank=7 imbalance_ratio=1.592041 max_load_reduction=26247 max_load_reduction_pct=0.800995 moved_to_echo=11919 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[14, 0, 0, 2427, 0, 1994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 8, 0, 3567, 8, 0, 623, 598, 0, 0, 0, 0, 0, 0, 5, 2065, 1, 1126, 1, 0, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1980, 1797, 0, 0, 1, 876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2610, 8, 0, 0, 0, 104, 122, 0, 1364, 1243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1405, 1877, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 104, 0, 0, 0, 4, 3, 3030, 133, 0, 0, 0, 0, 0, 0, 2285, 0, 0, 970, 0, 2, 2, 0, 144, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4435, 4811, 3441, 4654, 5451, 3296, 3277, 3403] avg=4096.0000 max=5451 max_rank=4 min=3277 min_rank=6 imbalance_ratio=1.330811 max_load_reduction=27317 max_load_reduction_pct=0.833649 moved_to_echo=15427 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 944, 0, 2067, 2, 9, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 23, 0, 2682, 1, 1, 798, 620, 0, 0, 0, 0, 0, 0, 0, 2649, 38, 2128, 0, 6, 6, 13, 0, 0, 0, 0, 0, 0, 0, 0, 182, 2630, 26, 0, 0, 917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 0, 0, 0, 6, 244, 2, 14, 1582, 1289, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 1272, 2178, 0, 762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1246, 0, 0, 0, 0, 0, 3173, 155, 0, 0, 0, 0, 0, 0, 2829, 0, 2, 1513, 0, 0, 0, 0, 184, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3022, 4126, 4840, 3755, 3684, 4239, 4574, 4528] avg=4096.0000 max=4840 max_rank=2 min=3022 min_rank=0 imbalance_ratio=1.181641 max_load_reduction=27928 max_load_reduction_pct=0.852295 moved_to_echo=17025 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 36, 0, 1935, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 3384, 0, 0, 339, 580, 0, 0, 0, 0, 0, 0, 0, 1126, 16, 241, 0, 0, 50, 3, 0, 0, 0, 0, 0, 0, 0, 0, 802, 3796, 0, 0, 0, 911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 46, 0, 1655, 1206, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 235, 2278, 0, 2401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3718, 0, 0, 0, 0, 0, 3153, 162, 0, 0, 0, 0, 0, 0, 1746, 2, 0, 2787, 0, 0, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1977, 4309, 1436, 5509, 2972, 4919, 7033, 4613] avg=4096.0000 max=7033 max_rank=6 min=1436 min_rank=2 imbalance_ratio=1.717041 max_load_reduction=25735 max_load_reduction_pct=0.785370 moved_to_echo=19537 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 133, 0, 1269, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 178, 3, 238, 0, 0, 726, 380, 0, 0, 0, 0, 0, 0, 0, 2407, 28, 2804, 0, 0, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3934, 2, 0, 0, 916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2571, 0, 0, 0, 0, 150, 4, 0, 1633, 790, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 647, 2248, 0, 2964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 265, 0, 0, 0, 0, 0, 3169, 160, 0, 0, 0, 0, 0, 0, 541, 0, 0, 4082, 0, 0, 0, 0, 168, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1423, 1527, 5573, 4852, 5148, 5860, 3594, 4791] avg=4096.0000 max=5860 max_rank=5 min=1423 min_rank=0 imbalance_ratio=1.430664 max_load_reduction=26908 max_load_reduction_pct=0.821167 moved_to_echo=19393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3520, 0, 1216, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 3381, 1, 0, 826, 364, 0, 0, 0, 0, 0, 0, 0, 2739, 8, 1971, 0, 0, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 2194, 13, 0, 0, 916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2404, 0, 0, 0, 3, 1, 10, 0, 1656, 758, 0, 0, 0, 0, 0, 0, 0, 15, 0, 16, 4, 2278, 0, 2290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 327, 0, 0, 0, 0, 0, 3168, 162, 0, 0, 0, 0, 0, 0, 8, 0, 0, 2043, 0, 0, 0, 0, 191, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4737, 4658, 4836, 3203, 4832, 4603, 3657, 2242] avg=4096.0000 max=4836 max_rank=2 min=2242 min_rank=7 imbalance_ratio=1.180664 max_load_reduction=27932 max_load_reduction_pct=0.852417 moved_to_echo=15334 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 2054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 63, 0, 3347, 1, 0, 837, 616, 0, 0, 0, 0, 0, 0, 0, 2774, 103, 2950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 23, 37, 0, 0, 666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 859, 0, 0, 0, 0, 0, 4, 0, 1654, 1280, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 222, 2277, 0, 1469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 0, 0, 0, 0, 0, 2306, 162, 0, 0, 0, 0, 0, 0, 940, 0, 0, 3538, 0, 0, 0, 0, 193, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6149, 4869, 5827, 872, 3797, 3998, 2585, 4671] avg=4096.0000 max=6149 max_rank=0 min=872 min_rank=3 imbalance_ratio=1.501221 max_load_reduction=26619 max_load_reduction_pct=0.812347 moved_to_echo=15051 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4058, 0, 1715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 10, 0, 3651, 0, 0, 392, 521, 0, 0, 0, 0, 0, 0, 0, 1284, 154, 179, 0, 0, 2, 10, 0, 0, 0, 0, 0, 0, 0, 0, 272, 2221, 0, 0, 0, 909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177, 0, 0, 0, 0, 5, 1408, 0, 1641, 1078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 2244, 0, 641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 3167, 164, 0, 0, 0, 0, 0, 0, 2966, 0, 0, 3704, 0, 0, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5773, 4578, 1629, 3402, 4309, 2956, 3357, 6764] avg=4096.0000 max=6764 max_rank=7 min=1629 min_rank=2 imbalance_ratio=1.651367 max_load_reduction=26004 max_load_reduction_pct=0.793579 moved_to_echo=17386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[9472, 128, 0, 0, 128, 640, 1664, 128, 12928, 8448, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[6656, 19200, 128, 0, 128, 7040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 17408, 384, 14336, 128, 128, 768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 3968, 17664, 0, 11264, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 0, 19328, 0, 13568, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[12544, 128, 128, 19072, 0, 128, 128, 0, 1280, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 512, 128, 23040, 128, 128, 5248, 4096, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 7296, 0, 0, 0, 128, 128, 24320, 1280, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 2915, 0, 0, 62, 0, 0, 0, 0, 0, 0, 1787, 4072, 0, 1614, 0, 0, 0, 0, 0, 1172, 0, 240, 0, 0, 1, 4093, 0, 2381, 0, 3859, 155, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 1710, 0, 0, 0, 2141, 162, 0, 0, 14, 0, 0, 2040, 0, 0, 0, 0, 0, 0, 0, 253, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 648, 4095, 0, 1307, 0, 0, 0, 0, 0, 2271, 0, 118, 0, 0, 52, 4096, 0, 2897, 0, 4095, 5, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 1294, 0, 0, 0, 93, 58, 0, 0, 5, 0, 0, 1738, 0, 0, 0, 0, 0, 0, 0, 1876, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 2665, 0, 1, 3, 0, 0, 0, 0, 0, 0, 343, 3176, 0, 744, 0, 0, 0, 0, 0, 133, 0, 1516, 0, 0, 380, 4094, 0, 3311, 1, 4071, 1485, 0, 0, 0, 4076, 0, 7, 0, 0, 12, 547, 0, 0, 0, 7, 41, 4, 0, 3, 1, 0, 3426, 0, 0, 0, 0, 0, 0, 0, 2721, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 3808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 370, 3259, 0, 268, 2, 0, 0, 0, 0, 2795, 0, 27, 0, 0, 2, 4088, 0, 1431, 0, 4057, 982, 0, 0, 0, 3808, 0, 0, 0, 0, 0, 2817, 0, 0, 0, 642, 13, 0, 0, 0, 0, 0, 2269, 0, 0, 0, 0, 0, 0, 0, 2130, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 2672, 0, 0, 4, 0, 0, 0, 0, 0, 0, 120, 2252, 0, 102, 10, 0, 0, 0, 0, 913, 2, 316, 0, 0, 56, 4089, 0, 3940, 0, 4052, 469, 0, 0, 0, 4081, 0, 0, 1, 0, 0, 3850, 0, 0, 0, 918, 764, 0, 0, 0, 0, 0, 108, 5, 0, 0, 0, 0, 0, 0, 4044, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[47, 0, 3117, 0, 1, 19, 0, 0, 8, 0, 6, 3, 14, 3797, 0, 1447, 0, 0, 0, 0, 0, 3470, 27, 4, 1, 0, 0, 4087, 0, 3125, 0, 4089, 1202, 2, 9, 0, 4085, 10, 4, 0, 0, 0, 2341, 0, 17, 2, 709, 102, 0, 4, 19, 0, 0, 709, 31, 3, 0, 15, 5, 0, 0, 230, 7, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 3679, 0, 0, 0, 0, 0, 0, 0, 0, 1, 371, 3677, 0, 2311, 3, 0, 0, 0, 0, 376, 0, 523, 0, 0, 68, 4088, 2, 3662, 0, 3707, 855, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 3088, 0, 0, 0, 33, 780, 4, 0, 6, 0, 0, 1204, 0, 0, 0, 0, 0, 0, 0, 226, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 3899, 0, 0, 33, 0, 0, 0, 0, 0, 0, 47, 3883, 0, 3067, 0, 0, 0, 0, 0, 348, 0, 553, 0, 0, 0, 4096, 0, 3236, 0, 4043, 56, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 338, 0, 0, 0, 197, 2, 0, 0, 1, 0, 0, 1923, 0, 0, 0, 0, 0, 0, 0, 2962, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2915, 0, 0, 62, 0, 0, 613, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1787, 2627, 0, 1614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1172, 0, 240, 2166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2381, 0, 994, 0, 0, 0, 0, 0, 0, 0, 0, 155, 1, 0, 0, 3483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1710, 0, 0, 0, 2141, 162, 1313, 118, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 2040, 0, 0, 2408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 253, 0, 0, 1685, 581, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3722, 6028, 3578, 3376, 3639, 5444, 4462, 2519] avg=4096.0000 max=6028 max_rank=1 min=2519 min_rank=7 imbalance_ratio=1.471680 max_load_reduction=26740 max_load_reduction_pct=0.816040 moved_to_echo=16064 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2665, 0, 1, 3, 0, 0, 610, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 343, 2049, 0, 744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 133, 0, 1516, 2285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 380, 0, 0, 3311, 1, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1485, 0, 0, 0, 3466, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 547, 0, 0, 0, 7, 41, 1024, 125, 0, 0, 0, 0, 0, 0, 4, 0, 3, 1, 0, 3426, 0, 0, 2407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2721, 0, 0, 1687, 613, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3382, 3136, 3934, 4740, 4958, 1756, 5841, 5021] avg=4096.0000 max=5841 max_rank=6 min=1756 min_rank=5 imbalance_ratio=1.426025 max_load_reduction=26927 max_load_reduction_pct=0.821747 moved_to_echo=17576 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3899, 0, 0, 33, 0, 0, 611, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 2505, 0, 3067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 348, 0, 553, 2269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3236, 0, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 3473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, 197, 2, 1252, 124, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1923, 0, 0, 2409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2962, 0, 0, 1687, 609, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4669, 5619, 3170, 4277, 3529, 1913, 4333, 5258] avg=4096.0000 max=5619 max_rank=1 min=1913 min_rank=5 imbalance_ratio=1.371826 max_load_reduction=27149 max_load_reduction_pct=0.828522 moved_to_echo=15033 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4070, 0, 0, 0, 0, 0, 606, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 648, 2642, 0, 1307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2271, 0, 118, 2298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 2897, 0, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 3444, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1294, 0, 0, 0, 93, 58, 1320, 126, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 1738, 0, 0, 2409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1876, 0, 0, 1687, 617, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4809, 4597, 4687, 4003, 3449, 2891, 4152, 4180] avg=4096.0000 max=4809 max_rank=0 min=2891 min_rank=5 imbalance_ratio=1.174072 max_load_reduction=27959 max_load_reduction_pct=0.853241 moved_to_echo=14672 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2672, 0, 0, 4, 0, 0, 611, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 1453, 0, 102, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 913, 2, 316, 2274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 3940, 0, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 469, 0, 0, 0, 3470, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3850, 0, 0, 0, 918, 764, 726, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 5, 0, 2404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 0, 1685, 610, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3360, 1675, 3515, 5040, 3940, 6382, 2517, 6339] avg=4096.0000 max=6382 max_rank=5 min=1675 min_rank=1 imbalance_ratio=1.558105 max_load_reduction=26386 max_load_reduction_pct=0.805237 moved_to_echo=19178 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 3679, 0, 0, 0, 0, 0, 613, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 371, 2371, 0, 2311, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 376, 0, 523, 2081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 2, 3662, 0, 954, 0, 0, 0, 0, 0, 0, 0, 0, 855, 0, 0, 0, 3483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3088, 0, 0, 0, 33, 780, 1186, 114, 0, 0, 0, 0, 0, 0, 4, 0, 6, 0, 0, 1204, 0, 0, 2405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 226, 0, 1, 1683, 558, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4413, 5054, 2983, 4686, 4338, 5207, 3619, 2468] avg=4096.0000 max=5207 max_rank=5 min=2468 min_rank=7 imbalance_ratio=1.271240 max_load_reduction=27561 max_load_reduction_pct=0.841095 moved_to_echo=15632 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[47, 0, 3117, 0, 1, 19, 0, 0, 612, 123, 0, 0, 0, 0, 0, 0, 8, 0, 6, 3, 14, 2450, 0, 1447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3470, 27, 4, 2295, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3125, 0, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1202, 2, 9, 0, 3473, 10, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2341, 0, 17, 2, 709, 102, 1224, 125, 0, 0, 0, 0, 0, 0, 0, 4, 19, 0, 0, 709, 31, 3, 2404, 0, 0, 0, 0, 0, 0, 0, 0, 15, 5, 0, 0, 230, 7, 0, 1683, 616, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3919, 3928, 5796, 4179, 4700, 4520, 3170, 2556] avg=4096.0000 max=5796 max_rank=2 min=2556 min_rank=7 imbalance_ratio=1.415039 max_load_reduction=26972 max_load_reduction_pct=0.823120 moved_to_echo=14946 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3808, 0, 0, 0, 0, 0, 575, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 370, 2093, 0, 268, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2795, 0, 27, 2281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1431, 0, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 982, 0, 0, 0, 3233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2817, 0, 0, 0, 642, 13, 1055, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2269, 0, 0, 2405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2130, 0, 0, 1683, 614, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4494, 2731, 5105, 2467, 4215, 4655, 4674, 4427] avg=4096.0000 max=5105 max_rank=2 min=2467 min_rank=3 imbalance_ratio=1.246338 max_load_reduction=27663 max_load_reduction_pct=0.844208 moved_to_echo=17971 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 0, 14464, 128, 128, 13568, 4864, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 16000, 0, 128, 128, 4864, 2048, 9216, 1024, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 0, 11520, 128, 3328, 18048, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 640, 0, 128, 24064, 128, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 3712, 18304, 0, 10880, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 0, 13440, 128, 128, 19328, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 26880, 0, 128, 128, 0, 0, 4864, 1024, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[5248, 128, 128, 0, 27648, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 2, 0, 5, 152, 0, 0, 0, 0, 0, 0, 4051, 0, 0, 0, 0, 0, 2, 0, 0, 3922, 0, 0, 150, 27, 817, 0, 2, 0, 0, 0, 0, 650, 4010, 4093, 988, 0, 0, 0, 552, 0, 0, 0, 1809, 3784, 0, 691, 0, 0, 2, 0, 0, 2432, 0, 0, 3824, 0, 0, 0, 799, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 449, 0, 0, 870, 6, 0, 0, 5, 1, 0, 0, 3958, 0, 0, 0, 0, 0, 65, 0, 0, 4032, 0, 0, 30, 3, 3302, 0, 0, 0, 0, 0, 0, 590, 3576, 3543, 0, 0, 0, 4, 2364, 0, 0, 0, 1430, 2765, 0, 1736, 0, 0, 0, 0, 41, 2973, 0, 0, 1022, 0, 3, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 16, 0, 0, 8, 305, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3876, 0, 0, 0, 70, 1257, 0, 0, 0, 0, 0, 0, 1684, 2928, 4073, 55, 0, 0, 0, 2722, 0, 0, 0, 2141, 3445, 0, 773, 0, 0, 0, 0, 0, 847, 0, 0, 3248, 0, 0, 0, 1219, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[2, 0, 2, 0, 0, 0, 26, 43, 0, 2, 16, 0, 4, 13, 4057, 0, 7, 3, 19, 0, 4, 0, 0, 4089, 0, 6, 434, 228, 330, 4, 16, 2, 0, 0, 0, 604, 2777, 4064, 366, 20, 15, 0, 3194, 0, 0, 1, 6, 3404, 0, 137, 0, 0, 0, 1, 0, 3214, 0, 0, 3987, 16, 0, 2, 1653, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 0, 130, 579, 0, 0, 0, 0, 0, 0, 4034, 0, 0, 0, 0, 0, 0, 0, 4, 3856, 0, 0, 15, 34, 531, 0, 0, 0, 0, 1, 0, 1897, 4095, 4085, 1258, 0, 0, 0, 3682, 0, 0, 0, 1471, 3918, 0, 451, 0, 0, 3, 0, 2, 1775, 0, 0, 888, 0, 0, 0, 55, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 26, 0, 0, 2027, 10, 0, 0, 0, 0, 0, 0, 3891, 0, 0, 0, 0, 0, 0, 0, 1, 4087, 0, 0, 40, 49, 1389, 0, 0, 0, 0, 0, 0, 72, 1742, 3930, 184, 2, 0, 0, 3831, 0, 0, 0, 206, 3585, 0, 3065, 0, 0, 10, 0, 0, 2598, 0, 0, 1260, 0, 0, 0, 747, 16] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 9, 0, 0, 3504, 351, 23, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 7, 0, 0, 4080, 9, 0, 40, 32, 1582, 0, 0, 0, 0, 0, 0, 10, 2010, 2352, 33, 0, 0, 2, 2438, 0, 0, 0, 1473, 4080, 0, 2305, 0, 0, 0, 0, 0, 3756, 0, 0, 580, 0, 1, 0, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 310, 0, 0, 133, 184, 1, 0, 55, 0, 0, 0, 4010, 0, 0, 0, 0, 0, 7, 0, 9, 3975, 0, 0, 2, 150, 2565, 0, 5, 0, 0, 0, 0, 2231, 334, 4056, 33, 0, 0, 26, 3993, 0, 0, 0, 481, 3950, 0, 1029, 0, 0, 0, 0, 0, 3857, 0, 0, 990, 0, 0, 0, 376, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 2, 0, 5, 152, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3922, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 27, 817, 0, 2, 0, 1789, 808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 650, 4010, 85, 988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 0, 0, 0, 1809, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 691, 0, 0, 2, 0, 0, 2432, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3824, 0, 0, 0, 799, 0, 1578, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3363, 4111, 4018, 3593, 5733, 2485, 3264, 6201] avg=4096.0000 max=6201 max_rank=7 min=2485 min_rank=5 imbalance_ratio=1.513916 max_load_reduction=26567 max_load_reduction_pct=0.810760 moved_to_echo=17683 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 449, 0, 0, 870, 6, 2770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 0, 3958, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 4032, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 3, 3302, 0, 0, 0, 1307, 700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 590, 3576, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2364, 0, 0, 0, 1430, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1736, 0, 0, 0, 0, 41, 2973, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 0, 3, 0, 0, 0, 1153, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4008, 4166, 5342, 4239, 3888, 4852, 2178] avg=4096.0000 max=5342 max_rank=3 min=2178 min_rank=7 imbalance_ratio=1.304199 max_load_reduction=27426 max_load_reduction_pct=0.836975 moved_to_echo=15157 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 2, 0, 0, 0, 26, 43, 3177, 0, 0, 0, 0, 0, 0, 0, 0, 2, 16, 0, 4, 13, 4057, 0, 54, 0, 0, 0, 0, 0, 0, 0, 7, 3, 19, 0, 4, 0, 0, 4089, 85, 0, 0, 0, 0, 0, 0, 0, 0, 6, 434, 228, 330, 4, 16, 2, 1609, 802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 604, 2777, 85, 366, 20, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 3194, 0, 0, 1, 6, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 0, 1, 0, 3214, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3987, 16, 0, 2, 1653, 0, 1419, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3250, 4146, 4207, 3431, 3852, 3328, 3477, 7077] avg=4096.0000 max=7077 max_rank=7 min=3250 min_rank=0 imbalance_ratio=1.727783 max_load_reduction=25691 max_load_reduction_pct=0.784027 moved_to_echo=17734 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 16, 0, 0, 8, 305, 3184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3876, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 1257, 0, 0, 0, 1629, 804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1684, 2928, 85, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2722, 0, 0, 0, 2141, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 773, 0, 0, 0, 0, 0, 847, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3248, 0, 0, 0, 1219, 5, 1437, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3513, 4151, 3962, 3760, 4752, 4974, 1747, 5909] avg=4096.0000 max=5909 max_rank=7 min=1747 min_rank=6 imbalance_ratio=1.442627 max_load_reduction=26859 max_load_reduction_pct=0.819672 moved_to_echo=17382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 26, 0, 0, 2027, 10, 3073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3891, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4087, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 49, 1389, 0, 0, 0, 1695, 776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 1742, 81, 184, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3831, 0, 0, 0, 206, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3065, 0, 0, 10, 0, 0, 2598, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1260, 0, 0, 0, 747, 16, 1495, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5136, 3948, 4177, 3949, 2081, 4154, 5805, 3518] avg=4096.0000 max=5805 max_rank=6 min=2081 min_rank=4 imbalance_ratio=1.417236 max_load_reduction=26963 max_load_reduction_pct=0.822845 moved_to_echo=15558 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 9, 0, 0, 3504, 351, 1839, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 4089, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4080, 101, 0, 0, 0, 0, 0, 0, 0, 9, 0, 40, 32, 1582, 0, 0, 0, 1929, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 2010, 49, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2438, 0, 0, 0, 1473, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2305, 0, 0, 0, 0, 0, 3756, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 580, 0, 1, 0, 0, 2, 1701, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5703, 4177, 4188, 4056, 2102, 4047, 6211, 2284] avg=4096.0000 max=6211 max_rank=6 min=2102 min_rank=4 imbalance_ratio=1.516357 max_load_reduction=26557 max_load_reduction_pct=0.810455 moved_to_echo=14644 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 310, 0, 0, 133, 184, 3175, 0, 0, 0, 0, 0, 0, 0, 1, 0, 55, 0, 0, 0, 4010, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 9, 3975, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 150, 2565, 0, 5, 0, 1871, 805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2231, 334, 76, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 3993, 0, 0, 0, 481, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 0, 0, 0, 0, 0, 3857, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 990, 0, 0, 0, 376, 6, 1651, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3802, 4131, 4093, 5398, 2674, 4610, 5037, 3023] avg=4096.0000 max=5398 max_rank=3 min=2674 min_rank=4 imbalance_ratio=1.317871 max_load_reduction=27370 max_load_reduction_pct=0.835266 moved_to_echo=15344 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 0, 130, 579, 3194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3856, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 34, 531, 0, 0, 0, 1852, 807, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1897, 4095, 84, 1258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3682, 0, 0, 0, 1471, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 451, 0, 0, 3, 0, 2, 1775, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 888, 0, 0, 0, 55, 1, 1634, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3906, 4096, 3957, 3239, 7335, 5282, 2375, 2578] avg=4096.0000 max=7335 max_rank=4 min=2375 min_rank=6 imbalance_ratio=1.790771 max_load_reduction=25433 max_load_reduction_pct=0.776154 moved_to_echo=17570 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 0, 7808, 21504, 640, 2944, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 0, 15872, 128, 128, 128, 4864, 128, 12160, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 10240, 0, 0, 128, 128, 128, 21504, 1152, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 768, 640, 11776, 128, 128, 128, 13696, 6016, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 128, 896, 128, 0, 6784, 1664, 23680, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 22784, 0, 0, 128, 9088, 1024, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 128, 32256, 0, 512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 128, 0, 128, 32000, 768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 4086, 11, 970, 16, 4, 4087, 0, 4024, 3349, 0, 0, 11, 0, 10, 0, 189, 2, 0, 4064, 0, 0, 0, 3950, 0, 0, 61, 0, 344, 11, 0, 8, 9, 0, 0, 4014, 27, 14, 8, 3249, 0, 0, 0, 83, 73, 0, 0, 0, 1, 13, 0, 14, 45, 6, 0, 5, 1, 0, 3, 0, 0, 0, 6, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 4076, 0, 1428, 0, 0, 4096, 0, 4001, 4074, 0, 0, 0, 0, 0, 0, 785, 0, 0, 4096, 0, 0, 0, 1595, 0, 0, 81, 0, 309, 0, 0, 0, 0, 0, 0, 4088, 10, 0, 0, 4089, 0, 0, 0, 14, 10, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 3923, 0, 7, 0, 0, 4091, 5, 4017, 4093, 0, 0, 1572, 0, 0, 0, 101, 0, 0, 4093, 0, 0, 0, 2294, 0, 0, 7, 0, 114, 0, 0, 0, 0, 0, 0, 4060, 15, 0, 0, 3973, 0, 0, 0, 165, 1, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 178, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 3987, 0, 19, 0, 0, 4086, 0, 4096, 4030, 0, 0, 34, 0, 0, 4, 65, 0, 0, 4094, 0, 0, 0, 4060, 0, 0, 6, 0, 5, 0, 0, 0, 0, 1, 0, 4048, 133, 0, 0, 3697, 4, 0, 0, 8, 12, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 373, 0, 0, 0, 0, 2, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 4082, 0, 104, 0, 0, 4084, 3, 3910, 4056, 0, 0, 30, 0, 0, 0, 73, 0, 0, 4053, 0, 1, 0, 3537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 84, 0, 0, 3882, 0, 0, 0, 637, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 149, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 0, 46, 0, 0, 4096, 2, 3837, 4081, 0, 0, 31, 0, 0, 0, 536, 0, 0, 4090, 0, 0, 0, 3794, 0, 0, 0, 0, 108, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 0, 3951, 0, 0, 0, 38, 5, 0, 0, 0, 0, 3, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 4072, 0, 896, 0, 0, 4096, 11, 3975, 2090, 0, 0, 385, 0, 0, 0, 322, 0, 0, 4096, 0, 0, 0, 3707, 0, 0, 14, 0, 103, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 418, 324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 4084, 0, 14, 0, 0, 4091, 10, 3880, 3710, 0, 0, 7, 0, 0, 0, 201, 0, 0, 4089, 0, 0, 0, 3722, 0, 0, 36, 0, 294, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 0, 0, 0, 431, 9, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3676, 0, 1428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 785, 0, 0, 481, 0, 0, 0, 1595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 309, 0, 0, 0, 3753, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 10, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 10, 0, 0, 0, 3428, 400, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 13, 3903, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5104, 4226, 2861, 4330, 4281, 3852, 4015, 4099] avg=4096.0000 max=5104 max_rank=0 min=2861 min_rank=2 imbalance_ratio=1.246094 max_load_reduction=27664 max_load_reduction_pct=0.844238 moved_to_echo=16247 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3685, 11, 970, 16, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 3349, 0, 0, 11, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 2, 0, 478, 0, 0, 0, 3950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 344, 11, 0, 8, 3774, 185, 0, 0, 0, 0, 0, 0, 9, 0, 0, 178, 27, 14, 8, 3249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 73, 0, 0, 0, 3401, 401, 0, 0, 0, 0, 0, 0, 1, 13, 0, 14, 45, 6, 0, 5, 3833, 96, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 6, 0, 4087, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4686, 3524, 4619, 4383, 3485, 3958, 4013, 4100] avg=4096.0000 max=4686 max_rank=0 min=3485 min_rank=4 imbalance_ratio=1.144043 max_load_reduction=28082 max_load_reduction_pct=0.856995 moved_to_echo=15556 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3681, 0, 104, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 149, 4056, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 476, 0, 1, 0, 3537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3667, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 84, 0, 0, 3882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 637, 0, 0, 0, 0, 3392, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 149, 3897, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3788, 4235, 4087, 3852, 4148, 4430, 4141, 4087] avg=4096.0000 max=4430 max_rank=5 min=3788 min_rank=0 imbalance_ratio=1.081543 max_load_reduction=28338 max_load_reduction_pct=0.864807 moved_to_echo=16806 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3538, 0, 7, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 153, 4093, 0, 0, 1572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 0, 481, 0, 0, 0, 2294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 114, 0, 0, 0, 3768, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 15, 0, 0, 3973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 1, 0, 0, 0, 3425, 385, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 178, 3876, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3550, 5818, 2876, 4076, 4169, 3976, 4209, 4094] avg=4096.0000 max=5818 max_rank=1 min=2876 min_rank=2 imbalance_ratio=1.420410 max_load_reduction=26950 max_load_reduction_pct=0.822449 moved_to_echo=16448 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3596, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 4030, 0, 0, 34, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 481, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 5, 0, 0, 0, 3842, 187, 0, 0, 0, 0, 0, 0, 0, 1, 0, 180, 133, 0, 0, 3697, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 8, 12, 0, 0, 4, 3426, 391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 373, 3865, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4086, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3615, 4224, 4606, 4040, 4011, 3845, 4336, 4091] avg=4096.0000 max=4606 max_rank=2 min=3615 min_rank=0 imbalance_ratio=1.124512 max_load_reduction=28162 max_load_reduction_pct=0.859436 moved_to_echo=16283 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3694, 0, 46, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 146, 4081, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 0, 0, 481, 0, 0, 0, 3794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 0, 0, 0, 3599, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 0, 0, 0, 3951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 5, 0, 0, 0, 3423, 402, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 0, 1, 3867, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3742, 4258, 4811, 3893, 4131, 3868, 3966, 4099] avg=4096.0000 max=4811 max_rank=2 min=3742 min_rank=0 imbalance_ratio=1.174561 max_load_reduction=27957 max_load_reduction_pct=0.853180 moved_to_echo=16064 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3683, 0, 14, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 148, 3710, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 201, 0, 0, 481, 0, 0, 0, 3722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 294, 0, 0, 0, 3639, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 431, 9, 0, 0, 0, 3422, 401, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 3911, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3707, 3865, 4404, 4155, 4274, 4263, 4006, 4094] avg=4096.0000 max=4404 max_rank=2 min=3707 min_rank=0 imbalance_ratio=1.075195 max_load_reduction=28364 max_load_reduction_pct=0.865601 moved_to_echo=16637 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3669, 0, 896, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 143, 2090, 0, 0, 385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 322, 0, 0, 475, 0, 0, 0, 3707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 103, 0, 0, 0, 3732, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 418, 324, 0, 0, 0, 3431, 403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 3915, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4576, 2618, 4504, 4039, 4269, 4576, 4082, 4104] avg=4096.0000 max=4576 max_rank=0 min=2618 min_rank=1 imbalance_ratio=1.117188 max_load_reduction=28192 max_load_reduction_pct=0.860352 moved_to_echo=17031 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 29312, 128, 3584, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1280, 29568, 0, 0, 2176, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2304, 128, 0, 3840, 0, 128, 0, 26752, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 0, 1920, 512, 0, 0, 128, 27392, 3200, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 256, 0, 1280, 128, 0, 128, 29824, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 0, 1536, 384, 128, 128, 31104, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 128, 0, 896, 31104, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 128, 0, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 19, 18, 0, 0, 0, 3339, 1, 15, 41, 0, 0, 1508, 3495, 0, 3, 1, 0, 0, 0, 4088, 0, 0, 94, 0, 0, 0, 0, 2811, 0, 0, 0, 0, 0, 0, 0, 486, 77, 0, 4096, 0, 0, 0, 0, 4086, 0, 4062, 0, 0, 0, 0, 487, 4041, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 482, 69, 0, 0, 0, 3906, 0, 0, 1, 0, 0, 472, 2678, 0, 242, 0, 0, 0, 9, 4039, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1461, 41, 0, 4096, 0, 0, 0, 0, 4081, 0, 4094, 0, 0, 0, 0, 3548, 3545, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 3541, 1, 7, 433, 0, 0, 186, 1804, 0, 94, 0, 0, 0, 0, 4094, 0, 0, 37, 0, 0, 0, 0, 966, 0, 0, 0, 0, 0, 0, 0, 232, 835, 0, 4096, 0, 0, 0, 0, 4096, 0, 4076, 0, 0, 0, 0, 4053, 4089, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 380, 5, 0, 0, 0, 3961, 0, 86, 394, 0, 0, 732, 3434, 0, 71, 0, 0, 0, 0, 4083, 0, 0, 560, 0, 0, 0, 0, 329, 0, 0, 0, 0, 0, 0, 0, 1, 46, 0, 4096, 0, 0, 0, 0, 3983, 0, 4091, 0, 0, 0, 0, 2549, 3967, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 19, 0, 8, 60, 0, 0, 0, 0, 4064, 65, 0, 117, 0, 0, 1463, 2965, 6, 235, 0, 0, 0, 0, 4066, 19, 0, 0, 7, 0, 0, 0, 204, 57, 5, 0, 8, 0, 0, 0, 40, 8, 0, 4024, 0, 7, 19, 0, 3644, 0, 4061, 0, 7, 3, 0, 4065, 3496, 0, 3, 0, 3, 9, 7, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2042, 21, 0, 0, 0, 1862, 0, 89, 2023, 0, 0, 376, 2470, 0, 16, 0, 0, 0, 0, 4053, 0, 0, 93, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8, 9, 0, 4095, 0, 0, 0, 0, 4016, 0, 4028, 0, 0, 0, 0, 3941, 3625, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 521, 149, 0, 0, 2, 3050, 0, 19, 45, 0, 0, 34, 2961, 0, 115, 2, 0, 0, 0, 4087, 0, 0, 17, 0, 0, 0, 0, 293, 0, 0, 0, 0, 0, 0, 0, 2151, 1457, 0, 4096, 0, 0, 0, 0, 4079, 0, 4074, 0, 0, 0, 0, 1521, 4095, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 3548, 0, 24, 135, 0, 0, 1768, 2507, 0, 238, 0, 0, 0, 0, 3908, 0, 0, 249, 0, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 0, 415, 179, 0, 4096, 0, 0, 0, 0, 4088, 0, 4050, 0, 0, 0, 0, 3327, 4093, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 128, 0, 0, 3608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3541, 1, 7, 433, 0, 0, 250, 0, 0, 0, 0, 0, 0, 0, 186, 1804, 0, 94, 0, 0, 0, 0, 361, 0, 0, 0, 0, 0, 0, 0, 4003, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 966, 0, 0, 0, 0, 0, 0, 0, 2912, 468, 181, 0, 0, 0, 0, 0, 232, 835, 0, 3171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1184, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 133, 91, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3736, 4232, 2445, 4040, 4527, 4238, 5237, 4313] avg=4096.0000 max=5237 max_rank=6 min=2445 min_rank=2 imbalance_ratio=1.278564 max_load_reduction=27531 max_load_reduction_pct=0.840179 moved_to_echo=18315 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 19, 18, 0, 3597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3339, 1, 15, 41, 0, 0, 250, 0, 0, 0, 0, 0, 0, 0, 1508, 3495, 0, 3, 1, 0, 0, 0, 361, 0, 0, 0, 0, 0, 0, 0, 3997, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2811, 0, 0, 0, 0, 0, 0, 0, 2905, 465, 181, 0, 0, 0, 0, 0, 486, 77, 0, 3171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1181, 0, 0, 0, 0, 0, 0, 487, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 133, 91, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3634, 3646, 5368, 4091, 6362, 3734, 1668, 4265] avg=4096.0000 max=6362 max_rank=4 min=1668 min_rank=6 imbalance_ratio=1.553223 max_load_reduction=26406 max_load_reduction_pct=0.805847 moved_to_echo=16029 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 482, 69, 0, 3624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3906, 0, 0, 1, 0, 0, 250, 0, 0, 0, 0, 0, 0, 0, 472, 2678, 0, 242, 0, 0, 0, 9, 361, 0, 0, 0, 0, 0, 0, 0, 3949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 2901, 470, 181, 0, 0, 0, 0, 0, 1461, 41, 0, 3171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1180, 0, 0, 0, 0, 0, 0, 3548, 0, 0, 0, 0, 0, 0, 0, 0, 3545, 0, 0, 0, 0, 0, 0, 0, 133, 90, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4175, 4157, 3762, 3949, 3556, 4673, 4728, 3768] avg=4096.0000 max=4728 max_rank=6 min=3556 min_rank=4 imbalance_ratio=1.154297 max_load_reduction=28040 max_load_reduction_pct=0.855713 moved_to_echo=16725 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2042, 21, 0, 3567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1862, 0, 89, 2023, 0, 0, 250, 0, 0, 0, 0, 0, 0, 0, 376, 2470, 0, 16, 0, 0, 0, 0, 361, 0, 0, 0, 0, 0, 0, 0, 3963, 0, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2855, 461, 181, 0, 0, 0, 0, 0, 8, 9, 0, 3170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1161, 0, 0, 0, 0, 0, 0, 3941, 0, 0, 0, 0, 0, 0, 0, 0, 3625, 0, 0, 0, 0, 0, 0, 0, 133, 90, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5630, 4224, 3223, 4056, 3498, 3187, 5102, 3848] avg=4096.0000 max=5630 max_rank=0 min=3187 min_rank=5 imbalance_ratio=1.374512 max_load_reduction=27138 max_load_reduction_pct=0.828186 moved_to_echo=15635 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 521, 149, 0, 3608, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3050, 0, 19, 45, 0, 0, 250, 0, 0, 0, 0, 0, 0, 0, 34, 2961, 0, 115, 2, 0, 0, 0, 361, 0, 0, 0, 0, 0, 0, 0, 3996, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 0, 0, 0, 0, 0, 0, 0, 2900, 466, 181, 0, 0, 0, 0, 0, 2151, 1457, 0, 3171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1179, 0, 0, 0, 0, 0, 0, 1521, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 133, 91, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4278, 3366, 3473, 4013, 3840, 6779, 2700, 4319] avg=4096.0000 max=6779 max_rank=5 min=2700 min_rank=6 imbalance_ratio=1.655029 max_load_reduction=25989 max_load_reduction_pct=0.793121 moved_to_echo=17638 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 39, 0, 0, 3585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3548, 0, 24, 135, 0, 0, 250, 0, 0, 0, 0, 0, 0, 0, 1768, 2507, 0, 238, 0, 0, 0, 0, 361, 0, 0, 0, 0, 0, 0, 0, 3821, 0, 0, 249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 0, 2906, 465, 181, 0, 0, 0, 0, 0, 415, 179, 0, 3171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1182, 0, 0, 0, 0, 0, 0, 3327, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 133, 87, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3624, 3957, 4874, 4070, 3656, 3765, 4509, 4313] avg=4096.0000 max=4874 max_rank=2 min=3624 min_rank=0 imbalance_ratio=1.189941 max_load_reduction=27894 max_load_reduction_pct=0.851257 moved_to_echo=16243 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 19, 0, 8, 60, 0, 0, 3596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 65, 0, 117, 0, 0, 246, 0, 0, 0, 0, 0, 0, 0, 1463, 2965, 6, 235, 0, 0, 0, 0, 354, 0, 0, 0, 0, 0, 0, 0, 3975, 19, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 204, 57, 5, 0, 8, 0, 0, 0, 2591, 465, 178, 0, 0, 0, 0, 0, 40, 8, 0, 3115, 0, 7, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 0, 0, 0, 7, 3, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 3496, 0, 3, 0, 3, 9, 7, 4, 131, 91, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3683, 4492, 5023, 4001, 3508, 3189, 5128, 3744] avg=4096.0000 max=5128 max_rank=6 min=3189 min_rank=5 imbalance_ratio=1.251953 max_load_reduction=27640 max_load_reduction_pct=0.843506 moved_to_echo=15569 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 380, 5, 0, 3623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3961, 0, 86, 394, 0, 0, 253, 0, 0, 0, 0, 0, 0, 0, 732, 3434, 0, 71, 0, 0, 0, 0, 363, 0, 0, 0, 0, 0, 0, 0, 3988, 0, 0, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 329, 0, 0, 0, 0, 0, 0, 0, 2836, 468, 188, 0, 0, 0, 0, 0, 1, 46, 0, 3156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1147, 0, 0, 0, 0, 0, 0, 2549, 0, 0, 0, 0, 0, 0, 0, 0, 3967, 0, 0, 0, 0, 0, 0, 0, 136, 95, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4008, 4694, 4600, 4548, 3821, 3203, 3696, 4198] avg=4096.0000 max=4694 max_rank=1 min=3203 min_rank=5 imbalance_ratio=1.145996 max_load_reduction=28074 max_load_reduction_pct=0.856750 moved_to_echo=14918 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[9344, 0, 0, 0, 128, 128, 0, 23552, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[6656, 22400, 128, 1024, 128, 0, 0, 128, 2944, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4864, 2688, 0, 25344, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4736, 128, 128, 0, 128, 0, 0, 0, 22912, 3840, 1536, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 128, 3712, 384, 0, 28928, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[31744, 128, 0, 1152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[30976, 0, 128, 0, 128, 128, 128, 128, 1152, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 27392, 128, 256, 3200, 0, 0, 2048, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 186, 0, 4, 0, 3289, 0, 0, 1872, 0, 0, 4079, 0, 0, 0, 0, 0, 24, 0, 3777, 0, 0, 0, 0, 3366, 150, 22, 0, 0, 0, 0, 943, 0, 0, 0, 1, 0, 0, 0, 3768, 0, 0, 0, 0, 4079, 0, 0, 0, 4096, 0, 0, 3066, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 567, 64, 18, 0, 2482, 1, 9, 875, 0, 0, 4001, 10, 0, 0, 11, 0, 9, 0, 3934, 0, 0, 0, 0, 3790, 0, 517, 0, 0, 3, 0, 500, 0, 0, 0, 96, 0, 0, 0, 3806, 0, 4, 11, 13, 3779, 0, 0, 9, 4055, 0, 16, 4044, 0, 0, 0, 12, 3, 2, 15, 0, 87, 6, 8, 11] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 0, 7, 0, 490, 5, 0, 1680, 0, 0, 4004, 0, 0, 0, 0, 0, 1070, 0, 3971, 0, 1, 0, 0, 3921, 3, 1570, 0, 0, 0, 0, 421, 0, 0, 0, 0, 0, 0, 0, 3564, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 29, 3837, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 10, 2, 0, 0, 3221, 5, 0, 803, 0, 0, 3943, 0, 0, 0, 0, 0, 1006, 0, 3759, 0, 0, 0, 0, 3933, 0, 24, 0, 0, 0, 0, 717, 0, 0, 0, 0, 0, 0, 0, 3173, 0, 0, 0, 0, 4089, 0, 0, 0, 4096, 0, 0, 3980, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 0, 3644, 9, 0, 1143, 0, 4, 4036, 0, 0, 1, 0, 0, 18, 0, 4093, 0, 0, 1, 0, 4094, 60, 30, 0, 0, 0, 0, 375, 0, 0, 0, 1, 0, 0, 0, 3034, 0, 0, 0, 0, 3976, 0, 0, 0, 4096, 0, 5, 3617, 0, 0, 0, 0, 0, 523, 0, 4, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 797, 31, 1, 0, 3325, 1, 0, 541, 35, 0, 4088, 0, 0, 0, 0, 0, 76, 0, 4089, 0, 1, 0, 0, 3973, 11, 2, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 3516, 0, 0, 0, 0, 4092, 0, 0, 0, 4096, 0, 0, 4015, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 230, 0, 13, 0, 4073, 475, 0, 990, 0, 0, 4034, 0, 0, 0, 0, 0, 43, 0, 3744, 0, 15, 0, 0, 4024, 5, 327, 0, 0, 0, 0, 446, 0, 0, 0, 0, 0, 0, 0, 2035, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 0, 15, 4049, 0, 0, 3, 0, 0, 56, 0, 4, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 18, 0, 0, 0, 2394, 0, 0, 1428, 0, 0, 4096, 0, 0, 0, 0, 0, 300, 0, 3975, 0, 0, 0, 0, 4070, 60, 40, 0, 0, 0, 0, 379, 0, 0, 0, 3, 0, 0, 0, 3691, 0, 0, 0, 0, 4064, 0, 0, 0, 4090, 0, 58, 4037, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 10, 2, 0, 0, 3221, 5, 0, 902, 0, 0, 0, 0, 0, 0, 0, 803, 0, 0, 2857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 0, 3623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3296, 0, 24, 0, 0, 0, 0, 717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3173, 637, 81, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 249, 0, 0, 3980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 3847, 142, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4140, 3660, 4629, 4037, 3933, 4144, 4229, 3996] avg=4096.0000 max=4629 max_rank=2 min=3660 min_rank=1 imbalance_ratio=1.130127 max_load_reduction=28139 max_load_reduction_pct=0.858734 moved_to_echo=16302 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 567, 64, 18, 0, 2482, 1, 9, 916, 0, 0, 0, 0, 0, 0, 0, 875, 0, 0, 2898, 10, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 3792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3176, 0, 517, 0, 0, 3, 0, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 3806, 614, 85, 43, 0, 0, 0, 0, 0, 0, 4, 11, 13, 3779, 0, 0, 9, 57, 0, 0, 0, 0, 0, 0, 0, 247, 0, 16, 4044, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 15, 0, 87, 6, 8, 11, 3808, 144, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4057, 3794, 3801, 4196, 4644, 3873, 4319, 4084] avg=4096.0000 max=4644 max_rank=4 min=3794 min_rank=1 imbalance_ratio=1.133789 max_load_reduction=28124 max_load_reduction_pct=0.858276 moved_to_echo=16920 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 186, 0, 4, 0, 3289, 0, 0, 934, 0, 0, 0, 0, 0, 0, 0, 1872, 0, 0, 2955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 3641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2821, 150, 22, 0, 0, 0, 0, 943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3768, 545, 81, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 249, 0, 0, 3066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 3847, 147, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4413, 4827, 3665, 3936, 4438, 4134, 3315, 4040] avg=4096.0000 max=4827 max_rank=1 min=3315 min_rank=6 imbalance_ratio=1.178467 max_load_reduction=27941 max_load_reduction_pct=0.852692 moved_to_echo=15927 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 0, 3644, 9, 0, 924, 0, 0, 0, 0, 0, 0, 0, 1143, 0, 4, 2924, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 3945, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3431, 60, 30, 0, 0, 0, 0, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3034, 663, 88, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3976, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 249, 0, 5, 3617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 0, 4, 0, 0, 0, 0, 3847, 145, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4581, 4072, 3964, 3896, 3829, 4036, 3871, 4519] avg=4096.0000 max=4581 max_rank=0 min=3829 min_rank=4 imbalance_ratio=1.118408 max_load_reduction=28187 max_load_reduction_pct=0.860199 moved_to_echo=16255 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 7, 0, 490, 5, 0, 916, 0, 0, 0, 0, 0, 0, 0, 1680, 0, 0, 2901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1070, 0, 3828, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3286, 3, 1570, 0, 0, 0, 0, 421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3564, 635, 85, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 249, 0, 29, 3837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3847, 144, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1419, 4581, 4899, 5280, 4327, 4154, 4115, 3993] avg=4096.0000 max=5280 max_rank=3 min=1419 min_rank=0 imbalance_ratio=1.289062 max_load_reduction=27488 max_load_reduction_pct=0.838867 moved_to_echo=16589 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 230, 0, 13, 0, 4073, 475, 0, 923, 0, 0, 0, 0, 0, 0, 0, 990, 0, 0, 2923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 3610, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3372, 5, 327, 0, 0, 0, 0, 446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2035, 652, 80, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 249, 0, 15, 4049, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 4, 0, 0, 0, 0, 3842, 145, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5714, 3913, 3668, 4150, 2810, 4150, 4316, 4047] avg=4096.0000 max=5714 max_rank=0 min=2810 min_rank=4 imbalance_ratio=1.395020 max_load_reduction=27054 max_load_reduction_pct=0.825623 moved_to_echo=15323 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 797, 31, 1, 0, 3325, 1, 0, 936, 0, 0, 0, 0, 0, 0, 0, 541, 35, 0, 2961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 3941, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3329, 11, 2, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3516, 644, 88, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 249, 0, 0, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 3847, 147, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5091, 3537, 4018, 3388, 4292, 4152, 4264, 4026] avg=4096.0000 max=5091 max_rank=0 min=3388 min_rank=3 imbalance_ratio=1.242920 max_load_reduction=27677 max_load_reduction_pct=0.844635 moved_to_echo=16734 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 18, 0, 0, 0, 2394, 0, 0, 941, 0, 0, 0, 0, 0, 0, 0, 1428, 0, 0, 2956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 0, 3824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3406, 60, 40, 0, 0, 0, 0, 379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3691, 664, 90, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 244, 0, 58, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 3846, 152, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3353, 4384, 4124, 3885, 4495, 4125, 4339, 4063] avg=4096.0000 max=4495 max_rank=4 min=3353 min_rank=0 imbalance_ratio=1.097412 max_load_reduction=28273 max_load_reduction_pct=0.862823 moved_to_echo=17022 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[9344, 128, 128, 23424, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 1920, 128, 128, 0, 23040, 512, 128, 7424, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 26624, 5120, 768, 384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[26240, 384, 2560, 0, 0, 128, 0, 3840, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 2560, 0, 30208, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2048, 0, 128, 30720, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 32384, 0, 0, 128, 512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[128, 768, 128, 128, 128, 128, 128, 128, 30848, 1280, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33792 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3144, 0, 4084, 124, 0, 0, 4076, 11, 1809, 10, 0, 0, 0, 4068, 5, 0, 4078, 1612, 1, 0, 2827, 0, 1, 0, 2, 10, 3310, 20, 12, 0, 3, 0, 2, 0, 8, 2, 0, 0, 0, 3298, 0, 45, 13, 0, 0, 7, 13, 0, 11, 0, 11, 113, 0, 0, 0, 26, 0, 0, 12, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1504, 0, 4096, 1567, 0, 0, 4096, 0, 3922, 0, 0, 0, 0, 4096, 0, 0, 4060, 1062, 0, 0, 3657, 0, 0, 0, 3, 0, 4057, 0, 10, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1972, 0, 4096, 62, 0, 0, 4096, 0, 3968, 0, 0, 1, 10, 4096, 0, 0, 4005, 2275, 0, 0, 3016, 0, 0, 0, 86, 0, 2860, 0, 1, 636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2264, 0, 4096, 452, 0, 0, 4096, 0, 4038, 0, 0, 0, 1, 4012, 0, 0, 4056, 3810, 0, 0, 3563, 0, 0, 0, 4, 0, 443, 0, 55, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 28, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1651, 0, 4096, 8, 0, 0, 4096, 0, 4074, 0, 0, 0, 0, 4096, 0, 0, 4019, 1793, 0, 0, 3934, 0, 0, 0, 0, 0, 2849, 0, 126, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3025, 0, 4096, 4, 0, 0, 4096, 0, 4074, 0, 0, 0, 0, 4096, 0, 0, 4018, 1569, 0, 0, 1741, 0, 0, 0, 0, 0, 2272, 0, 147, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4087, 0, 4089, 170, 0, 0, 4096, 0, 3888, 0, 0, 0, 0, 3801, 0, 0, 4014, 3205, 0, 0, 1883, 0, 0, 0, 65, 0, 1218, 0, 723, 981, 0, 0, 0, 0, 0, 0, 0, 7, 0, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2332, 0, 4095, 1631, 0, 0, 4096, 0, 3979, 0, 0, 0, 3, 4096, 0, 0, 3791, 9, 0, 0, 3631, 0, 0, 0, 0, 0, 2554, 0, 251, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1651, 0, 1098, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 376, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2186, 1793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2112, 0, 0, 0, 0, 0, 2849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 69, 0, 0, 0, 0, 0, 0, 2602, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1957, 0, 0, 0, 0, 1822, 752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3678, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 42, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2757, 4450, 3979, 4961, 3878, 4531, 4074, 4138] avg=4096.0000 max=4961 max_rank=3 min=2757 min_rank=0 imbalance_ratio=1.211182 max_load_reduction=27807 max_load_reduction_pct=0.848602 moved_to_echo=16621 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3144, 0, 1095, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 373, 11, 1809, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2218, 1612, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1518, 0, 1, 0, 2, 10, 3310, 20, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 3, 0, 2, 0, 8, 2, 2594, 1097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3298, 0, 45, 13, 0, 1309, 763, 0, 0, 0, 0, 0, 0, 0, 7, 13, 0, 11, 0, 11, 113, 3661, 395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 12, 0, 4068, 42, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4363, 2203, 3836, 4861, 3718, 5428, 4211, 4148] avg=4096.0000 max=5428 max_rank=5 min=2203 min_rank=1 imbalance_ratio=1.325195 max_load_reduction=27340 max_load_reduction_pct=0.834351 moved_to_echo=17505 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1972, 0, 1098, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 376, 0, 3968, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 2177, 2275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1619, 0, 0, 0, 86, 0, 2860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 636, 0, 0, 0, 0, 0, 0, 2602, 1078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1588, 0, 0, 0, 0, 1397, 750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3678, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 42, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3132, 4345, 4462, 4565, 4317, 3735, 4074, 4138] avg=4096.0000 max=4565 max_rank=3 min=3132 min_rank=0 imbalance_ratio=1.114502 max_load_reduction=28203 max_load_reduction_pct=0.860687 moved_to_echo=16264 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4087, 0, 1096, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 376, 0, 3888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2183, 3205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 0, 0, 0, 65, 0, 1218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 723, 981, 0, 0, 0, 0, 0, 0, 2597, 1080, 0, 0, 0, 0, 0, 0, 0, 7, 0, 530, 0, 0, 0, 0, 872, 751, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 3678, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3801, 42, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5354, 4264, 5388, 2294, 5381, 2160, 4083, 3844] avg=4096.0000 max=5388 max_rank=2 min=2160 min_rank=5 imbalance_ratio=1.315430 max_load_reduction=27380 max_load_reduction_pct=0.835571 moved_to_echo=15468 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3025, 0, 1098, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 376, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2185, 1569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 935, 0, 0, 0, 0, 0, 2272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 91, 0, 0, 0, 0, 0, 0, 2602, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3539, 0, 0, 0, 0, 806, 752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3678, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 42, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4127, 4450, 3754, 3207, 3921, 5097, 4074, 4138] avg=4096.0000 max=5097 max_rank=5 min=3207 min_rank=3 imbalance_ratio=1.244385 max_load_reduction=27671 max_load_reduction_pct=0.844452 moved_to_echo=17230 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2264, 0, 1098, 452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 376, 0, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2206, 3810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1912, 0, 0, 0, 4, 0, 443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 101, 0, 0, 0, 0, 0, 0, 2602, 1091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1739, 0, 0, 0, 0, 1651, 759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3678, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 4012, 42, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3814, 4414, 6017, 2359, 3849, 4149, 4084, 4082] avg=4096.0000 max=6017 max_rank=2 min=2359 min_rank=3 imbalance_ratio=1.468994 max_load_reduction=26751 max_load_reduction_pct=0.816376 moved_to_echo=16164 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2332, 0, 1098, 1631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 376, 0, 3979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 2061, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1949, 0, 0, 0, 0, 0, 2554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 2, 0, 0, 0, 0, 0, 0, 2601, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2295, 0, 0, 0, 0, 1682, 710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3678, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 42, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5061, 4355, 2073, 4503, 3874, 4687, 4077, 4138] avg=4096.0000 max=5061 max_rank=0 min=2073 min_rank=2 imbalance_ratio=1.235596 max_load_reduction=27707 max_load_reduction_pct=0.845551 moved_to_echo=16776 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1504, 0, 1089, 1567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 365, 0, 3922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2197, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1958, 0, 0, 0, 3, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 116, 0, 0, 0, 0, 0, 0, 2606, 1098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 0, 0, 0, 0, 1699, 765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3685, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 46, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4160, 4287, 3259, 6018, 3830, 2981, 4091, 4142] avg=4096.0000 max=6018 max_rank=3 min=2981 min_rank=5 imbalance_ratio=1.469238 max_load_reduction=26750 max_load_reduction_pct=0.816345 moved_to_echo=15044 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[1408, 2048, 128, 0, 128, 0, 128, 128, 20864, 8704, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 0, 15488, 0, 128, 128, 0, 11264, 6016, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 0, 17536, 15360, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[13056, 0, 128, 0, 256, 128, 19584, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 20096, 0, 8832, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 128, 0, 128, 256, 29440, 3200, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 3072, 128, 29824, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 0, 0, 128, 0, 32384, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[735, 734, 0, 2034, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 32, 3121, 0, 1, 0, 0, 3644, 0, 0, 8, 11, 1593, 0, 0, 1673, 0, 0, 0, 2, 1, 0, 0, 0, 0, 1843, 0, 3704, 0, 0, 1188, 0, 4, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 4092, 0, 141, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[2539, 222, 0, 770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3362, 0, 4, 0, 0, 2986, 0, 0, 25, 0, 2896, 0, 0, 1618, 0, 0, 0, 1, 0, 0, 0, 0, 0, 219, 0, 2945, 0, 0, 2843, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 50, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1164, 347, 0, 3626, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2972, 0, 0, 0, 0, 2944, 0, 0, 2100, 1, 2297, 0, 0, 44, 0, 0, 0, 39, 0, 0, 0, 0, 0, 126, 0, 3033, 0, 0, 1707, 0, 4, 0, 0, 1, 0, 0, 0, 4089, 0, 0, 4096, 0, 81, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 566, 9, 3886, 2, 12, 0, 0, 12, 56, 6, 0, 0, 0, 0, 13, 0, 4069, 0, 0, 4085, 0, 116, 0, 6, 3858, 0, 11, 9, 0, 2548, 0, 0, 159, 0, 0, 0, 29, 0, 0, 4, 0, 0, 84, 0, 3737, 12, 0, 263, 0, 0, 0, 0, 0, 0, 13, 0, 4077, 0, 0, 3978, 0, 1144, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[187, 527, 0, 4007, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3960, 0, 0, 0, 0, 3802, 0, 0, 43, 0, 754, 0, 0, 1993, 0, 0, 0, 1, 0, 0, 0, 0, 0, 964, 0, 2839, 0, 0, 498, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 4092, 0, 878, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[936, 1757, 0, 3184, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 3801, 0, 0, 0, 0, 1228, 0, 0, 188, 1, 3032, 0, 0, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1289, 0, 826, 0, 0, 2195, 0, 10, 0, 1, 8, 0, 0, 0, 4078, 10, 0, 4065, 0, 1902, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[455, 3857, 0, 4075, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 99, 3744, 0, 0, 0, 0, 3104, 0, 0, 643, 0, 1233, 0, 0, 1084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 278, 0, 937, 0, 0, 707, 0, 6, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 258, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[931, 213, 0, 1530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 3477, 0, 0, 0, 0, 3990, 0, 0, 0, 0, 35, 0, 0, 280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2967, 0, 4057, 0, 0, 1187, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1811, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2539, 222, 0, 587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 500, 0, 0, 3362, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1792, 0, 0, 25, 0, 2896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1618, 0, 0, 0, 1, 0, 0, 3202, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 219, 0, 2945, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 2843, 0, 0, 0, 0, 0, 0, 0, 1194, 768, 394, 87, 0, 0, 0, 0, 0, 0, 0, 0, 3328, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3348, 4083, 3866, 4713, 4834, 3260, 5286, 3378] avg=4096.0000 max=5286 max_rank=6 min=3260 min_rank=5 imbalance_ratio=1.290527 max_load_reduction=27482 max_load_reduction_pct=0.838684 moved_to_echo=16758 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[735, 734, 0, 1548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 500, 0, 32, 3121, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2187, 0, 0, 8, 11, 1593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1673, 0, 0, 0, 2, 1, 0, 3202, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1843, 0, 3704, 0, 0, 255, 0, 0, 0, 0, 0, 0, 0, 1188, 0, 4, 0, 0, 0, 0, 0, 1457, 768, 394, 231, 0, 0, 0, 0, 0, 0, 0, 0, 3324, 0, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3017, 4098, 3654, 3799, 4891, 5802, 4042, 3465] avg=4096.0000 max=5802 max_rank=5 min=3017 min_rank=0 imbalance_ratio=1.416504 max_load_reduction=26966 max_load_reduction_pct=0.822937 moved_to_echo=18200 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 566, 9, 2956, 2, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 56, 6, 0, 0, 0, 0, 13, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 497, 0, 0, 4085, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2315, 0, 11, 9, 0, 2548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 159, 0, 0, 0, 29, 0, 0, 3181, 12, 0, 0, 0, 0, 0, 0, 4, 0, 0, 84, 0, 3737, 12, 0, 488, 0, 0, 0, 0, 0, 0, 0, 263, 0, 0, 0, 0, 0, 0, 13, 1543, 746, 391, 442, 0, 0, 0, 0, 0, 0, 0, 0, 3232, 0, 1144, 4, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3545, 4152, 4698, 4889, 3381, 4325, 3398, 4380] avg=4096.0000 max=4889 max_rank=3 min=3381 min_rank=4 imbalance_ratio=1.193604 max_load_reduction=27879 max_load_reduction_pct=0.850800 moved_to_echo=15484 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1164, 347, 0, 2758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 500, 0, 0, 2972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1767, 0, 0, 2100, 1, 2297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 39, 0, 0, 3202, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 0, 3033, 0, 0, 455, 0, 0, 0, 0, 0, 0, 0, 1707, 0, 4, 0, 0, 1, 0, 0, 1177, 768, 394, 413, 0, 0, 0, 0, 0, 0, 0, 0, 3328, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4269, 4078, 3472, 6165, 3297, 3614, 4464, 3409] avg=4096.0000 max=6165 max_rank=3 min=3297 min_rank=4 imbalance_ratio=1.505127 max_load_reduction=26603 max_load_reduction_pct=0.811859 moved_to_echo=14784 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[936, 1757, 0, 2422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 500, 0, 0, 3801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 737, 0, 0, 188, 1, 3032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 0, 0, 0, 0, 0, 0, 3201, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1289, 0, 826, 0, 0, 400, 0, 0, 0, 0, 0, 0, 0, 2195, 0, 10, 0, 1, 8, 0, 0, 491, 763, 394, 362, 0, 0, 0, 0, 0, 0, 10, 0, 3302, 0, 1902, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5115, 4079, 4301, 3958, 3362, 2515, 4224, 5214] avg=4096.0000 max=5214 max_rank=7 min=2515 min_rank=5 imbalance_ratio=1.272949 max_load_reduction=27554 max_load_reduction_pct=0.840881 moved_to_echo=15315 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[455, 3857, 0, 3099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 500, 0, 99, 3744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1863, 0, 0, 643, 0, 1233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1084, 0, 0, 0, 0, 0, 0, 3202, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 278, 0, 937, 0, 0, 512, 0, 0, 0, 0, 0, 0, 0, 707, 0, 6, 0, 0, 0, 0, 0, 1241, 768, 394, 464, 0, 0, 0, 0, 0, 0, 0, 0, 3328, 0, 258, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7411, 4084, 4343, 3739, 4298, 1727, 3580, 3586] avg=4096.0000 max=7411 max_rank=0 min=1727 min_rank=5 imbalance_ratio=1.809326 max_load_reduction=25357 max_load_reduction_pct=0.773834 moved_to_echo=13191 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[931, 213, 0, 1164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 500, 0, 2, 3477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2395, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 280, 0, 0, 0, 0, 0, 0, 3202, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2967, 0, 4057, 0, 0, 192, 0, 0, 0, 0, 0, 0, 0, 1187, 0, 0, 0, 0, 0, 0, 0, 1595, 768, 394, 174, 0, 0, 0, 0, 0, 0, 0, 0, 3328, 0, 1811, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2308, 4083, 3979, 2430, 3495, 7216, 4118, 5139] avg=4096.0000 max=7216 max_rank=5 min=2308 min_rank=0 imbalance_ratio=1.761719 max_load_reduction=25552 max_load_reduction_pct=0.779785 moved_to_echo=19968 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[187, 527, 0, 3041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 495, 0, 0, 3960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2278, 0, 0, 43, 0, 754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1993, 0, 0, 0, 1, 0, 0, 3204, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 964, 0, 2839, 0, 0, 506, 0, 0, 0, 0, 0, 0, 0, 498, 0, 0, 0, 0, 4, 0, 0, 1524, 773, 397, 460, 0, 0, 0, 0, 0, 0, 0, 0, 3319, 0, 878, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3755, 4111, 4455, 3075, 5210, 4309, 3656, 4197] avg=4096.0000 max=5210 max_rank=4 min=3075 min_rank=3 imbalance_ratio=1.271973 max_load_reduction=27558 max_load_reduction_pct=0.841003 moved_to_echo=17372 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[10624, 0, 128, 0, 128, 128, 0, 128, 10240, 6144, 3200, 2688, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 0, 7808, 0, 22144, 128, 0, 2944, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 0, 26496, 0, 6272, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 15360, 0, 128, 3072, 128, 14464, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[7040, 8320, 128, 17664, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 7040, 0, 0, 0, 128, 128, 0, 25600, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 4096, 0, 256, 28544, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 256, 128, 0, 0, 0, 0, 128, 32640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1745, 0, 4096, 0, 0, 1200, 0, 0, 0, 0, 0, 0, 103, 0, 0, 0, 4087, 0, 0, 4069, 293, 0, 17, 0, 0, 4010, 0, 4028, 4096, 895, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 27] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[12, 6, 0, 50, 0, 3, 4084, 0, 0, 0, 0, 0, 1, 15, 0, 0, 0, 0, 0, 5, 0, 3486, 0, 4072, 57, 5, 379, 194, 0, 0, 10, 10, 24, 953, 0, 30, 7, 4082, 12, 11, 2717, 677, 2, 0, 0, 0, 3756, 0, 3875, 4083, 12, 0, 0, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1474, 0, 4092, 0, 0, 6, 49, 12, 29, 0, 0, 0, 16, 0, 0, 0, 4073, 0, 0, 4095, 2611, 0, 2, 0, 0, 4033, 0, 4076, 4071, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 50] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2521, 0, 4095, 0, 0, 1820, 2, 0, 9, 0, 0, 0, 2, 0, 0, 0, 3926, 0, 0, 3597, 624, 0, 0, 0, 0, 4005, 0, 3929, 4096, 59, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2148, 0, 4096, 0, 0, 1238, 64, 5, 5, 0, 0, 0, 9, 0, 0, 0, 3443, 0, 0, 3989, 1235, 0, 111, 0, 0, 4062, 0, 4093, 4096, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 598, 0, 4090, 0, 0, 1239, 466, 3, 16, 0, 0, 0, 122, 0, 0, 0, 3537, 0, 0, 4068, 2612, 0, 40, 0, 0, 3474, 0, 3946, 4088, 415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3913, 0, 4096, 0, 0, 195, 36, 0, 9, 0, 0, 0, 18, 0, 0, 0, 4082, 0, 0, 3392, 835, 0, 64, 0, 0, 3722, 0, 3892, 4087, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2122, 0, 4057, 0, 0, 250, 1, 0, 6, 0, 0, 0, 7, 0, 0, 0, 4067, 0, 0, 3756, 2726, 0, 4, 0, 0, 4074, 0, 3389, 4095, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 30] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 4096, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1745, 0, 1849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1200, 0, 0, 0, 0, 0, 2057, 1201, 34, 0, 0, 0, 0, 0, 0, 103, 0, 0, 0, 4087, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 2868, 293, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3958, 0, 895, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 4005, 190, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4101, 3594, 4492, 4210, 3178, 4857, 4222] avg=4096.0000 max=4857 max_rank=6 min=3178 min_rank=5 imbalance_ratio=1.185791 max_load_reduction=27911 max_load_reduction_pct=0.851776 moved_to_echo=16467 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[12, 6, 0, 50, 0, 3, 4084, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 15, 0, 0, 4083, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3486, 0, 1838, 0, 0, 0, 0, 0, 0, 0, 0, 57, 5, 379, 194, 0, 0, 10, 10, 2045, 802, 32, 0, 0, 0, 0, 0, 24, 953, 0, 30, 7, 4082, 12, 11, 19, 0, 0, 0, 0, 0, 0, 0, 1915, 677, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3808, 0, 12, 0, 0, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3752, 189, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4171, 4103, 5329, 3534, 5138, 2594, 3956, 3943] avg=4096.0000 max=5329 max_rank=2 min=2594 min_rank=5 imbalance_ratio=1.301025 max_load_reduction=27439 max_load_reduction_pct=0.837372 moved_to_echo=15631 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4056, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4071, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1474, 0, 1848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 49, 12, 29, 0, 0, 2055, 1209, 34, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 4073, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 2886, 2611, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4005, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 50, 4028, 189, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4077, 3322, 3394, 4109, 5499, 4026, 4268] avg=4096.0000 max=5499 max_rank=5 min=3322 min_rank=2 imbalance_ratio=1.342529 max_load_reduction=27269 max_load_reduction_pct=0.832184 moved_to_echo=17902 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 0, 4065, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2148, 0, 1849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1238, 64, 5, 5, 0, 0, 2057, 1178, 34, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 3443, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 2811, 1235, 0, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4057, 190, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4101, 3997, 4581, 3472, 4157, 4120, 4255] avg=4096.0000 max=4581 max_rank=3 min=3472 min_rank=4 imbalance_ratio=1.118408 max_load_reduction=28187 max_load_reduction_pct=0.860199 moved_to_echo=16004 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4066, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2521, 0, 1849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1820, 2, 0, 9, 0, 0, 2056, 1062, 33, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3926, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 2535, 624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3861, 0, 59, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4000, 190, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4101, 4370, 4982, 3947, 3159, 3937, 4190] avg=4096.0000 max=4982 max_rank=3 min=3159 min_rank=5 imbalance_ratio=1.216309 max_load_reduction=27786 max_load_reduction_pct=0.847961 moved_to_echo=15233 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4040, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 598, 0, 1847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1239, 466, 3, 16, 0, 0, 2054, 1201, 33, 0, 0, 0, 0, 0, 0, 122, 0, 0, 0, 3537, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 2867, 2612, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3878, 0, 415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 3469, 189, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4056, 4093, 2445, 5012, 3678, 5519, 4293, 3672] avg=4096.0000 max=5519 max_rank=5 min=2445 min_rank=2 imbalance_ratio=1.347412 max_load_reduction=27249 max_load_reduction_pct=0.831573 moved_to_echo=17162 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4092, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2122, 0, 1832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 250, 1, 0, 6, 0, 0, 2037, 1109, 28, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4067, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 2647, 2726, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3330, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 30, 4069, 188, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4100, 3954, 3431, 4091, 5377, 3416, 4293] avg=4096.0000 max=5377 max_rank=5 min=3416 min_rank=6 imbalance_ratio=1.312744 max_load_reduction=27391 max_load_reduction_pct=0.835907 moved_to_echo=17177 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4062, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3913, 0, 1844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 195, 36, 0, 9, 0, 0, 2059, 1006, 37, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 4082, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 2386, 835, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3813, 0, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 3717, 193, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4092, 5757, 3342, 4123, 3285, 4163, 3925] avg=4096.0000 max=5757 max_rank=2 min=3285 min_rank=5 imbalance_ratio=1.405518 max_load_reduction=27011 max_load_reduction_pct=0.824310 moved_to_echo=15496 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[128, 128, 6400, 896, 128, 128, 128, 128, 16512, 8832, 384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 0, 18048, 0, 14848, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[20992, 11648, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 1280, 0, 128, 128, 31360, 128, 128, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[30720, 0, 2048, 0, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 256, 31104, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 0, 128, 32640, 0, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33792 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4086, 4096, 0, 0, 0, 11, 0, 5, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 0, 0, 1, 0, 0, 2015, 0, 4091, 2898, 0, 0, 0, 0, 0, 0, 0, 3273, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4082, 3966, 2, 11, 0, 0, 5, 1, 0, 0, 0, 0, 0, 4081, 0, 0, 4080, 9, 0, 0, 0, 0, 0, 0, 0, 11, 2, 1379, 4083, 0, 0, 7, 6, 49, 0, 6, 0, 0, 6, 394, 0, 4084, 4078, 13, 0, 0, 0, 0, 0, 0, 2375, 0, 13, 0, 0, 10, 0, 0, 7, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4074, 4096, 0, 0, 10, 4, 0, 6, 0, 0, 0, 0, 0, 4091, 0, 0, 4010, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3646, 0, 4095, 3999, 0, 0, 0, 0, 0, 0, 0, 636, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4094, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1255, 0, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 2851, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4079, 4096, 0, 0, 44, 0, 0, 19, 0, 0, 0, 0, 5, 4069, 0, 0, 4080, 0, 0, 0, 4, 0, 38, 0, 0, 0, 0, 1, 4089, 0, 0, 0, 541, 0, 0, 0, 17, 0, 0, 2586, 0, 4087, 4020, 0, 0, 0, 0, 0, 0, 0, 993, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 4096, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4095, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 3851, 0, 4076, 3668, 0, 0, 0, 0, 0, 0, 0, 637, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4008, 4096, 0, 0, 22, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 4096, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 593, 0, 4096, 3998, 0, 0, 76, 0, 0, 0, 0, 3560, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4045, 4096, 0, 0, 61, 8, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4082, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 106, 0, 0, 0, 6, 0, 0, 1752, 0, 4086, 3567, 0, 0, 0, 0, 0, 0, 0, 2759, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4082, 25, 2, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 0, 0, 0, 4055, 5, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 4080, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 2, 1379, 3798, 0, 0, 0, 0, 0, 0, 0, 3985, 0, 0, 7, 6, 49, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 394, 0, 0, 2238, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2375, 0, 1840, 138, 98, 0, 0, 0, 0, 0, 13, 0, 0, 10, 0, 0, 7, 7, 4084, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4066, 4113, 5190, 4053, 2651, 4451, 4123] avg=4096.0000 max=5190 max_rank=3 min=2651 min_rank=5 imbalance_ratio=1.267090 max_load_reduction=27578 max_load_reduction_pct=0.841614 moved_to_echo=15278 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 4070, 5, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 6, 3921, 0, 0, 0, 0, 0, 0, 0, 3998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1255, 0, 0, 2239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2851, 0, 1840, 143, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4077, 4120, 3928, 3998, 3494, 4932, 4098] avg=4096.0000 max=4932 max_rank=6 min=3494 min_rank=5 imbalance_ratio=1.204102 max_load_reduction=27836 max_load_reduction_pct=0.849487 moved_to_echo=16522 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4086, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 5, 0, 0, 0, 0, 4067, 5, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3923, 0, 0, 0, 0, 0, 0, 0, 3998, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2015, 0, 0, 1591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3273, 0, 1307, 143, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4088, 4120, 3924, 4003, 3607, 4821, 4093] avg=4096.0000 max=4821 max_rank=6 min=3607 min_rank=5 imbalance_ratio=1.177002 max_load_reduction=27947 max_load_reduction_pct=0.852875 moved_to_echo=16524 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4074, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4, 0, 6, 0, 0, 0, 0, 4065, 5, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 4010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 3923, 0, 0, 0, 0, 0, 0, 0, 3998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3646, 0, 0, 2195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 636, 0, 1804, 143, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4090, 4034, 3926, 3998, 5843, 2681, 4097] avg=4096.0000 max=5843 max_rank=5 min=2681 min_rank=6 imbalance_ratio=1.426514 max_load_reduction=26925 max_load_reduction_pct=0.821686 moved_to_echo=16619 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4079, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 19, 0, 0, 0, 0, 4043, 5, 0, 0, 0, 0, 0, 0, 5, 24, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 38, 0, 0, 0, 0, 1, 3923, 0, 0, 0, 0, 0, 0, 0, 3991, 0, 0, 0, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 2586, 0, 0, 2206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 993, 0, 1814, 143, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4111, 4109, 3966, 4532, 4809, 3048, 4089] avg=4096.0000 max=4809 max_rank=5 min=3048 min_rank=6 imbalance_ratio=1.174072 max_load_reduction=27959 max_load_reduction_pct=0.853241 moved_to_echo=16478 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4045, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 8, 0, 0, 0, 0, 0, 0, 4067, 5, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 3923, 0, 0, 0, 0, 0, 0, 0, 3996, 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 1752, 0, 0, 1958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2759, 0, 1609, 143, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4070, 4141, 4106, 3936, 4102, 3716, 4609, 4088] avg=4096.0000 max=4609 max_rank=6 min=3716 min_rank=5 imbalance_ratio=1.125244 max_load_reduction=28159 max_load_reduction_pct=0.859344 moved_to_echo=16515 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 4069, 5, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3923, 0, 0, 0, 0, 0, 0, 0, 3997, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3851, 0, 0, 2013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 637, 0, 1655, 143, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4087, 4103, 3926, 4058, 5864, 2533, 4078] avg=4096.0000 max=5864 max_rank=5 min=2533 min_rank=6 imbalance_ratio=1.431641 max_load_reduction=26904 max_load_reduction_pct=0.821045 moved_to_echo=16533 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4008, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 1, 0, 0, 0, 0, 4075, 10, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 3927, 0, 0, 0, 0, 0, 0, 0, 3991, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 593, 0, 0, 2191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0, 3560, 0, 1807, 145, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4022, 4108, 4063, 3972, 4024, 2784, 5693, 4102] avg=4096.0000 max=5693 max_rank=6 min=2784 min_rank=5 imbalance_ratio=1.389893 max_load_reduction=27075 max_load_reduction_pct=0.826263 moved_to_echo=16603 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 16128, 0, 0, 16640, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 0, 0, 17152, 0, 13696, 1152, 896, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 128, 128, 128, 0, 0, 0, 0, 32512, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 0, 128, 128, 1536, 31360, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 32640, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32000, 0, 0, 128, 768, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 256, 0, 0, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 0, 128, 0, 0, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 3988, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2157, 0, 0, 4044, 0, 0, 3, 0, 0, 0, 0, 1791, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 1, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3059, 0, 0, 3973, 0, 0, 0, 0, 0, 0, 0, 764, 287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 3972, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 2691, 0, 0, 4092, 0, 0, 25, 0, 0, 0, 0, 1501, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 2, 3727, 0, 0, 10, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3893, 0, 0, 4095, 0, 0, 48, 0, 0, 0, 0, 100, 426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4052, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3163, 0, 0, 4015, 5, 0, 26, 0, 0, 0, 0, 940, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4086, 0, 29, 0, 0, 0, 4078, 0, 13, 3, 0, 2, 4044, 0, 0, 0, 0, 0, 1333, 5, 0, 0, 0, 10, 12, 0, 0, 4, 0, 0, 0, 1, 0, 4078, 20, 10, 16, 1533, 22, 0, 4076, 0, 13, 28, 0, 11, 0, 0, 1379, 5, 0, 0, 0, 0, 0, 23, 0, 0, 0, 3854, 4080, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 1, 4029, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 3867, 0, 0, 4093, 0, 0, 4, 0, 0, 0, 0, 10, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 3597, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 3765, 0, 0, 4096, 0, 0, 18, 0, 0, 0, 0, 458, 377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4096, 0, 0, 0, 0, 0, 147, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 0, 0, 0, 3889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1077, 0, 0, 0, 3059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3973, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 764, 287, 0, 0, 0, 0, 2862, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4244, 4101, 3997, 4094, 4136, 3983, 4072, 4141] avg=4096.0000 max=4244 max_rank=0 min=3983 min_rank=5 imbalance_ratio=1.036133 max_load_reduction=28524 max_load_reduction_pct=0.870483 moved_to_echo=16332 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3988, 0, 0, 0, 0, 0, 147, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1077, 0, 0, 0, 2157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 0, 3, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1791, 341, 0, 0, 0, 0, 2862, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4135, 4101, 3891, 4094, 3234, 4057, 5153, 4103] avg=4096.0000 max=5153 max_rank=6 min=3234 min_rank=4 imbalance_ratio=1.258057 max_load_reduction=27615 max_load_reduction_pct=0.842743 moved_to_echo=16547 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 3727, 0, 0, 10, 0, 0, 147, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 3889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1077, 0, 0, 0, 3893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 48, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 426, 0, 0, 0, 0, 2862, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3886, 4101, 3910, 4094, 4970, 4153, 3547, 4107] avg=4096.0000 max=4970 max_rank=4 min=3547 min_rank=6 imbalance_ratio=1.213379 max_load_reduction=27798 max_load_reduction_pct=0.848328 moved_to_echo=16777 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4086, 0, 29, 0, 0, 0, 146, 0, 0, 0, 0, 0, 0, 0, 5, 0, 13, 3, 0, 2, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1333, 5, 0, 0, 3874, 0, 0, 0, 0, 0, 0, 0, 0, 10, 12, 0, 0, 4, 0, 0, 4073, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1073, 20, 10, 16, 1533, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 4076, 0, 13, 28, 0, 11, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1379, 5, 0, 0, 0, 0, 2849, 159, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 3854, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4261, 4067, 5212, 4102, 2653, 4160, 4392, 3921] avg=4096.0000 max=5212 max_rank=2 min=2653 min_rank=4 imbalance_ratio=1.272461 max_load_reduction=27556 max_load_reduction_pct=0.840942 moved_to_echo=15126 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3972, 0, 0, 0, 0, 0, 147, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1075, 0, 0, 0, 2691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 25, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1501, 50, 0, 0, 0, 0, 2858, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4101, 3891, 4094, 3766, 4127, 4568, 4102] avg=4096.0000 max=4568 max_rank=6 min=3766 min_rank=4 imbalance_ratio=1.115234 max_load_reduction=28200 max_load_reduction_pct=0.860596 moved_to_echo=16563 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4052, 0, 0, 0, 0, 0, 147, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 3889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1077, 0, 0, 0, 3163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4015, 5, 0, 26, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 940, 90, 0, 0, 0, 0, 2862, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4199, 4101, 3894, 4094, 4240, 4056, 4051, 4133] avg=4096.0000 max=4240 max_rank=4 min=3894 min_rank=2 imbalance_ratio=1.035156 max_load_reduction=28528 max_load_reduction_pct=0.870605 moved_to_echo=16480 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3597, 0, 0, 0, 0, 0, 146, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 3889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1072, 0, 0, 0, 3765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 18, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 458, 377, 0, 0, 0, 0, 2846, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3743, 4100, 3898, 4095, 4837, 4124, 3840, 4131] avg=4096.0000 max=4837 max_rank=4 min=3743 min_rank=0 imbalance_ratio=1.180908 max_load_reduction=27931 max_load_reduction_pct=0.852386 moved_to_echo=16932 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4029, 0, 0, 0, 0, 0, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 0, 0, 0, 3891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1065, 0, 0, 0, 3867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 112, 0, 0, 0, 0, 2857, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4181, 4096, 4075, 4101, 4932, 4108, 3145, 4130] avg=4096.0000 max=4932 max_rank=4 min=3145 min_rank=6 imbalance_ratio=1.204102 max_load_reduction=27836 max_load_reduction_pct=0.849487 moved_to_echo=16315 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 0, 8704, 128, 128, 128, 24192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 7040, 1792, 0, 0, 0, 0, 22912, 1280, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 31616, 0, 128, 128, 0, 0, 1280, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 0, 32512, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 1664, 128, 0, 0, 31104, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 0, 0, 128, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 32512, 128, 128, 256, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=32896 slack_slots=2816 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True + [2026-09-21 08:25:38] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 257.0 | throughput per GPU (TFLOP/s/GPU): 125.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.590282E+00 | loss scale: 1.0 | grad norm: 3.381 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-09-21 08:25:38 +[rank0]:[W921 08:25:40.060674379 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0921 08:25:43.109000 641899 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0921 08:25:43.110000 641899 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 642176 closing signal SIGTERM +W0921 08:25:43.110000 641899 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 642177 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 641899 got signal: 15 diff --git a/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__cap1p08984375__forcebal/rep1/mem_samples.csv b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__cap1p08984375__forcebal/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..05e8fcaf574ba6fad5e722d9fd563e9948ccbc9c --- /dev/null +++ b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__cap1p08984375__forcebal/rep1/mem_samples.csv @@ -0,0 +1,224 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 2 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 821 +1, 821 +2, 821 +3, 821 +4, 819 +5, 819 +6, 819 +7, 819 +0, 1537 +1, 1537 +2, 1537 +3, 1537 +4, 1537 +5, 1537 +6, 1537 +7, 1537 +0, 1875 +1, 1875 +2, 1875 +3, 1823 +4, 1875 +5, 1823 +6, 1823 +7, 1823 +0, 13303 +1, 13319 +2, 13319 +3, 13319 +4, 13303 +5, 13319 +6, 13319 +7, 13303 +0, 14197 +1, 14199 +2, 14199 +3, 14201 +4, 14195 +5, 14199 +6, 14201 +7, 14195 +0, 14829 +1, 14831 +2, 14831 +3, 14833 +4, 14827 +5, 14831 +6, 14833 +7, 14827 +0, 14913 +1, 14915 +2, 14915 +3, 14917 +4, 14911 +5, 14915 +6, 14917 +7, 14911 +0, 15067 +1, 15085 +2, 15053 +3, 15083 +4, 15083 +5, 15067 +6, 15085 +7, 15085 +0, 15709 +1, 15711 +2, 15711 +3, 15713 +4, 15707 +5, 15711 +6, 15713 +7, 15707 +0, 15849 +1, 15851 +2, 15851 +3, 15853 +4, 15847 +5, 15851 +6, 15853 +7, 15847 +0, 16133 +1, 16133 +2, 16133 +3, 16135 +4, 16129 +5, 16135 +6, 16135 +7, 16129 +0, 16347 +1, 16349 +2, 16349 +3, 16351 +4, 16345 +5, 16349 +6, 16349 +7, 16345 +0, 28507 +1, 28515 +2, 28525 +3, 28517 +4, 28527 +5, 28499 +6, 28509 +7, 28525 +0, 28965 +1, 28965 +2, 28975 +3, 28967 +4, 28961 +5, 28965 +6, 28967 +7, 28961 +0, 28967 +1, 28967 +2, 28977 +3, 28969 +4, 28963 +5, 28967 +6, 28969 +7, 28963 +0, 28967 +1, 28967 +2, 28977 +3, 28969 +4, 28963 +5, 28967 +6, 28969 +7, 28963 +0, 28967 +1, 28967 +2, 28977 +3, 28969 +4, 28963 +5, 28967 +6, 28969 +7, 28963 +0, 29577 +1, 29577 +2, 29587 +3, 29579 +4, 29573 +5, 29897 +6, 29579 +7, 29573 +0, 30527 +1, 30549 +2, 30563 +3, 30557 +4, 30525 +5, 30851 +6, 30529 +7, 30553 +0, 37859 +1, 37885 +2, 37897 +3, 37863 +4, 37873 +5, 38209 +6, 37861 +7, 37873 +0, 37859 +1, 37885 +2, 37897 +3, 37863 +4, 37873 +5, 38209 +6, 37861 +7, 37873 +0, 37709 +1, 37735 +2, 37747 +3, 37713 +4, 37723 +5, 38059 +6, 37711 +7, 37723 +0, 563 +1, 566 +2, 566 +3, 566 +4, 566 +5, 822 +6, 37065 +7, 1846 diff --git a/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__cap1p08984375__forcebal/rep1/run.env b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__cap1p08984375__forcebal/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..f95158201ca149a21313a1286c06c8c47d072530 --- /dev/null +++ b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__cap1p08984375__forcebal/rep1/run.env @@ -0,0 +1,102 @@ +backend=echo +config=OLMOE_1B_7B_L16 +family=tierB +source_model=OLMoE-1B-7B-0924 +standard_num_layers=16 +base_rung= +repo=/workspace/Megatron-MoE-Echo +branch=token-drop-analysis +commit=b099fd84 +num_layers=16 +hidden=2048 +ffn_hidden=1024 +moe_ffn_hidden=1024 +num_experts=64 +topk=8 +experts_per_rank=8 +expert_params=6291456 +heads=16 +kv_groups=16 +kv_channels=128 +qk_layernorm=0 +shared_ffn=0 +shared_expert_overlap=0 +moe_token_dispatcher=flex +mla=0 +q_lora_rank= +kv_lora_rank=512 +qk_head_dim=128 +qk_pos_emb_head_dim=64 +v_head_dim=128 +dense_first_layers=0 +moe_layers=16 +rotary_base=10000 +rotary_percent=1.0 +rotary_scaling= +mscale= +mscale_all_dim= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-2 +router_pre_softmax=1 +router_fusion=0 +router_dtype=fp32 +router_groups= +router_group_topk= +router_score_function=softmax +router_expert_bias=0 +router_bias_update_rate= +router_topk_scaling_factor= +shared_expert_gate=0 +apply_layernorm_1p=0 +attention_output_gate=0 +linear_attention_type= +linear_attention_freq= +linear_conv_kernel_dim= +linear_key_head_dim= +linear_value_head_dim= +linear_num_key_heads= +linear_num_value_heads= +mtp_num_layers=0 +mtp_loss_scaling_factor=0.1 +no_weight_decay_cond_type= +max_position_embeddings=4096 +init_method_std=0.02 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-5 +seq_length=4096 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=32768 +mean_tokens_per_expert=4096 +slot_policy=matched +slots_per_rank=8 +capacity_mode=static +capacity=1.08984375 +cuda_graph=0 +probe_mode=1 +async_grad_reduce=0 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=1 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_mode=mxfp8 +fp8_recipe=mxfp8 +training_seed=1234 +learning_rate=1e-4 +min_learning_rate=1.0e-5 +lr_warmup_iters=5 +state_dump_enable=0 +state_dump_dir= +moonep_assert_no_pending_reduce=0 diff --git a/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__cap1p08984375__forcebal/rep1/train.log b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__cap1p08984375__forcebal/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..66acb89a29f7bd95a861465e02189bbed6a23886 --- /dev/null +++ b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__cap1p08984375__forcebal/rep1/train.log @@ -0,0 +1,8900 @@ +[08:25:45] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +============================================================== + OLMOE_1B_7B_L16 / echo (slots/rank=8 policy=matched) + E=64 K=8 H'=1024 m=8 + L=16 seq=4096 gbs=8 (1 microbatches) + cap_mode=static cap=1.08984375 cg=0 recompute=none + memory_dump=none snapshots=/workspace/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__cap1p08984375__forcebal/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ b099fd84 +============================================================== +W0921 08:25:46.890000 650825 torch/distributed/run.py:766] +W0921 08:25:46.890000 650825 torch/distributed/run.py:766] ***************************************** +W0921 08:25:46.890000 650825 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0921 08:25:46.890000 650825 torch/distributed/run.py:766] ***************************************** +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 16 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 1024 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 2048 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.02 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 4096 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.01 + moe_debug_drop_logging .......................... True + moe_debug_drop_logging_interval ................. 1 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ True + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. True + moe_enable_hybridep ............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 1024 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ 64 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.08984375 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. True + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... True + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-05 + normalization ................................... RMSNorm + num_attention_heads ............................. 16 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 64 + num_layers ...................................... 16 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 16 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 25 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 + +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 7 is connected to 76 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. +Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[rank5]:[W921 08:26:03.719038314 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W921 08:26:03.719040427 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W921 08:26:03.719060717 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W921 08:26:03.719148274 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W921 08:26:03.719148294 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W921 08:26:03.722359938 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W921 08:26:03.723914261 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.037 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W921 08:26:03.877146979 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.840 seconds +time to initialize megatron (seconds): 7.522 +[after megatron is initialized] datetime: 2026-09-21 08:26:09 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 1281951744 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 7 +Params for bucket 1 (103022592 elements, 103022592 padded size): + module.output_layer.weight +Params for bucket 2 (46542848 elements, 46542848 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.self_attention.linear_qkv.weight + module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.self_attention.linear_qkv.weight + module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.linear_qkv.weight +Params for bucket 3 (50737152 elements, 50737152 padded size): + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.self_attention.linear_qkv.weight + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.linear_qkv.weight +Params for bucket 4 (50737152 elements, 50737152 padded size): + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.weight +Params for bucket 5 (50737152 elements, 50737152 padded size): + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight +Params for bucket 6 (50737152 elements, 50737152 padded size): + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight +Params for bucket 7 (124131328 elements, 124131328 padded size): + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 20 +Params for bucket 1 (41943040 elements, 41943040 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 +Params for bucket 2 (41943040 elements, 41943040 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 +Params for bucket 3 (41943040 elements, 41943040 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 +Params for bucket 4 (41943040 elements, 41943040 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 +Params for bucket 5 (41943040 elements, 41943040 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 +Params for bucket 6 (41943040 elements, 41943040 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 +Params for bucket 7 (41943040 elements, 41943040 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 +Params for bucket 8 (41943040 elements, 41943040 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 +Params for bucket 9 (41943040 elements, 41943040 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 +Params for bucket 10 (41943040 elements, 41943040 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 +Params for bucket 11 (41943040 elements, 41943040 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 +Params for bucket 12 (41943040 elements, 41943040 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 +Params for bucket 13 (41943040 elements, 41943040 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 +Params for bucket 14 (41943040 elements, 41943040 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 +Params for bucket 15 (41943040 elements, 41943040 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 +Params for bucket 16 (41943040 elements, 41943040 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 +Params for bucket 17 (41943040 elements, 41943040 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 +Params for bucket 18 (41943040 elements, 41943040 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 +Params for bucket 19 (41943040 elements, 41943040 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 +Params for bucket 20 (8388608 elements, 8388608 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-09-21 08:26:10 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 96 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-09-21 08:26:10 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (315.54, 318.41) + train/valid/test-data-iterators-setup ..........: (65.81, 80.60) +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-09-21 08:26:10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[539, 519, 527, 475, 515, 535, 551, 550, 525, 528, 516, 465, 522, 484, 503, 512, 523, 494, 514, 493, 485, 482, 454, 482, 493, 537, 524, 494, 535, 520, 556, 500, 489, 510, 517, 504, 508, 500, 496, 488, 515, 498, 552, 504, 488, 527, 519, 533, 500, 513, 510, 496, 508, 496, 534, 533, 512, 534, 531, 525, 531, 473, 560, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[552, 489, 547, 518, 500, 529, 512, 547, 504, 536, 523, 507, 548, 482, 548, 526, 510, 532, 463, 532, 485, 504, 533, 519, 506, 516, 511, 496, 540, 507, 551, 517, 511, 460, 489, 504, 476, 481, 547, 475, 527, 475, 519, 502, 502, 493, 516, 509, 524, 500, 498, 514, 498, 501, 559, 481, 484, 560, 522, 498, 508, 508, 532, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[559, 498, 501, 525, 503, 530, 508, 502, 535, 529, 527, 526, 526, 490, 511, 530, 521, 520, 489, 500, 542, 533, 509, 498, 498, 533, 527, 510, 522, 509, 511, 485, 521, 491, 542, 472, 472, 483, 544, 508, 513, 467, 486, 505, 519, 461, 519, 526, 571, 479, 508, 535, 522, 520, 494, 531, 484, 505, 504, 560, 530, 501, 497, 491] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[547, 506, 492, 505, 480, 535, 489, 478, 523, 494, 521, 528, 479, 521, 526, 519, 482, 502, 529, 492, 513, 496, 488, 492, 521, 480, 491, 554, 547, 499, 523, 501, 546, 509, 505, 489, 493, 537, 502, 500, 533, 558, 502, 507, 489, 532, 532, 505, 539, 542, 541, 484, 527, 522, 541, 563, 494, 517, 503, 476, 520, 499, 506, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[531, 518, 539, 521, 546, 504, 489, 471, 489, 503, 509, 528, 508, 519, 497, 470, 497, 508, 512, 521, 516, 565, 554, 493, 522, 476, 506, 538, 500, 494, 457, 520, 541, 497, 517, 520, 482, 533, 492, 514, 512, 504, 512, 487, 487, 522, 508, 484, 510, 551, 496, 500, 522, 530, 538, 540, 559, 517, 517, 491, 522, 500, 518, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[547, 504, 520, 509, 523, 518, 504, 507, 526, 518, 518, 525, 493, 530, 484, 536, 524, 460, 550, 504, 515, 511, 499, 501, 513, 493, 513, 497, 498, 509, 563, 479, 514, 539, 489, 527, 502, 525, 514, 540, 511, 498, 498, 528, 515, 503, 483, 510, 514, 533, 527, 490, 507, 538, 495, 505, 496, 496, 502, 514, 531, 524, 517, 492] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[493, 516, 502, 523, 493, 481, 510, 528, 531, 532, 515, 528, 524, 499, 484, 481, 545, 541, 541, 481, 517, 523, 518, 533, 506, 524, 509, 529, 500, 538, 513, 509, 489, 533, 491, 544, 537, 542, 487, 510, 537, 521, 527, 528, 474, 496, 511, 526, 534, 482, 503, 516, 521, 513, 495, 505, 509, 500, 475, 495, 483, 533, 475, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[492, 493, 512, 532, 511, 525, 513, 511, 503, 505, 496, 472, 477, 511, 526, 502, 490, 530, 512, 486, 548, 498, 516, 534, 536, 530, 538, 509, 521, 521, 533, 543, 487, 519, 505, 505, 502, 467, 564, 515, 525, 564, 507, 478, 526, 514, 511, 543, 508, 500, 517, 482, 519, 498, 515, 509, 488, 529, 514, 504, 523, 526, 468, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[526, 506, 492, 505, 480, 535, 489, 478, 0, 0, 0, 0, 0, 0, 0, 0, 523, 494, 521, 528, 479, 521, 526, 519, 1, 0, 0, 0, 0, 0, 0, 0, 482, 502, 529, 492, 513, 496, 488, 492, 10, 4, 0, 0, 0, 0, 0, 0, 521, 480, 491, 554, 547, 499, 501, 501, 0, 0, 0, 0, 0, 0, 0, 0, 546, 509, 505, 489, 493, 537, 502, 500, 29, 3, 0, 0, 0, 0, 0, 0, 533, 558, 502, 507, 489, 532, 532, 505, 10, 0, 0, 0, 0, 0, 0, 0, 510, 542, 541, 484, 527, 522, 541, 563, 0, 0, 0, 0, 0, 0, 0, 0, 494, 517, 503, 476, 520, 499, 506, 502, 15, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4011, 4112, 4008, 4094, 4113, 4168, 4230, 4032] avg=4096.0000 max=4230 max_rank=6 min=4008 min_rank=2 imbalance_ratio=1.032715 max_load_reduction=28538 max_load_reduction_pct=0.870911 moved_to_echo=16543 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[526, 504, 520, 509, 523, 518, 504, 507, 0, 0, 0, 0, 0, 0, 0, 0, 526, 518, 518, 525, 493, 530, 484, 536, 1, 0, 0, 0, 0, 0, 0, 0, 524, 460, 550, 504, 515, 511, 499, 501, 10, 4, 0, 0, 0, 0, 0, 0, 513, 493, 513, 497, 498, 509, 540, 479, 0, 0, 0, 0, 0, 0, 0, 0, 514, 539, 489, 527, 502, 525, 514, 540, 28, 3, 0, 0, 0, 0, 0, 0, 511, 498, 498, 528, 515, 503, 483, 510, 10, 0, 0, 0, 0, 0, 0, 0, 486, 533, 527, 490, 507, 538, 495, 505, 0, 0, 0, 0, 0, 0, 0, 0, 496, 496, 502, 514, 531, 524, 517, 492, 16, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4131, 4078, 4042, 4181, 4056, 4081, 4088] avg=4096.0000 max=4181 max_rank=4 min=4042 min_rank=3 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16406 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[474, 516, 502, 523, 493, 481, 510, 528, 0, 0, 0, 0, 0, 0, 0, 0, 531, 532, 515, 528, 524, 499, 484, 481, 1, 0, 0, 0, 0, 0, 0, 0, 545, 541, 541, 481, 517, 523, 518, 533, 9, 4, 0, 0, 0, 0, 0, 0, 506, 524, 509, 529, 500, 538, 492, 509, 0, 0, 0, 0, 0, 0, 0, 0, 489, 533, 491, 544, 537, 542, 487, 510, 29, 3, 0, 0, 0, 0, 0, 0, 537, 521, 527, 528, 474, 496, 511, 526, 9, 0, 0, 0, 0, 0, 0, 0, 505, 482, 503, 516, 521, 513, 495, 505, 0, 0, 0, 0, 0, 0, 0, 0, 509, 500, 475, 495, 483, 533, 475, 509, 14, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4027, 4095, 4212, 4107, 4165, 4129, 4040, 3993] avg=4096.0000 max=4212 max_rank=2 min=3993 min_rank=7 imbalance_ratio=1.028320 max_load_reduction=28556 max_load_reduction_pct=0.871460 moved_to_echo=16327 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[537, 498, 501, 525, 503, 530, 508, 502, 0, 0, 0, 0, 0, 0, 0, 0, 535, 529, 527, 526, 526, 490, 511, 530, 1, 0, 0, 0, 0, 0, 0, 0, 521, 520, 489, 500, 542, 533, 509, 498, 10, 4, 0, 0, 0, 0, 0, 0, 498, 533, 527, 510, 522, 509, 490, 485, 0, 0, 0, 0, 0, 0, 0, 0, 521, 491, 542, 472, 472, 483, 544, 508, 31, 3, 0, 0, 0, 0, 0, 0, 513, 467, 486, 505, 519, 461, 519, 526, 11, 0, 0, 0, 0, 0, 0, 0, 540, 479, 508, 535, 522, 520, 494, 531, 0, 0, 0, 0, 0, 0, 0, 0, 484, 505, 504, 560, 530, 501, 497, 491, 14, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4175, 4126, 4074, 4067, 4007, 4129, 4086] avg=4096.0000 max=4175 max_rank=1 min=4007 min_rank=5 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16289 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[495, 518, 539, 521, 546, 504, 489, 471, 0, 0, 0, 0, 0, 0, 0, 0, 489, 503, 509, 528, 508, 519, 497, 470, 8, 0, 0, 0, 0, 0, 0, 0, 497, 508, 512, 521, 516, 565, 554, 493, 13, 5, 0, 0, 0, 0, 0, 0, 522, 476, 506, 538, 500, 494, 429, 520, 0, 0, 0, 0, 0, 0, 0, 0, 541, 497, 517, 520, 482, 533, 492, 514, 30, 6, 0, 0, 0, 0, 0, 0, 512, 504, 512, 487, 487, 522, 508, 484, 15, 0, 0, 0, 0, 0, 0, 0, 480, 551, 496, 500, 522, 530, 538, 540, 0, 0, 0, 0, 0, 0, 0, 0, 559, 517, 517, 491, 522, 500, 518, 524, 17, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4031, 4184, 3985, 4132, 4031, 4157, 4165] avg=4096.0000 max=4184 max_rank=2 min=3985 min_rank=3 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16485 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[518, 519, 527, 475, 515, 535, 551, 550, 0, 0, 0, 0, 0, 0, 0, 0, 525, 528, 516, 465, 522, 484, 503, 512, 1, 0, 0, 0, 0, 0, 0, 0, 523, 494, 514, 493, 485, 482, 454, 482, 10, 4, 0, 0, 0, 0, 0, 0, 493, 537, 524, 494, 535, 520, 533, 500, 0, 0, 0, 0, 0, 0, 0, 0, 489, 510, 517, 504, 508, 500, 496, 488, 27, 3, 0, 0, 0, 0, 0, 0, 515, 498, 552, 504, 488, 527, 519, 533, 10, 0, 0, 0, 0, 0, 0, 0, 473, 513, 510, 496, 508, 496, 534, 533, 0, 0, 0, 0, 0, 0, 0, 0, 512, 534, 531, 525, 531, 473, 560, 512, 16, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4190, 4056, 3941, 4136, 4042, 4146, 4063, 4194] avg=4096.0000 max=4194 max_rank=7 min=3941 min_rank=2 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16445 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[530, 489, 547, 518, 500, 529, 512, 547, 0, 0, 0, 0, 0, 0, 0, 0, 504, 536, 523, 507, 548, 482, 548, 526, 1, 0, 0, 0, 0, 0, 0, 0, 510, 532, 463, 532, 485, 504, 533, 519, 10, 4, 0, 0, 0, 0, 0, 0, 506, 516, 511, 496, 540, 507, 528, 517, 0, 0, 0, 0, 0, 0, 0, 0, 511, 460, 489, 504, 476, 481, 547, 475, 28, 3, 0, 0, 0, 0, 0, 0, 527, 475, 519, 502, 502, 493, 516, 509, 11, 0, 0, 0, 0, 0, 0, 0, 496, 500, 498, 514, 498, 501, 559, 481, 0, 0, 0, 0, 0, 0, 0, 0, 484, 560, 522, 498, 508, 508, 532, 505, 16, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4172, 4175, 4092, 4121, 3974, 4054, 4047, 4133] avg=4096.0000 max=4175 max_rank=1 min=3974 min_rank=4 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16208 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[473, 493, 512, 532, 511, 525, 513, 511, 0, 0, 0, 0, 0, 0, 0, 0, 503, 505, 496, 472, 477, 511, 526, 502, 1, 0, 0, 0, 0, 0, 0, 0, 490, 530, 512, 486, 548, 498, 516, 534, 9, 4, 0, 0, 0, 0, 0, 0, 536, 530, 538, 509, 521, 521, 511, 543, 0, 0, 0, 0, 0, 0, 0, 0, 487, 519, 505, 505, 502, 467, 564, 515, 27, 3, 0, 0, 0, 0, 0, 0, 525, 564, 507, 478, 526, 514, 511, 543, 9, 0, 0, 0, 0, 0, 0, 0, 481, 500, 517, 482, 519, 498, 515, 509, 0, 0, 0, 0, 0, 0, 0, 0, 488, 529, 514, 504, 523, 526, 468, 510, 15, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4070, 3993, 4127, 4209, 4094, 4177, 4021, 4077] avg=4096.0000 max=4209 max_rank=3 min=3993 min_rank=1 imbalance_ratio=1.027588 max_load_reduction=28559 max_load_reduction_pct=0.871552 moved_to_echo=16369 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[486, 531, 540, 506, 545, 501, 561, 527, 519, 480, 488, 505, 534, 488, 534, 492, 539, 525, 484, 523, 498, 503, 493, 502, 504, 516, 551, 481, 515, 528, 498, 514, 489, 464, 546, 537, 540, 494, 505, 525, 517, 466, 547, 508, 529, 524, 562, 465, 499, 470, 473, 552, 536, 526, 529, 474, 553, 507, 472, 504, 507, 495, 529, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[522, 470, 513, 496, 511, 439, 532, 541, 533, 494, 523, 515, 477, 507, 508, 507, 496, 490, 522, 499, 509, 486, 539, 573, 494, 512, 506, 510, 542, 514, 522, 533, 544, 499, 549, 538, 525, 511, 521, 530, 548, 502, 502, 486, 514, 497, 496, 554, 501, 496, 510, 499, 543, 526, 485, 513, 539, 481, 538, 540, 526, 475, 474, 471] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[496, 526, 540, 539, 515, 504, 476, 535, 509, 542, 530, 500, 507, 509, 481, 530, 492, 531, 489, 537, 507, 497, 519, 517, 540, 530, 559, 486, 529, 505, 501, 533, 489, 528, 529, 489, 489, 504, 498, 503, 505, 534, 514, 531, 513, 488, 507, 537, 483, 475, 508, 529, 475, 475, 551, 528, 489, 541, 524, 518, 479, 506, 511, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[526, 517, 542, 515, 524, 467, 514, 504, 511, 490, 510, 486, 535, 506, 498, 508, 519, 528, 526, 502, 490, 532, 533, 513, 526, 525, 533, 512, 530, 558, 499, 504, 520, 526, 483, 468, 501, 497, 516, 501, 510, 517, 490, 558, 493, 520, 531, 523, 511, 520, 469, 494, 482, 506, 518, 518, 514, 510, 500, 531, 531, 527, 467, 533] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[508, 521, 525, 467, 498, 544, 529, 488, 492, 527, 529, 508, 494, 501, 507, 506, 544, 523, 543, 527, 536, 493, 517, 505, 537, 493, 488, 507, 505, 527, 522, 535, 516, 473, 518, 501, 516, 486, 537, 510, 517, 481, 498, 526, 509, 494, 484, 524, 545, 504, 493, 508, 525, 524, 481, 502, 511, 527, 529, 544, 527, 497, 455, 560] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[495, 491, 538, 524, 525, 498, 551, 505, 528, 486, 516, 462, 516, 519, 545, 555, 527, 515, 517, 506, 516, 542, 545, 517, 492, 503, 529, 531, 522, 491, 528, 521, 519, 494, 512, 487, 446, 494, 497, 470, 533, 494, 474, 565, 505, 528, 507, 501, 508, 492, 505, 518, 538, 537, 543, 520, 467, 525, 500, 497, 496, 513, 508, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[521, 543, 508, 487, 504, 503, 504, 531, 500, 514, 541, 538, 553, 517, 530, 467, 512, 515, 517, 513, 527, 487, 486, 489, 483, 527, 523, 524, 498, 508, 507, 489, 497, 524, 540, 515, 488, 525, 495, 524, 500, 504, 525, 529, 473, 502, 522, 516, 541, 478, 494, 498, 472, 544, 495, 561, 513, 521, 557, 512, 523, 514, 507, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[500, 531, 498, 494, 478, 493, 496, 503, 537, 523, 503, 524, 522, 529, 547, 549, 470, 523, 496, 516, 527, 495, 537, 465, 499, 517, 515, 523, 532, 511, 487, 525, 562, 479, 525, 507, 484, 491, 511, 484, 541, 528, 515, 503, 475, 491, 504, 490, 561, 510, 549, 525, 497, 498, 510, 515, 506, 543, 533, 509, 530, 518, 502, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[486, 531, 540, 506, 545, 501, 561, 527, 0, 0, 0, 0, 0, 0, 0, 0, 519, 480, 488, 505, 534, 488, 526, 492, 0, 0, 0, 0, 0, 0, 0, 0, 539, 525, 484, 523, 498, 503, 483, 502, 0, 0, 0, 0, 0, 0, 0, 0, 504, 516, 520, 481, 515, 528, 498, 514, 0, 0, 0, 0, 0, 0, 0, 0, 489, 464, 546, 537, 540, 494, 505, 525, 31, 4, 0, 0, 0, 0, 0, 0, 517, 466, 547, 508, 529, 524, 562, 465, 2, 0, 0, 0, 0, 0, 0, 0, 499, 470, 473, 552, 536, 526, 529, 474, 6, 6, 0, 0, 0, 0, 0, 0, 553, 507, 472, 504, 507, 495, 529, 513, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4197, 4032, 4057, 4076, 4135, 4120, 4071, 4080] avg=4096.0000 max=4197 max_rank=0 min=4032 min_rank=1 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16406 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[508, 521, 525, 467, 498, 544, 529, 488, 0, 0, 0, 0, 0, 0, 0, 0, 492, 527, 529, 508, 494, 501, 499, 506, 0, 0, 0, 0, 0, 0, 0, 0, 544, 523, 543, 527, 536, 493, 506, 505, 0, 0, 0, 0, 0, 0, 0, 0, 537, 493, 461, 507, 505, 527, 522, 535, 0, 0, 0, 0, 0, 0, 0, 0, 516, 473, 518, 501, 516, 486, 537, 510, 27, 5, 0, 0, 0, 0, 0, 0, 517, 481, 498, 526, 509, 494, 484, 524, 2, 0, 0, 0, 0, 0, 0, 0, 545, 504, 493, 508, 525, 524, 481, 502, 6, 6, 0, 0, 0, 0, 0, 0, 511, 527, 529, 544, 527, 497, 455, 560, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4056, 4177, 4087, 4089, 4035, 4094, 4150] avg=4096.0000 max=4177 max_rank=2 min=4035 min_rank=5 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[495, 491, 538, 524, 525, 498, 551, 505, 0, 0, 0, 0, 0, 0, 0, 0, 528, 486, 516, 462, 516, 519, 537, 555, 0, 0, 0, 0, 0, 0, 0, 0, 527, 515, 517, 506, 516, 542, 534, 517, 0, 0, 0, 0, 0, 0, 0, 0, 492, 503, 499, 531, 522, 491, 528, 521, 0, 0, 0, 0, 0, 0, 0, 0, 519, 494, 512, 487, 446, 494, 497, 470, 30, 5, 0, 0, 0, 0, 0, 0, 533, 494, 474, 565, 505, 528, 507, 501, 2, 0, 0, 0, 0, 0, 0, 0, 508, 492, 505, 518, 538, 537, 543, 520, 6, 6, 0, 0, 0, 0, 0, 0, 467, 525, 500, 497, 496, 513, 508, 519, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4127, 4119, 4174, 4087, 3954, 4109, 4173, 4025] avg=4096.0000 max=4174 max_rank=2 min=3954 min_rank=4 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16261 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[522, 470, 513, 496, 511, 439, 532, 541, 0, 0, 0, 0, 0, 0, 0, 0, 533, 494, 523, 515, 477, 507, 500, 507, 0, 0, 0, 0, 0, 0, 0, 0, 496, 490, 522, 499, 509, 486, 528, 573, 0, 0, 0, 0, 0, 0, 0, 0, 494, 512, 477, 510, 542, 514, 522, 533, 0, 0, 0, 0, 0, 0, 0, 0, 544, 499, 549, 538, 525, 511, 521, 530, 29, 5, 0, 0, 0, 0, 0, 0, 548, 502, 502, 486, 514, 497, 496, 554, 2, 0, 0, 0, 0, 0, 0, 0, 501, 496, 510, 499, 543, 526, 485, 513, 6, 6, 0, 0, 0, 0, 0, 0, 539, 481, 538, 540, 526, 475, 474, 471, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4024, 4056, 4103, 4104, 4251, 4101, 4085, 4044] avg=4096.0000 max=4251 max_rank=4 min=4024 min_rank=0 imbalance_ratio=1.037842 max_load_reduction=28517 max_load_reduction_pct=0.870270 moved_to_echo=16481 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[526, 517, 542, 515, 524, 467, 514, 504, 6, 0, 0, 0, 0, 0, 0, 0, 511, 490, 510, 486, 535, 506, 481, 508, 0, 0, 0, 0, 0, 0, 0, 0, 519, 528, 526, 502, 490, 532, 514, 513, 0, 0, 0, 0, 0, 0, 0, 0, 526, 525, 499, 512, 530, 558, 499, 504, 0, 0, 0, 0, 0, 0, 0, 0, 520, 526, 483, 468, 501, 497, 516, 501, 34, 9, 0, 0, 0, 0, 0, 0, 510, 517, 490, 558, 493, 520, 531, 523, 7, 1, 0, 0, 0, 0, 0, 0, 511, 520, 469, 494, 482, 506, 518, 518, 10, 10, 0, 0, 0, 0, 0, 0, 514, 503, 500, 531, 531, 527, 467, 533, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4027, 4124, 4153, 4055, 4150, 4038, 4106] avg=4096.0000 max=4153 max_rank=3 min=4027 min_rank=1 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16349 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[496, 526, 540, 539, 515, 504, 476, 535, 0, 0, 0, 0, 0, 0, 0, 0, 509, 542, 530, 500, 507, 509, 473, 530, 0, 0, 0, 0, 0, 0, 0, 0, 492, 531, 489, 537, 507, 497, 508, 517, 0, 0, 0, 0, 0, 0, 0, 0, 540, 530, 527, 486, 529, 505, 501, 533, 0, 0, 0, 0, 0, 0, 0, 0, 489, 528, 529, 489, 489, 504, 498, 503, 32, 5, 0, 0, 0, 0, 0, 0, 505, 534, 514, 531, 513, 488, 507, 537, 2, 0, 0, 0, 0, 0, 0, 0, 483, 475, 508, 529, 475, 475, 551, 528, 6, 6, 0, 0, 0, 0, 0, 0, 489, 541, 524, 518, 479, 506, 511, 507, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4131, 4100, 4078, 4151, 4066, 4131, 4036, 4075] avg=4096.0000 max=4151 max_rank=3 min=4036 min_rank=6 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16308 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[521, 543, 508, 487, 504, 503, 504, 531, 0, 0, 0, 0, 0, 0, 0, 0, 500, 514, 541, 538, 553, 517, 522, 467, 0, 0, 0, 0, 0, 0, 0, 0, 512, 515, 517, 513, 527, 487, 477, 489, 0, 0, 0, 0, 0, 0, 0, 0, 483, 527, 494, 524, 498, 508, 507, 489, 0, 0, 0, 0, 0, 0, 0, 0, 497, 524, 540, 515, 488, 525, 495, 524, 29, 4, 0, 0, 0, 0, 0, 0, 500, 504, 525, 529, 473, 502, 522, 516, 2, 0, 0, 0, 0, 0, 0, 0, 541, 478, 494, 498, 472, 544, 495, 561, 6, 5, 0, 0, 0, 0, 0, 0, 513, 521, 557, 512, 523, 514, 507, 493, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4152, 4037, 4030, 4141, 4073, 4094, 4140] avg=4096.0000 max=4152 max_rank=1 min=4030 min_rank=3 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16448 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[500, 531, 498, 494, 478, 493, 496, 503, 0, 0, 0, 0, 0, 0, 0, 0, 537, 523, 503, 524, 522, 529, 539, 549, 0, 0, 0, 0, 0, 0, 0, 0, 470, 523, 496, 516, 527, 495, 526, 465, 0, 0, 0, 0, 0, 0, 0, 0, 499, 517, 486, 523, 532, 511, 487, 525, 0, 0, 0, 0, 0, 0, 0, 0, 562, 479, 525, 507, 484, 491, 511, 484, 29, 5, 0, 0, 0, 0, 0, 0, 541, 528, 515, 503, 475, 491, 504, 490, 2, 0, 0, 0, 0, 0, 0, 0, 561, 510, 549, 525, 497, 498, 510, 515, 6, 6, 0, 0, 0, 0, 0, 0, 506, 543, 533, 509, 530, 518, 502, 507, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3993, 4226, 4018, 4080, 4077, 4049, 4177, 4148] avg=4096.0000 max=4226 max_rank=1 min=3993 min_rank=0 imbalance_ratio=1.031738 max_load_reduction=28542 max_load_reduction_pct=0.871033 moved_to_echo=16451 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 3968, 4096, 4224, 4096, 4224, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 3968, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 3968, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[545, 521, 513, 505, 503, 481, 496, 525, 512, 509, 535, 482, 529, 509, 498, 514, 498, 514, 539, 494, 513, 536, 524, 538, 534, 505, 556, 535, 501, 517, 512, 504, 543, 483, 509, 487, 503, 514, 513, 501, 508, 515, 466, 494, 520, 527, 500, 499, 529, 519, 509, 545, 488, 504, 503, 479, 547, 522, 479, 501, 511, 539, 501, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[508, 489, 541, 533, 526, 522, 520, 519, 547, 538, 498, 519, 499, 498, 465, 520, 533, 493, 479, 520, 525, 525, 499, 512, 495, 500, 532, 535, 547, 507, 527, 546, 499, 504, 485, 534, 488, 486, 483, 504, 519, 539, 551, 501, 509, 476, 548, 504, 508, 507, 479, 533, 504, 507, 546, 522, 511, 534, 491, 503, 503, 488, 475, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[525, 487, 508, 555, 528, 489, 521, 535, 511, 521, 463, 484, 522, 496, 596, 526, 497, 522, 529, 520, 542, 533, 506, 482, 500, 517, 479, 465, 516, 479, 498, 501, 521, 496, 490, 488, 505, 508, 519, 544, 511, 529, 524, 528, 551, 530, 489, 512, 522, 517, 492, 538, 524, 513, 502, 504, 497, 478, 485, 505, 578, 530, 486, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[536, 517, 547, 542, 529, 488, 487, 561, 498, 497, 477, 495, 509, 499, 495, 480, 472, 553, 530, 520, 520, 504, 530, 491, 473, 501, 510, 519, 505, 512, 496, 528, 483, 562, 476, 533, 549, 523, 466, 513, 495, 548, 520, 497, 517, 529, 545, 516, 471, 511, 505, 533, 498, 488, 498, 531, 511, 533, 497, 528, 503, 522, 529, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[534, 539, 512, 491, 529, 516, 533, 534, 545, 503, 501, 528, 509, 475, 492, 533, 467, 527, 545, 514, 509, 476, 519, 529, 514, 524, 513, 499, 526, 512, 533, 519, 536, 452, 544, 491, 482, 510, 509, 479, 505, 502, 487, 528, 528, 499, 503, 523, 504, 507, 477, 532, 516, 531, 539, 493, 509, 539, 506, 484, 533, 515, 503, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[517, 529, 511, 505, 514, 509, 543, 490, 552, 536, 510, 488, 503, 476, 496, 527, 519, 514, 495, 520, 495, 497, 527, 495, 495, 533, 529, 500, 490, 494, 518, 514, 502, 506, 531, 510, 531, 512, 509, 496, 512, 499, 522, 501, 513, 544, 489, 527, 516, 504, 525, 479, 506, 543, 501, 534, 526, 510, 555, 489, 501, 515, 503, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[534, 507, 540, 496, 516, 479, 532, 504, 516, 463, 503, 514, 511, 497, 536, 488, 533, 526, 510, 495, 478, 483, 506, 541, 510, 500, 498, 481, 523, 545, 519, 534, 488, 508, 538, 524, 513, 511, 521, 511, 521, 512, 523, 537, 506, 507, 543, 497, 479, 518, 499, 518, 492, 532, 549, 492, 507, 540, 536, 536, 553, 479, 501, 459] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[533, 509, 506, 505, 517, 497, 508, 507, 555, 508, 518, 521, 494, 528, 518, 511, 488, 509, 499, 483, 531, 515, 499, 490, 498, 482, 531, 480, 506, 527, 511, 514, 512, 512, 517, 492, 532, 500, 506, 534, 536, 523, 489, 508, 527, 514, 512, 509, 515, 510, 478, 532, 503, 502, 550, 513, 510, 486, 553, 503, 516, 497, 536, 543] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[469, 489, 541, 533, 526, 522, 520, 519, 0, 0, 0, 0, 0, 0, 0, 0, 547, 538, 498, 519, 499, 498, 465, 520, 11, 9, 0, 0, 0, 0, 0, 0, 533, 493, 479, 520, 525, 525, 499, 512, 2, 2, 0, 0, 0, 0, 0, 0, 495, 500, 532, 535, 547, 507, 527, 546, 5, 0, 0, 0, 0, 0, 0, 0, 499, 504, 485, 534, 488, 486, 483, 504, 28, 0, 0, 0, 0, 0, 0, 0, 519, 539, 551, 501, 487, 476, 548, 504, 0, 0, 0, 0, 0, 0, 0, 0, 508, 507, 479, 533, 504, 507, 546, 522, 6, 0, 0, 0, 0, 0, 0, 0, 511, 534, 491, 503, 501, 488, 475, 510, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4104, 4090, 4194, 4011, 4125, 4112, 4013] avg=4096.0000 max=4194 max_rank=3 min=4011 min_rank=4 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16261 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[494, 521, 513, 505, 503, 481, 496, 525, 0, 0, 0, 0, 0, 0, 0, 0, 512, 509, 535, 482, 529, 509, 498, 514, 16, 14, 0, 0, 0, 0, 0, 0, 498, 514, 539, 494, 513, 536, 524, 538, 8, 5, 0, 0, 0, 0, 0, 0, 534, 505, 556, 535, 501, 517, 512, 504, 9, 0, 0, 0, 0, 0, 0, 0, 543, 483, 509, 487, 503, 514, 513, 501, 35, 0, 0, 0, 0, 0, 0, 0, 508, 515, 466, 494, 481, 527, 500, 499, 0, 0, 0, 0, 0, 0, 0, 0, 529, 519, 509, 545, 488, 504, 503, 479, 8, 0, 0, 0, 0, 0, 0, 0, 547, 522, 479, 501, 506, 539, 501, 513, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4038, 4118, 4169, 4173, 4088, 3990, 4084, 4108] avg=4096.0000 max=4173 max_rank=3 min=3990 min_rank=5 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16270 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[494, 539, 512, 491, 529, 516, 533, 534, 0, 0, 0, 0, 0, 0, 0, 0, 545, 503, 501, 528, 509, 475, 492, 533, 11, 10, 0, 0, 0, 0, 0, 0, 467, 527, 545, 514, 509, 476, 519, 529, 2, 2, 0, 0, 0, 0, 0, 0, 514, 524, 513, 499, 526, 512, 533, 519, 5, 0, 0, 0, 0, 0, 0, 0, 536, 452, 544, 491, 482, 510, 509, 479, 29, 0, 0, 0, 0, 0, 0, 0, 505, 502, 487, 528, 505, 499, 503, 523, 0, 0, 0, 0, 0, 0, 0, 0, 504, 507, 477, 532, 516, 531, 539, 493, 6, 0, 0, 0, 0, 0, 0, 0, 509, 539, 506, 484, 531, 515, 503, 502, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4148, 4107, 4090, 4145, 4032, 4052, 4105, 4089] avg=4096.0000 max=4148 max_rank=0 min=4032 min_rank=4 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16278 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[495, 517, 547, 542, 529, 488, 487, 561, 0, 0, 0, 0, 0, 0, 0, 0, 498, 497, 477, 495, 509, 499, 495, 480, 11, 9, 0, 0, 0, 0, 0, 0, 472, 553, 530, 520, 520, 504, 530, 491, 2, 2, 0, 0, 0, 0, 0, 0, 473, 501, 510, 519, 505, 512, 496, 528, 5, 0, 0, 0, 0, 0, 0, 0, 483, 562, 476, 533, 549, 523, 466, 513, 30, 0, 0, 0, 0, 0, 0, 0, 495, 548, 520, 497, 495, 529, 545, 516, 0, 0, 0, 0, 0, 0, 0, 0, 471, 511, 505, 533, 498, 488, 498, 531, 6, 0, 0, 0, 0, 0, 0, 0, 511, 533, 497, 528, 501, 522, 529, 517, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4166, 3970, 4124, 4049, 4135, 4145, 4041, 4138] avg=4096.0000 max=4166 max_rank=0 min=3970 min_rank=1 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16459 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[485, 487, 508, 555, 528, 489, 521, 535, 0, 0, 0, 0, 0, 0, 0, 0, 511, 521, 463, 484, 522, 496, 596, 526, 11, 10, 0, 0, 0, 0, 0, 0, 497, 522, 529, 520, 542, 533, 506, 482, 2, 2, 0, 0, 0, 0, 0, 0, 500, 517, 479, 465, 516, 479, 498, 501, 5, 0, 0, 0, 0, 0, 0, 0, 521, 496, 490, 488, 505, 508, 519, 544, 29, 0, 0, 0, 0, 0, 0, 0, 511, 529, 524, 528, 528, 530, 489, 512, 0, 0, 0, 0, 0, 0, 0, 0, 522, 517, 492, 538, 524, 513, 502, 504, 6, 0, 0, 0, 0, 0, 0, 0, 497, 478, 485, 505, 576, 530, 486, 499, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4140, 4135, 3960, 4100, 4151, 4118, 4056] avg=4096.0000 max=4151 max_rank=5 min=3960 min_rank=3 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16425 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[478, 529, 511, 505, 514, 509, 543, 490, 0, 0, 0, 0, 0, 0, 0, 0, 552, 536, 510, 488, 503, 476, 496, 527, 11, 9, 0, 0, 0, 0, 0, 0, 519, 514, 495, 520, 495, 497, 527, 495, 2, 2, 0, 0, 0, 0, 0, 0, 495, 533, 529, 500, 490, 494, 518, 514, 5, 0, 0, 0, 0, 0, 0, 0, 502, 506, 531, 510, 531, 512, 509, 496, 28, 0, 0, 0, 0, 0, 0, 0, 512, 499, 522, 501, 491, 544, 489, 527, 0, 0, 0, 0, 0, 0, 0, 0, 516, 504, 525, 479, 506, 543, 501, 534, 6, 0, 0, 0, 0, 0, 0, 0, 526, 510, 555, 489, 499, 515, 503, 516, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4108, 4066, 4078, 4125, 4085, 4114, 4113] avg=4096.0000 max=4125 max_rank=4 min=4066 min_rank=2 imbalance_ratio=1.007080 max_load_reduction=28643 max_load_reduction_pct=0.874115 moved_to_echo=16437 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[493, 509, 506, 505, 517, 497, 508, 507, 0, 0, 0, 0, 0, 0, 0, 0, 555, 508, 518, 521, 494, 528, 518, 511, 11, 9, 0, 0, 0, 0, 0, 0, 488, 509, 499, 483, 531, 515, 499, 490, 2, 2, 0, 0, 0, 0, 0, 0, 498, 482, 531, 480, 506, 527, 511, 514, 5, 0, 0, 0, 0, 0, 0, 0, 512, 512, 517, 492, 532, 500, 506, 534, 29, 0, 0, 0, 0, 0, 0, 0, 536, 523, 489, 508, 505, 514, 512, 509, 0, 0, 0, 0, 0, 0, 0, 0, 515, 510, 478, 532, 503, 502, 550, 513, 6, 0, 0, 0, 0, 0, 0, 0, 510, 486, 553, 503, 514, 497, 536, 543, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4042, 4173, 4018, 4054, 4134, 4096, 4109, 4142] avg=4096.0000 max=4173 max_rank=1 min=4018 min_rank=2 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16481 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[494, 507, 540, 496, 516, 479, 532, 504, 0, 0, 0, 0, 0, 0, 0, 0, 516, 463, 503, 514, 511, 497, 536, 488, 11, 9, 0, 0, 0, 0, 0, 0, 533, 526, 510, 495, 478, 483, 506, 541, 2, 2, 0, 0, 0, 0, 0, 0, 510, 500, 498, 481, 523, 545, 519, 534, 5, 0, 0, 0, 0, 0, 0, 0, 488, 508, 538, 524, 513, 511, 521, 511, 29, 0, 0, 0, 0, 0, 0, 0, 521, 512, 523, 537, 484, 507, 543, 497, 0, 0, 0, 0, 0, 0, 0, 0, 479, 518, 499, 518, 492, 532, 549, 492, 6, 0, 0, 0, 0, 0, 0, 0, 507, 540, 536, 536, 551, 479, 501, 459, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4048, 4076, 4115, 4143, 4124, 4085, 4109] avg=4096.0000 max=4143 max_rank=4 min=4048 min_rank=1 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16461 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 3968, 4224, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4352, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3968, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[529, 574, 495, 510, 531, 526, 523, 510, 551, 524, 508, 504, 527, 507, 504, 536, 515, 521, 513, 521, 490, 488, 503, 505, 523, 522, 503, 504, 538, 495, 485, 499, 483, 512, 488, 503, 512, 542, 529, 521, 546, 489, 521, 502, 491, 508, 523, 514, 503, 530, 522, 520, 486, 502, 477, 512, 533, 493, 514, 508, 518, 521, 447, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[526, 527, 530, 467, 497, 525, 523, 532, 504, 519, 519, 477, 504, 535, 485, 517, 500, 505, 531, 494, 533, 491, 498, 503, 520, 556, 511, 519, 520, 520, 571, 514, 462, 497, 538, 523, 525, 505, 510, 498, 453, 529, 522, 472, 524, 494, 530, 507, 512, 526, 531, 475, 560, 514, 501, 500, 519, 543, 517, 534, 488, 524, 491, 471] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[473, 548, 494, 496, 542, 514, 527, 498, 502, 547, 522, 541, 492, 541, 520, 498, 519, 531, 542, 494, 565, 500, 542, 468, 523, 509, 510, 533, 520, 507, 481, 526, 504, 542, 522, 499, 468, 490, 507, 562, 550, 461, 468, 510, 482, 515, 549, 500, 474, 497, 471, 541, 515, 575, 523, 466, 491, 527, 514, 493, 499, 519, 497, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[528, 506, 495, 526, 498, 521, 519, 492, 512, 534, 504, 508, 530, 503, 502, 528, 515, 508, 487, 553, 537, 520, 505, 497, 494, 534, 520, 516, 508, 529, 518, 595, 555, 508, 518, 490, 521, 482, 526, 501, 515, 513, 531, 529, 501, 521, 503, 484, 484, 501, 514, 518, 506, 502, 535, 498, 518, 495, 458, 508, 492, 484, 497, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[517, 502, 481, 490, 493, 498, 483, 538, 538, 504, 486, 536, 535, 542, 499, 496, 496, 531, 543, 517, 479, 522, 519, 502, 517, 530, 494, 542, 511, 521, 508, 572, 500, 511, 468, 554, 485, 496, 532, 480, 521, 539, 516, 511, 560, 524, 498, 528, 514, 481, 530, 533, 523, 475, 529, 496, 478, 482, 546, 461, 515, 492, 538, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[511, 477, 492, 484, 517, 534, 531, 506, 521, 504, 515, 517, 519, 494, 471, 510, 513, 524, 513, 500, 520, 554, 525, 485, 503, 534, 502, 545, 475, 529, 499, 544, 501, 502, 518, 484, 493, 493, 525, 507, 514, 515, 544, 490, 497, 494, 541, 491, 538, 520, 544, 472, 491, 555, 499, 510, 549, 522, 509, 508, 540, 510, 509, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[503, 522, 507, 564, 504, 493, 523, 474, 504, 489, 478, 551, 502, 514, 501, 492, 527, 519, 493, 517, 502, 487, 505, 513, 505, 519, 513, 506, 520, 518, 518, 547, 502, 539, 503, 519, 502, 548, 505, 520, 495, 484, 517, 521, 519, 496, 529, 515, 521, 499, 526, 470, 501, 489, 538, 540, 521, 533, 515, 502, 513, 545, 517, 494] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[545, 517, 550, 503, 475, 521, 558, 473, 505, 486, 518, 527, 519, 524, 503, 502, 494, 513, 510, 491, 483, 523, 498, 497, 515, 529, 533, 484, 498, 501, 528, 515, 503, 491, 494, 518, 505, 528, 517, 512, 484, 535, 526, 548, 514, 486, 507, 515, 509, 556, 510, 491, 512, 537, 491, 500, 523, 543, 568, 484, 506, 511, 510, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[473, 548, 494, 496, 542, 514, 525, 498, 0, 0, 0, 0, 0, 0, 0, 0, 502, 547, 522, 537, 492, 541, 520, 498, 0, 0, 0, 0, 0, 0, 0, 0, 519, 531, 542, 494, 565, 500, 542, 468, 4, 2, 0, 0, 0, 0, 0, 0, 523, 509, 510, 533, 520, 507, 481, 476, 0, 0, 0, 0, 0, 0, 0, 0, 504, 542, 522, 499, 468, 490, 507, 562, 20, 0, 0, 0, 0, 0, 0, 0, 550, 461, 468, 510, 482, 515, 549, 500, 2, 2, 0, 0, 0, 0, 0, 0, 474, 497, 471, 541, 515, 575, 523, 466, 9, 0, 0, 0, 0, 0, 0, 0, 491, 527, 514, 493, 499, 519, 497, 512, 17, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4159, 4167, 4059, 4114, 4039, 4071, 4069] avg=4096.0000 max=4167 max_rank=2 min=4039 min_rank=5 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16293 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[517, 502, 481, 490, 493, 498, 481, 538, 0, 0, 0, 0, 0, 0, 0, 0, 538, 504, 486, 532, 535, 542, 499, 496, 0, 0, 0, 0, 0, 0, 0, 0, 496, 531, 543, 517, 479, 522, 519, 502, 4, 2, 0, 0, 0, 0, 0, 0, 517, 530, 494, 542, 511, 521, 508, 517, 0, 0, 0, 0, 0, 0, 0, 0, 500, 511, 468, 554, 485, 496, 532, 480, 22, 0, 0, 0, 0, 0, 0, 0, 521, 539, 516, 511, 560, 524, 498, 528, 2, 2, 0, 0, 0, 0, 0, 0, 514, 481, 530, 533, 523, 475, 529, 496, 10, 0, 0, 0, 0, 0, 0, 0, 478, 482, 546, 461, 515, 492, 538, 510, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4000, 4132, 4115, 4140, 4048, 4201, 4091, 4041] avg=4096.0000 max=4201 max_rank=5 min=4000 min_rank=0 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[529, 574, 495, 510, 531, 526, 521, 510, 0, 0, 0, 0, 0, 0, 0, 0, 551, 524, 508, 500, 527, 507, 504, 536, 0, 0, 0, 0, 0, 0, 0, 0, 515, 521, 513, 521, 490, 488, 503, 505, 4, 2, 0, 0, 0, 0, 0, 0, 523, 522, 503, 504, 538, 495, 485, 452, 0, 0, 0, 0, 0, 0, 0, 0, 483, 512, 488, 503, 512, 542, 529, 521, 19, 0, 0, 0, 0, 0, 0, 0, 546, 489, 521, 502, 491, 508, 523, 514, 2, 2, 0, 0, 0, 0, 0, 0, 503, 530, 522, 520, 486, 502, 477, 512, 8, 0, 0, 0, 0, 0, 0, 0, 533, 493, 514, 508, 518, 521, 447, 514, 16, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4196, 4157, 4062, 4022, 4109, 4098, 4060, 4064] avg=4096.0000 max=4196 max_rank=0 min=4022 min_rank=3 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16331 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[503, 522, 507, 564, 504, 493, 521, 474, 0, 0, 0, 0, 0, 0, 0, 0, 504, 489, 478, 547, 502, 514, 501, 492, 0, 0, 0, 0, 0, 0, 0, 0, 527, 519, 493, 517, 502, 487, 505, 513, 4, 2, 0, 0, 0, 0, 0, 0, 505, 519, 513, 506, 520, 518, 518, 495, 0, 0, 0, 0, 0, 0, 0, 0, 502, 539, 503, 519, 502, 548, 505, 520, 21, 0, 0, 0, 0, 0, 0, 0, 495, 484, 517, 521, 519, 496, 529, 515, 2, 2, 0, 0, 0, 0, 0, 0, 521, 499, 526, 470, 501, 489, 538, 540, 9, 0, 0, 0, 0, 0, 0, 0, 521, 533, 515, 502, 513, 545, 517, 494, 18, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4027, 4069, 4094, 4159, 4080, 4093, 4158] avg=4096.0000 max=4159 max_rank=4 min=4027 min_rank=1 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16490 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[528, 506, 495, 526, 498, 521, 513, 492, 0, 0, 0, 0, 0, 0, 0, 0, 512, 534, 504, 496, 530, 503, 502, 528, 0, 0, 0, 0, 0, 0, 0, 0, 515, 508, 487, 553, 537, 520, 505, 497, 9, 8, 0, 0, 0, 0, 0, 0, 494, 534, 520, 516, 508, 529, 518, 521, 0, 0, 0, 0, 0, 0, 0, 0, 555, 508, 518, 490, 521, 482, 526, 501, 27, 0, 0, 0, 0, 0, 0, 0, 515, 513, 531, 529, 501, 521, 503, 484, 6, 4, 0, 0, 0, 0, 0, 0, 484, 501, 514, 518, 506, 502, 535, 498, 14, 0, 0, 0, 0, 0, 0, 0, 518, 495, 458, 508, 492, 484, 497, 518, 24, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4109, 4139, 4140, 4128, 4107, 4072, 3994] avg=4096.0000 max=4140 max_rank=3 min=3994 min_rank=7 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16301 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[526, 527, 530, 467, 497, 525, 521, 532, 0, 0, 0, 0, 0, 0, 0, 0, 504, 519, 519, 474, 504, 535, 485, 517, 0, 0, 0, 0, 0, 0, 0, 0, 500, 505, 531, 494, 533, 491, 498, 503, 4, 2, 0, 0, 0, 0, 0, 0, 520, 556, 511, 519, 520, 520, 571, 464, 0, 0, 0, 0, 0, 0, 0, 0, 462, 497, 538, 523, 525, 505, 510, 498, 20, 0, 0, 0, 0, 0, 0, 0, 453, 529, 522, 472, 524, 494, 530, 507, 2, 1, 0, 0, 0, 0, 0, 0, 512, 526, 531, 475, 560, 514, 501, 500, 9, 0, 0, 0, 0, 0, 0, 0, 519, 543, 517, 534, 488, 524, 491, 471, 17, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4057, 4061, 4181, 4078, 4034, 4128, 4104] avg=4096.0000 max=4181 max_rank=3 min=4034 min_rank=5 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16344 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[511, 477, 492, 484, 517, 534, 529, 506, 0, 0, 0, 0, 0, 0, 0, 0, 521, 504, 515, 513, 519, 494, 471, 510, 0, 0, 0, 0, 0, 0, 0, 0, 513, 524, 513, 500, 520, 554, 525, 485, 4, 2, 0, 0, 0, 0, 0, 0, 503, 534, 502, 545, 475, 529, 499, 492, 0, 0, 0, 0, 0, 0, 0, 0, 501, 502, 518, 484, 493, 493, 525, 507, 21, 0, 0, 0, 0, 0, 0, 0, 514, 515, 544, 490, 497, 494, 541, 491, 2, 2, 0, 0, 0, 0, 0, 0, 538, 520, 544, 472, 491, 555, 499, 510, 9, 0, 0, 0, 0, 0, 0, 0, 549, 522, 509, 508, 540, 510, 509, 515, 18, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4050, 4047, 4140, 4079, 4044, 4090, 4138, 4180] avg=4096.0000 max=4180 max_rank=7 min=4044 min_rank=4 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16452 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[545, 517, 550, 503, 475, 521, 556, 473, 0, 0, 0, 0, 0, 0, 0, 0, 505, 486, 518, 523, 519, 524, 503, 502, 0, 0, 0, 0, 0, 0, 0, 0, 494, 513, 510, 491, 483, 523, 498, 497, 4, 2, 0, 0, 0, 0, 0, 0, 515, 529, 533, 484, 498, 501, 528, 465, 0, 0, 0, 0, 0, 0, 0, 0, 503, 491, 494, 518, 505, 528, 517, 512, 20, 0, 0, 0, 0, 0, 0, 0, 484, 535, 526, 548, 514, 486, 507, 515, 2, 2, 0, 0, 0, 0, 0, 0, 509, 556, 510, 491, 512, 537, 491, 500, 9, 0, 0, 0, 0, 0, 0, 0, 523, 543, 568, 484, 506, 511, 510, 496, 17, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4140, 4080, 4015, 4053, 4088, 4119, 4115, 4158] avg=4096.0000 max=4158 max_rank=7 min=4015 min_rank=2 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16480 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4352, 4096, 4224, 4096, 4224, 4224, 3968, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[560, 512, 515, 487, 523, 493, 499, 500, 507, 540, 485, 505, 505, 504, 499, 516, 511, 495, 517, 527, 536, 498, 517, 520, 498, 508, 532, 511, 548, 552, 515, 533, 531, 462, 501, 539, 545, 476, 476, 521, 508, 468, 535, 512, 518, 515, 522, 463, 516, 507, 502, 519, 529, 507, 528, 509, 500, 497, 496, 528, 539, 523, 523, 485] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[469, 498, 483, 473, 493, 542, 511, 518, 511, 525, 540, 511, 496, 512, 497, 491, 541, 501, 499, 536, 521, 547, 506, 537, 494, 533, 547, 541, 537, 500, 510, 492, 520, 529, 505, 553, 505, 491, 501, 517, 504, 483, 500, 511, 581, 515, 529, 519, 472, 516, 476, 547, 521, 502, 516, 487, 509, 510, 502, 528, 493, 490, 489, 535] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[504, 481, 505, 503, 497, 489, 536, 505, 516, 558, 525, 529, 515, 511, 523, 511, 533, 519, 503, 526, 538, 525, 546, 497, 535, 543, 470, 499, 537, 500, 505, 549, 498, 503, 543, 496, 524, 500, 455, 531, 495, 520, 515, 523, 513, 494, 487, 493, 518, 504, 479, 532, 520, 540, 544, 496, 491, 482, 488, 504, 519, 530, 493, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[491, 540, 508, 508, 516, 502, 513, 522, 497, 539, 532, 531, 514, 552, 546, 469, 516, 517, 525, 504, 508, 508, 496, 524, 518, 523, 513, 542, 475, 468, 503, 545, 513, 506, 521, 520, 520, 508, 486, 483, 472, 519, 532, 490, 538, 534, 505, 505, 507, 520, 529, 494, 532, 507, 507, 498, 562, 518, 508, 501, 470, 519, 487, 492] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[513, 512, 492, 526, 524, 525, 499, 516, 520, 489, 509, 507, 503, 544, 514, 483, 519, 544, 497, 485, 547, 519, 513, 468, 531, 528, 479, 510, 512, 536, 540, 504, 494, 542, 563, 536, 502, 506, 507, 520, 526, 489, 504, 494, 529, 500, 494, 500, 495, 510, 504, 492, 510, 515, 545, 511, 500, 551, 480, 505, 524, 482, 508, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[471, 558, 539, 538, 515, 509, 541, 508, 512, 541, 495, 487, 491, 544, 544, 477, 492, 494, 485, 502, 505, 499, 516, 511, 532, 523, 486, 486, 497, 556, 500, 523, 498, 515, 524, 482, 504, 499, 531, 499, 531, 509, 512, 528, 533, 492, 531, 529, 522, 548, 502, 466, 528, 488, 514, 486, 528, 529, 511, 509, 569, 496, 490, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[501, 546, 469, 520, 487, 499, 489, 512, 537, 498, 497, 511, 542, 505, 504, 488, 514, 555, 476, 507, 528, 506, 518, 500, 521, 535, 498, 519, 498, 511, 469, 502, 524, 553, 464, 500, 526, 519, 521, 512, 507, 511, 503, 501, 528, 485, 516, 528, 486, 511, 511, 520, 532, 520, 540, 510, 511, 545, 538, 524, 500, 513, 541, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[518, 528, 549, 538, 530, 491, 509, 496, 524, 496, 491, 495, 535, 503, 508, 562, 498, 529, 488, 516, 521, 523, 498, 524, 519, 517, 525, 497, 487, 505, 525, 518, 495, 518, 520, 517, 484, 490, 518, 521, 521, 510, 476, 534, 494, 541, 530, 521, 502, 498, 503, 540, 486, 492, 542, 548, 510, 511, 478, 492, 486, 539, 500, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[560, 512, 515, 487, 523, 493, 499, 500, 10, 2, 0, 0, 0, 0, 0, 0, 507, 528, 485, 505, 505, 504, 499, 516, 0, 0, 0, 0, 0, 0, 0, 0, 511, 495, 517, 527, 522, 498, 517, 520, 0, 0, 0, 0, 0, 0, 0, 0, 498, 485, 532, 511, 548, 552, 515, 533, 0, 0, 0, 0, 0, 0, 0, 0, 531, 462, 501, 539, 545, 476, 476, 521, 6, 4, 0, 0, 0, 0, 0, 0, 508, 468, 535, 512, 518, 515, 522, 463, 5, 0, 0, 0, 0, 0, 0, 0, 516, 507, 502, 519, 529, 507, 528, 509, 1, 0, 0, 0, 0, 0, 0, 0, 500, 497, 496, 528, 539, 523, 523, 485, 21, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4049, 4107, 4174, 4061, 4046, 4118, 4112] avg=4096.0000 max=4174 max_rank=3 min=4046 min_rank=5 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16337 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[504, 481, 505, 503, 497, 489, 536, 505, 10, 2, 0, 0, 0, 0, 0, 0, 516, 546, 525, 529, 515, 511, 523, 511, 0, 0, 0, 0, 0, 0, 0, 0, 533, 519, 503, 526, 524, 525, 546, 497, 0, 0, 0, 0, 0, 0, 0, 0, 535, 519, 470, 499, 537, 500, 505, 549, 0, 0, 0, 0, 0, 0, 0, 0, 498, 503, 543, 496, 524, 500, 455, 531, 6, 4, 0, 0, 0, 0, 0, 0, 495, 520, 515, 523, 513, 494, 487, 493, 5, 0, 0, 0, 0, 0, 0, 0, 518, 504, 479, 532, 520, 540, 544, 496, 1, 0, 0, 0, 0, 0, 0, 0, 491, 482, 488, 504, 519, 530, 493, 505, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4032, 4176, 4173, 4114, 4060, 4045, 4134, 4034] avg=4096.0000 max=4176 max_rank=1 min=4032 min_rank=0 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16273 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[518, 528, 549, 538, 530, 491, 509, 496, 10, 2, 0, 0, 0, 0, 0, 0, 524, 485, 491, 495, 535, 503, 508, 562, 0, 0, 0, 0, 0, 0, 0, 0, 498, 529, 488, 516, 507, 523, 498, 524, 0, 0, 0, 0, 0, 0, 0, 0, 519, 494, 525, 497, 487, 505, 525, 518, 0, 0, 0, 0, 0, 0, 0, 0, 495, 518, 520, 517, 484, 490, 518, 521, 5, 4, 0, 0, 0, 0, 0, 0, 521, 510, 476, 534, 494, 541, 530, 521, 5, 0, 0, 0, 0, 0, 0, 0, 502, 498, 503, 540, 486, 492, 542, 548, 1, 0, 0, 0, 0, 0, 0, 0, 510, 511, 478, 492, 486, 539, 500, 488, 21, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4171, 4103, 4083, 4070, 4072, 4132, 4112, 4025] avg=4096.0000 max=4171 max_rank=0 min=4025 min_rank=7 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16341 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[469, 498, 483, 473, 493, 542, 511, 518, 14, 5, 0, 0, 0, 0, 0, 0, 511, 506, 540, 511, 496, 512, 497, 491, 0, 0, 0, 0, 0, 0, 0, 0, 541, 501, 499, 536, 497, 547, 506, 537, 0, 0, 0, 0, 0, 0, 0, 0, 494, 502, 547, 541, 537, 500, 510, 492, 0, 0, 0, 0, 0, 0, 0, 0, 520, 529, 505, 553, 505, 491, 501, 517, 8, 10, 0, 0, 0, 0, 0, 0, 504, 483, 500, 511, 581, 515, 529, 519, 8, 0, 0, 0, 0, 0, 0, 0, 472, 516, 476, 547, 521, 502, 516, 487, 3, 0, 0, 0, 0, 0, 0, 0, 509, 510, 502, 528, 493, 490, 489, 535, 26, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4006, 4064, 4164, 4123, 4139, 4150, 4040, 4082] avg=4096.0000 max=4164 max_rank=2 min=4006 min_rank=0 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16411 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[491, 540, 508, 508, 516, 502, 513, 522, 10, 2, 0, 0, 0, 0, 0, 0, 497, 527, 532, 531, 514, 552, 546, 469, 0, 0, 0, 0, 0, 0, 0, 0, 516, 517, 525, 504, 494, 508, 496, 524, 0, 0, 0, 0, 0, 0, 0, 0, 518, 499, 513, 542, 475, 468, 503, 545, 0, 0, 0, 0, 0, 0, 0, 0, 513, 506, 521, 520, 520, 508, 486, 483, 6, 4, 0, 0, 0, 0, 0, 0, 472, 519, 532, 490, 538, 534, 505, 505, 5, 0, 0, 0, 0, 0, 0, 0, 507, 520, 529, 494, 532, 507, 507, 498, 1, 0, 0, 0, 0, 0, 0, 0, 562, 518, 508, 501, 470, 519, 487, 492, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4168, 4084, 4063, 4067, 4100, 4095, 4079] avg=4096.0000 max=4168 max_rank=1 min=4063 min_rank=3 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16341 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[513, 512, 492, 526, 524, 525, 499, 516, 11, 2, 0, 0, 0, 0, 0, 0, 520, 478, 509, 507, 503, 544, 514, 483, 0, 0, 0, 0, 0, 0, 0, 0, 519, 544, 497, 485, 532, 519, 513, 468, 0, 0, 0, 0, 0, 0, 0, 0, 531, 504, 479, 510, 512, 536, 540, 504, 0, 0, 0, 0, 0, 0, 0, 0, 494, 542, 563, 536, 502, 506, 507, 520, 5, 4, 0, 0, 0, 0, 0, 0, 526, 489, 504, 494, 529, 500, 494, 500, 5, 0, 0, 0, 0, 0, 0, 0, 495, 510, 504, 492, 510, 515, 545, 511, 1, 0, 0, 0, 0, 0, 0, 0, 500, 551, 480, 505, 524, 482, 508, 522, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4058, 4077, 4116, 4179, 4041, 4083, 4094] avg=4096.0000 max=4179 max_rank=4 min=4041 min_rank=5 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[471, 558, 539, 538, 515, 509, 541, 508, 10, 2, 0, 0, 0, 0, 0, 0, 512, 529, 495, 487, 491, 544, 544, 477, 0, 0, 0, 0, 0, 0, 0, 0, 492, 494, 485, 502, 491, 499, 516, 511, 0, 0, 0, 0, 0, 0, 0, 0, 532, 499, 486, 486, 497, 556, 500, 523, 0, 0, 0, 0, 0, 0, 0, 0, 498, 515, 524, 482, 504, 499, 531, 499, 6, 4, 0, 0, 0, 0, 0, 0, 531, 509, 512, 528, 533, 492, 531, 529, 5, 0, 0, 0, 0, 0, 0, 0, 522, 548, 502, 466, 528, 488, 514, 486, 1, 0, 0, 0, 0, 0, 0, 0, 528, 529, 511, 509, 569, 496, 490, 488, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4191, 4079, 3990, 4079, 4062, 4170, 4055, 4142] avg=4096.0000 max=4191 max_rank=0 min=3990 min_rank=2 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[501, 546, 469, 520, 487, 499, 489, 512, 10, 2, 0, 0, 0, 0, 0, 0, 537, 487, 497, 511, 542, 505, 504, 488, 0, 0, 0, 0, 0, 0, 0, 0, 514, 555, 476, 507, 514, 506, 518, 500, 0, 0, 0, 0, 0, 0, 0, 0, 521, 511, 498, 519, 498, 511, 469, 502, 0, 0, 0, 0, 0, 0, 0, 0, 524, 553, 464, 500, 526, 519, 521, 512, 5, 4, 0, 0, 0, 0, 0, 0, 507, 511, 503, 501, 528, 485, 516, 528, 5, 0, 0, 0, 0, 0, 0, 0, 486, 511, 511, 520, 532, 520, 540, 510, 1, 0, 0, 0, 0, 0, 0, 0, 511, 545, 538, 524, 500, 513, 541, 506, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4035, 4071, 4090, 4029, 4128, 4084, 4131, 4200] avg=4096.0000 max=4200 max_rank=7 min=4029 min_rank=3 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16543 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4352, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4352, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[499, 519, 482, 501, 506, 501, 551, 508, 527, 504, 505, 536, 538, 511, 525, 513, 510, 512, 519, 527, 527, 496, 531, 506, 499, 509, 509, 507, 529, 491, 521, 489, 505, 508, 471, 539, 506, 499, 545, 505, 541, 534, 514, 552, 486, 498, 504, 491, 511, 535, 510, 468, 509, 550, 498, 498, 494, 499, 480, 517, 525, 547, 520, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[514, 506, 542, 529, 515, 510, 517, 509, 499, 507, 466, 512, 494, 498, 512, 505, 518, 510, 483, 525, 534, 524, 523, 512, 525, 520, 534, 509, 543, 508, 523, 513, 469, 515, 519, 461, 529, 496, 499, 526, 511, 560, 505, 500, 523, 523, 523, 510, 527, 513, 520, 494, 494, 516, 479, 533, 516, 516, 489, 494, 518, 518, 497, 536] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[494, 511, 480, 504, 490, 519, 543, 551, 523, 525, 522, 541, 515, 528, 491, 522, 456, 473, 497, 523, 530, 490, 494, 465, 504, 541, 496, 539, 506, 464, 508, 514, 545, 506, 505, 515, 484, 502, 499, 530, 557, 530, 514, 518, 532, 530, 550, 526, 529, 501, 505, 499, 515, 531, 534, 499, 537, 514, 501, 504, 456, 541, 489, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[547, 495, 512, 528, 487, 500, 492, 535, 528, 508, 487, 472, 516, 551, 528, 514, 514, 506, 460, 521, 537, 499, 504, 559, 485, 533, 492, 511, 509, 496, 484, 525, 521, 510, 524, 497, 541, 528, 490, 509, 511, 504, 519, 518, 561, 480, 510, 491, 484, 538, 528, 525, 522, 513, 538, 513, 474, 520, 450, 563, 517, 478, 526, 530] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[512, 526, 546, 519, 524, 527, 522, 532, 510, 504, 506, 488, 516, 511, 472, 468, 488, 522, 527, 537, 536, 513, 486, 526, 522, 531, 533, 513, 511, 518, 542, 532, 559, 479, 464, 551, 497, 498, 528, 492, 473, 475, 497, 503, 487, 528, 513, 523, 492, 498, 498, 547, 507, 476, 486, 514, 530, 513, 482, 534, 511, 505, 548, 540] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[500, 483, 533, 519, 520, 534, 510, 525, 482, 561, 552, 506, 540, 519, 495, 512, 515, 522, 544, 513, 505, 503, 483, 538, 572, 523, 531, 511, 510, 502, 520, 524, 509, 512, 504, 522, 453, 538, 499, 511, 494, 498, 518, 496, 516, 536, 508, 510, 499, 489, 480, 517, 511, 513, 534, 482, 492, 499, 498, 531, 514, 514, 505, 459] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[517, 523, 498, 513, 496, 503, 528, 534, 517, 519, 508, 528, 511, 516, 515, 490, 510, 529, 522, 549, 503, 514, 522, 534, 545, 503, 496, 512, 559, 513, 518, 516, 499, 502, 498, 479, 492, 522, 517, 525, 525, 521, 490, 507, 500, 516, 524, 516, 493, 533, 514, 525, 530, 501, 499, 483, 507, 464, 508, 481, 507, 517, 503, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[472, 498, 526, 531, 515, 505, 567, 503, 503, 524, 508, 515, 525, 484, 509, 528, 530, 554, 496, 513, 456, 539, 478, 514, 542, 505, 481, 506, 530, 503, 504, 535, 575, 539, 533, 506, 506, 503, 560, 511, 523, 479, 508, 468, 512, 529, 527, 476, 514, 510, 500, 515, 497, 506, 541, 540, 500, 518, 499, 515, 481, 500, 483, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[499, 519, 482, 501, 506, 501, 527, 508, 0, 0, 0, 0, 0, 0, 0, 0, 527, 504, 505, 536, 538, 511, 525, 513, 0, 0, 0, 0, 0, 0, 0, 0, 510, 512, 519, 523, 527, 496, 531, 506, 0, 0, 0, 0, 0, 0, 0, 0, 499, 509, 509, 507, 500, 491, 521, 489, 0, 0, 0, 0, 0, 0, 0, 0, 505, 508, 471, 539, 506, 499, 545, 505, 6, 4, 0, 0, 0, 0, 0, 0, 541, 534, 514, 552, 486, 498, 498, 491, 0, 0, 0, 0, 0, 0, 0, 0, 511, 535, 510, 468, 509, 550, 498, 498, 12, 0, 0, 0, 0, 0, 0, 0, 494, 499, 480, 517, 525, 547, 520, 501, 29, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4043, 4159, 4124, 4025, 4088, 4114, 4091, 4124] avg=4096.0000 max=4159 max_rank=1 min=4025 min_rank=3 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16417 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[514, 506, 542, 529, 515, 510, 495, 509, 0, 0, 0, 0, 0, 0, 0, 0, 499, 507, 466, 512, 494, 498, 512, 505, 0, 0, 0, 0, 0, 0, 0, 0, 518, 510, 483, 521, 534, 524, 523, 512, 0, 0, 0, 0, 0, 0, 0, 0, 525, 520, 534, 509, 514, 508, 523, 513, 0, 0, 0, 0, 0, 0, 0, 0, 469, 515, 519, 461, 529, 496, 499, 526, 6, 4, 0, 0, 0, 0, 0, 0, 511, 560, 505, 500, 523, 523, 517, 510, 0, 0, 0, 0, 0, 0, 0, 0, 527, 513, 520, 494, 494, 516, 479, 533, 11, 0, 0, 0, 0, 0, 0, 0, 516, 516, 489, 494, 518, 518, 497, 536, 29, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 3993, 4125, 4146, 4024, 4149, 4087, 4124] avg=4096.0000 max=4149 max_rank=5 min=3993 min_rank=1 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[512, 526, 546, 519, 524, 527, 500, 532, 0, 0, 0, 0, 0, 0, 0, 0, 510, 504, 506, 488, 516, 511, 472, 468, 0, 0, 0, 0, 0, 0, 0, 0, 488, 522, 527, 533, 536, 513, 486, 526, 0, 0, 0, 0, 0, 0, 0, 0, 522, 531, 533, 513, 483, 518, 542, 532, 0, 0, 0, 0, 0, 0, 0, 0, 559, 479, 464, 551, 497, 498, 528, 492, 6, 4, 0, 0, 0, 0, 0, 0, 473, 475, 497, 503, 487, 528, 507, 523, 0, 0, 0, 0, 0, 0, 0, 0, 492, 498, 498, 547, 507, 476, 486, 514, 11, 0, 0, 0, 0, 0, 0, 0, 530, 513, 482, 534, 511, 505, 548, 540, 28, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4186, 3975, 4131, 4174, 4078, 3993, 4029, 4202] avg=4096.0000 max=4202 max_rank=7 min=3975 min_rank=1 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16302 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[494, 511, 480, 504, 490, 519, 512, 551, 0, 0, 0, 0, 0, 0, 0, 0, 523, 525, 522, 541, 515, 528, 491, 522, 3, 0, 0, 0, 0, 0, 0, 0, 456, 473, 497, 513, 530, 490, 494, 465, 0, 0, 0, 0, 0, 0, 0, 0, 504, 541, 496, 539, 476, 464, 508, 514, 0, 0, 0, 0, 0, 0, 0, 0, 545, 506, 505, 515, 484, 502, 499, 530, 10, 7, 0, 0, 0, 0, 0, 0, 557, 530, 514, 518, 532, 530, 538, 526, 0, 0, 0, 0, 0, 0, 0, 0, 529, 501, 505, 499, 515, 531, 534, 499, 15, 2, 0, 0, 0, 0, 0, 0, 537, 514, 501, 504, 456, 541, 489, 511, 30, 16, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4170, 3918, 4042, 4103, 4245, 4130, 4099] avg=4096.0000 max=4245 max_rank=5 min=3918 min_rank=2 imbalance_ratio=1.036377 max_load_reduction=28523 max_load_reduction_pct=0.870453 moved_to_echo=16577 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[547, 495, 512, 528, 487, 500, 471, 535, 0, 0, 0, 0, 0, 0, 0, 0, 528, 508, 487, 472, 516, 551, 528, 514, 0, 0, 0, 0, 0, 0, 0, 0, 514, 506, 460, 517, 537, 499, 504, 559, 0, 0, 0, 0, 0, 0, 0, 0, 485, 533, 492, 511, 481, 496, 484, 525, 0, 0, 0, 0, 0, 0, 0, 0, 521, 510, 524, 497, 541, 528, 490, 509, 6, 4, 0, 0, 0, 0, 0, 0, 511, 504, 519, 518, 561, 480, 504, 491, 0, 0, 0, 0, 0, 0, 0, 0, 484, 538, 528, 525, 522, 513, 538, 513, 10, 0, 0, 0, 0, 0, 0, 0, 474, 520, 450, 563, 517, 478, 526, 530, 28, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4104, 4096, 4007, 4130, 4088, 4171, 4097] avg=4096.0000 max=4171 max_rank=6 min=4007 min_rank=3 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16486 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[517, 523, 498, 513, 496, 503, 505, 534, 0, 0, 0, 0, 0, 0, 0, 0, 517, 519, 508, 528, 511, 516, 515, 490, 0, 0, 0, 0, 0, 0, 0, 0, 510, 529, 522, 545, 503, 514, 522, 534, 0, 0, 0, 0, 0, 0, 0, 0, 545, 503, 496, 512, 529, 513, 518, 516, 0, 0, 0, 0, 0, 0, 0, 0, 499, 502, 498, 479, 492, 522, 517, 525, 6, 4, 0, 0, 0, 0, 0, 0, 525, 521, 490, 507, 500, 516, 518, 516, 0, 0, 0, 0, 0, 0, 0, 0, 493, 533, 514, 525, 530, 501, 499, 483, 11, 0, 0, 0, 0, 0, 0, 0, 507, 464, 508, 481, 507, 517, 503, 509, 30, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4104, 4179, 4132, 4044, 4093, 4089, 4038] avg=4096.0000 max=4179 max_rank=2 min=4038 min_rank=7 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16264 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[472, 498, 526, 531, 515, 505, 542, 503, 0, 0, 0, 0, 0, 0, 0, 0, 503, 524, 508, 515, 525, 484, 509, 528, 0, 0, 0, 0, 0, 0, 0, 0, 530, 554, 496, 509, 456, 539, 478, 514, 0, 0, 0, 0, 0, 0, 0, 0, 542, 505, 481, 506, 501, 503, 504, 535, 0, 0, 0, 0, 0, 0, 0, 0, 575, 539, 533, 506, 506, 503, 560, 511, 6, 4, 0, 0, 0, 0, 0, 0, 523, 479, 508, 468, 512, 529, 521, 476, 0, 0, 0, 0, 0, 0, 0, 0, 514, 510, 500, 515, 497, 506, 541, 540, 12, 0, 0, 0, 0, 0, 0, 0, 500, 518, 499, 515, 481, 500, 483, 495, 29, 13, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4096, 4076, 4077, 4243, 4016, 4135, 4033] avg=4096.0000 max=4243 max_rank=4 min=4016 min_rank=5 imbalance_ratio=1.035889 max_load_reduction=28525 max_load_reduction_pct=0.870514 moved_to_echo=16427 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[500, 483, 533, 519, 520, 534, 488, 525, 0, 0, 0, 0, 0, 0, 0, 0, 482, 561, 552, 506, 540, 519, 495, 512, 0, 0, 0, 0, 0, 0, 0, 0, 515, 522, 544, 509, 505, 503, 483, 538, 0, 0, 0, 0, 0, 0, 0, 0, 572, 523, 531, 511, 482, 502, 520, 524, 0, 0, 0, 0, 0, 0, 0, 0, 509, 512, 504, 522, 453, 538, 499, 511, 6, 4, 0, 0, 0, 0, 0, 0, 494, 498, 518, 496, 516, 536, 502, 510, 0, 0, 0, 0, 0, 0, 0, 0, 499, 489, 480, 517, 511, 513, 534, 482, 11, 0, 0, 0, 0, 0, 0, 0, 492, 499, 498, 531, 514, 514, 505, 459, 28, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4167, 4119, 4165, 4058, 4070, 4036, 4051] avg=4096.0000 max=4167 max_rank=1 min=4036 min_rank=6 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16215 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 3968, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 3968, 4224, 4096, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[523, 511, 499, 522, 530, 567, 527, 516, 531, 462, 495, 533, 488, 479, 470, 506, 527, 510, 497, 496, 524, 490, 501, 518, 480, 519, 489, 505, 502, 476, 530, 550, 485, 526, 559, 499, 554, 527, 508, 500, 488, 483, 519, 533, 531, 531, 514, 507, 490, 532, 512, 501, 534, 468, 518, 518, 500, 503, 492, 503, 564, 537, 530, 529] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[519, 513, 505, 517, 523, 510, 515, 507, 527, 502, 510, 512, 461, 508, 513, 533, 516, 520, 482, 539, 493, 513, 548, 522, 495, 478, 495, 538, 509, 502, 527, 476, 508, 546, 530, 510, 528, 520, 491, 507, 505, 479, 513, 486, 534, 513, 557, 523, 482, 529, 494, 527, 511, 570, 523, 501, 476, 542, 479, 494, 534, 485, 516, 527] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[468, 476, 497, 497, 520, 500, 508, 523, 513, 531, 529, 527, 517, 497, 519, 514, 496, 503, 546, 513, 499, 518, 503, 493, 543, 519, 510, 476, 504, 507, 549, 484, 488, 490, 521, 528, 523, 514, 515, 495, 501, 529, 520, 493, 493, 514, 553, 483, 506, 499, 510, 553, 492, 514, 528, 550, 533, 497, 481, 542, 543, 523, 514, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[518, 501, 540, 544, 500, 530, 530, 512, 532, 513, 533, 532, 533, 523, 521, 512, 548, 500, 524, 517, 458, 502, 481, 550, 491, 512, 504, 523, 508, 500, 530, 493, 482, 512, 522, 519, 503, 484, 544, 475, 521, 495, 547, 532, 491, 475, 512, 520, 489, 495, 505, 557, 530, 478, 514, 506, 509, 511, 498, 541, 485, 495, 518, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[499, 515, 513, 509, 503, 504, 469, 528, 523, 532, 504, 513, 482, 509, 496, 542, 504, 509, 500, 509, 474, 491, 494, 523, 501, 534, 514, 456, 546, 490, 505, 521, 499, 511, 503, 539, 507, 499, 480, 534, 544, 552, 527, 520, 524, 511, 510, 523, 496, 519, 500, 489, 558, 530, 513, 564, 487, 522, 486, 508, 508, 523, 512, 558] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[539, 523, 518, 500, 500, 523, 547, 540, 492, 494, 467, 538, 518, 519, 506, 530, 525, 535, 526, 511, 505, 523, 499, 533, 493, 523, 482, 457, 495, 517, 489, 531, 519, 483, 503, 502, 485, 501, 505, 516, 513, 509, 536, 518, 526, 488, 510, 491, 509, 550, 493, 520, 533, 536, 496, 531, 528, 547, 497, 512, 459, 562, 509, 483] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[521, 538, 466, 527, 546, 502, 553, 489, 537, 500, 530, 508, 483, 515, 528, 490, 549, 483, 484, 491, 512, 531, 539, 529, 484, 502, 503, 493, 500, 508, 542, 520, 524, 519, 512, 524, 518, 508, 530, 527, 491, 508, 499, 502, 514, 516, 525, 528, 485, 495, 493, 475, 503, 517, 545, 519, 536, 503, 499, 521, 501, 519, 512, 497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[503, 520, 540, 495, 541, 541, 516, 498, 500, 510, 506, 527, 517, 510, 530, 545, 533, 492, 515, 503, 476, 552, 561, 517, 520, 526, 540, 498, 494, 513, 489, 517, 502, 479, 533, 511, 489, 490, 507, 470, 501, 541, 517, 522, 500, 506, 493, 548, 503, 491, 482, 490, 495, 526, 494, 526, 486, 477, 562, 491, 501, 519, 519, 552] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[523, 511, 499, 522, 530, 537, 527, 516, 0, 0, 0, 0, 0, 0, 0, 0, 531, 462, 495, 532, 488, 479, 470, 506, 0, 0, 0, 0, 0, 0, 0, 0, 527, 510, 497, 496, 524, 490, 501, 518, 0, 0, 0, 0, 0, 0, 0, 0, 480, 519, 489, 505, 502, 476, 530, 550, 30, 14, 0, 0, 0, 0, 0, 0, 485, 526, 559, 499, 554, 527, 508, 500, 5, 5, 3, 1, 0, 0, 0, 0, 488, 483, 502, 533, 531, 531, 514, 507, 0, 0, 0, 0, 0, 0, 0, 0, 490, 532, 512, 501, 534, 468, 518, 513, 0, 0, 0, 0, 0, 0, 0, 0, 500, 503, 492, 503, 564, 532, 530, 529, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4165, 3963, 4063, 4095, 4172, 4089, 4068, 4153] avg=4096.0000 max=4172 max_rank=4 min=3963 min_rank=1 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16482 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[519, 513, 505, 517, 523, 483, 515, 507, 0, 0, 0, 0, 0, 0, 0, 0, 527, 502, 510, 511, 461, 508, 513, 533, 0, 0, 0, 0, 0, 0, 0, 0, 516, 520, 482, 539, 493, 513, 548, 522, 0, 0, 0, 0, 0, 0, 0, 0, 495, 478, 495, 538, 509, 502, 527, 476, 27, 14, 0, 0, 0, 0, 0, 0, 508, 546, 530, 510, 528, 520, 491, 507, 5, 4, 3, 1, 0, 0, 0, 0, 505, 479, 496, 486, 534, 513, 557, 523, 0, 0, 0, 0, 0, 0, 0, 0, 482, 529, 494, 527, 511, 570, 523, 496, 0, 0, 0, 0, 0, 0, 0, 0, 476, 542, 479, 494, 534, 481, 516, 527, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4065, 4133, 4061, 4153, 4093, 4132, 4049] avg=4096.0000 max=4153 max_rank=4 min=4049 min_rank=7 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16427 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[468, 476, 497, 497, 520, 473, 508, 523, 0, 0, 0, 0, 0, 0, 0, 0, 513, 531, 529, 526, 517, 497, 519, 514, 0, 0, 0, 0, 0, 0, 0, 0, 496, 503, 546, 513, 499, 518, 503, 493, 0, 0, 0, 0, 0, 0, 0, 0, 543, 519, 510, 476, 504, 507, 549, 484, 27, 14, 0, 0, 0, 0, 0, 0, 488, 490, 521, 528, 523, 514, 515, 495, 5, 5, 3, 1, 0, 0, 0, 0, 501, 529, 503, 493, 493, 514, 553, 483, 0, 0, 0, 0, 0, 0, 0, 0, 506, 499, 510, 553, 492, 514, 528, 545, 0, 0, 0, 0, 0, 0, 0, 0, 533, 497, 481, 542, 543, 518, 514, 524, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3962, 4146, 4071, 4133, 4088, 4069, 4147, 4152] avg=4096.0000 max=4152 max_rank=7 min=3962 min_rank=0 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16456 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[518, 501, 540, 544, 500, 502, 530, 512, 0, 0, 0, 0, 0, 0, 0, 0, 532, 513, 533, 531, 533, 523, 521, 512, 0, 0, 0, 0, 0, 0, 0, 0, 548, 500, 524, 517, 458, 502, 481, 550, 0, 0, 0, 0, 0, 0, 0, 0, 491, 512, 504, 523, 508, 500, 530, 493, 28, 15, 0, 0, 0, 0, 0, 0, 482, 512, 522, 519, 503, 484, 544, 475, 5, 4, 3, 1, 0, 0, 0, 0, 521, 495, 529, 532, 491, 475, 512, 520, 0, 0, 0, 0, 0, 0, 0, 0, 489, 495, 505, 557, 530, 478, 514, 501, 0, 0, 0, 0, 0, 0, 0, 0, 509, 511, 498, 541, 485, 491, 518, 488, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4147, 4198, 4080, 4104, 4054, 4075, 4069, 4041] avg=4096.0000 max=4198 max_rank=1 min=4041 min_rank=7 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16239 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[499, 515, 513, 509, 503, 477, 469, 528, 0, 0, 0, 0, 0, 0, 0, 0, 523, 532, 504, 512, 482, 509, 496, 542, 0, 0, 0, 0, 0, 0, 0, 0, 504, 509, 500, 509, 474, 491, 494, 523, 0, 0, 0, 0, 0, 0, 0, 0, 501, 534, 514, 456, 546, 490, 505, 521, 27, 14, 0, 0, 0, 0, 0, 0, 499, 511, 503, 539, 507, 499, 480, 534, 5, 5, 3, 1, 0, 0, 0, 0, 544, 552, 510, 520, 524, 511, 510, 523, 0, 0, 0, 0, 0, 0, 0, 0, 496, 519, 500, 489, 558, 530, 513, 559, 0, 0, 0, 0, 0, 0, 0, 0, 487, 522, 486, 508, 508, 518, 512, 558, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4013, 4100, 4004, 4108, 4086, 4194, 4164, 4099] avg=4096.0000 max=4194 max_rank=5 min=4004 min_rank=2 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16543 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[539, 523, 518, 500, 500, 492, 547, 540, 0, 0, 0, 0, 0, 0, 0, 0, 492, 494, 467, 536, 518, 519, 506, 530, 0, 0, 0, 0, 0, 0, 0, 0, 518, 535, 526, 511, 505, 523, 499, 533, 0, 0, 0, 0, 0, 0, 0, 0, 493, 523, 482, 457, 495, 517, 489, 531, 31, 17, 0, 0, 0, 0, 0, 0, 519, 483, 503, 502, 485, 501, 505, 516, 9, 8, 5, 2, 7, 0, 0, 0, 513, 509, 514, 518, 526, 488, 510, 491, 0, 0, 0, 0, 0, 0, 0, 0, 509, 550, 493, 520, 533, 536, 496, 522, 0, 0, 0, 0, 0, 0, 0, 0, 528, 547, 497, 512, 459, 554, 509, 483, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4159, 4062, 4150, 4035, 4045, 4069, 4159, 4089] avg=4096.0000 max=4159 max_rank=0 min=4035 min_rank=3 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16362 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[503, 520, 540, 495, 541, 512, 516, 498, 0, 0, 0, 0, 0, 0, 0, 0, 500, 510, 506, 526, 517, 510, 530, 545, 0, 0, 0, 0, 0, 0, 0, 0, 533, 492, 515, 503, 476, 552, 561, 517, 0, 0, 0, 0, 0, 0, 0, 0, 520, 526, 540, 498, 494, 513, 489, 517, 29, 14, 0, 0, 0, 0, 0, 0, 502, 479, 533, 511, 489, 490, 507, 470, 5, 5, 3, 1, 0, 0, 0, 0, 501, 541, 500, 522, 500, 506, 493, 548, 0, 0, 0, 0, 0, 0, 0, 0, 503, 491, 482, 490, 495, 526, 494, 521, 0, 0, 0, 0, 0, 0, 0, 0, 486, 477, 562, 491, 501, 514, 519, 552, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4144, 4149, 4140, 3995, 4111, 4002, 4102] avg=4096.0000 max=4149 max_rank=2 min=3995 min_rank=4 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16210 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[521, 538, 466, 527, 546, 475, 553, 489, 0, 0, 0, 0, 0, 0, 0, 0, 537, 500, 530, 507, 483, 515, 528, 490, 0, 0, 0, 0, 0, 0, 0, 0, 549, 483, 484, 491, 512, 531, 539, 529, 0, 0, 0, 0, 0, 0, 0, 0, 484, 502, 503, 493, 500, 508, 542, 520, 27, 13, 0, 0, 0, 0, 0, 0, 524, 519, 512, 524, 518, 508, 530, 527, 5, 5, 2, 1, 0, 0, 0, 0, 491, 508, 484, 502, 514, 516, 525, 528, 0, 0, 0, 0, 0, 0, 0, 0, 485, 495, 493, 475, 503, 517, 545, 514, 0, 0, 0, 0, 0, 0, 0, 0, 536, 503, 499, 521, 501, 514, 512, 497, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4090, 4118, 4092, 4175, 4068, 4027, 4083] avg=4096.0000 max=4175 max_rank=4 min=4027 min_rank=6 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4096, 3968, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 3968, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 3968, 4096, 4096, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3968, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 128, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33792 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[523, 529, 493, 529, 484, 497, 519, 524, 499, 512, 506, 516, 519, 533, 516, 527, 468, 528, 502, 528, 505, 539, 516, 498, 521, 493, 478, 494, 488, 526, 543, 520, 584, 528, 548, 521, 486, 511, 529, 517, 487, 537, 519, 504, 490, 473, 508, 539, 539, 558, 497, 457, 506, 512, 502, 509, 482, 534, 472, 533, 487, 524, 497, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[484, 481, 500, 542, 497, 465, 547, 538, 465, 497, 539, 517, 526, 527, 509, 537, 511, 470, 524, 479, 538, 499, 508, 483, 499, 504, 502, 490, 470, 486, 521, 517, 492, 520, 475, 525, 503, 536, 502, 525, 491, 540, 515, 532, 505, 525, 546, 516, 540, 532, 499, 554, 538, 536, 494, 501, 555, 458, 513, 500, 564, 494, 539, 531] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[471, 506, 495, 516, 493, 516, 515, 504, 524, 529, 488, 490, 530, 536, 474, 507, 501, 504, 494, 494, 493, 536, 556, 499, 509, 583, 481, 502, 543, 513, 512, 511, 533, 529, 530, 492, 554, 517, 545, 487, 489, 524, 511, 501, 512, 508, 528, 520, 485, 553, 492, 521, 498, 511, 535, 515, 535, 503, 534, 490, 489, 498, 458, 546] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[535, 546, 482, 503, 501, 480, 541, 537, 530, 528, 508, 543, 508, 470, 499, 487, 525, 480, 486, 497, 506, 532, 531, 538, 528, 478, 492, 490, 498, 541, 496, 446, 516, 535, 515, 517, 534, 497, 504, 530, 497, 534, 530, 525, 515, 507, 505, 503, 500, 527, 523, 510, 513, 535, 536, 504, 507, 525, 503, 502, 522, 481, 577, 477] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[500, 496, 485, 510, 512, 483, 503, 530, 520, 502, 494, 517, 520, 494, 532, 483, 540, 511, 530, 481, 486, 511, 556, 507, 497, 504, 530, 529, 508, 467, 549, 495, 518, 522, 532, 509, 523, 533, 530, 546, 538, 504, 495, 535, 539, 554, 503, 525, 478, 542, 514, 517, 473, 506, 520, 466, 535, 553, 506, 485, 481, 524, 512, 468] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[535, 529, 549, 492, 501, 528, 493, 519, 502, 522, 493, 520, 512, 501, 482, 511, 504, 510, 481, 512, 529, 466, 541, 508, 511, 514, 503, 477, 506, 517, 488, 517, 496, 529, 559, 540, 473, 504, 518, 543, 506, 530, 488, 520, 525, 501, 502, 492, 530, 495, 547, 555, 542, 498, 501, 510, 507, 545, 533, 474, 509, 515, 483, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[511, 457, 499, 515, 538, 521, 481, 496, 462, 493, 548, 529, 533, 467, 500, 522, 542, 479, 527, 506, 494, 519, 504, 543, 506, 490, 517, 499, 497, 500, 527, 516, 481, 531, 546, 509, 479, 511, 518, 541, 542, 532, 471, 505, 484, 487, 519, 498, 522, 509, 494, 543, 535, 554, 505, 477, 533, 525, 567, 517, 521, 500, 510, 564] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[482, 484, 543, 502, 499, 501, 519, 495, 507, 522, 496, 514, 496, 539, 522, 553, 541, 536, 533, 533, 555, 525, 537, 535, 514, 512, 470, 507, 519, 498, 505, 516, 506, 507, 516, 501, 507, 489, 473, 524, 471, 483, 512, 549, 521, 505, 503, 506, 515, 538, 529, 523, 508, 537, 514, 505, 519, 497, 520, 496, 474, 511, 494, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[523, 529, 493, 529, 484, 497, 519, 524, 34, 0, 0, 0, 0, 0, 0, 0, 499, 512, 506, 516, 519, 533, 516, 527, 7, 0, 0, 0, 0, 0, 0, 0, 468, 528, 502, 528, 505, 539, 508, 498, 0, 0, 0, 0, 0, 0, 0, 0, 521, 493, 478, 494, 488, 526, 543, 520, 49, 0, 0, 0, 0, 0, 0, 0, 584, 528, 499, 521, 486, 511, 529, 517, 0, 0, 0, 0, 0, 0, 0, 0, 487, 536, 519, 504, 490, 473, 508, 539, 0, 0, 0, 0, 0, 0, 0, 0, 539, 524, 497, 457, 506, 512, 502, 509, 0, 0, 0, 0, 0, 0, 0, 0, 482, 534, 472, 533, 487, 524, 497, 505, 1, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4132, 4135, 4076, 4112, 4175, 4056, 4046, 4036] avg=4096.0000 max=4175 max_rank=4 min=4036 min_rank=7 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16313 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[500, 496, 485, 510, 512, 483, 503, 530, 33, 0, 0, 0, 0, 0, 0, 0, 520, 502, 494, 517, 520, 494, 532, 483, 8, 0, 0, 0, 0, 0, 0, 0, 540, 511, 530, 481, 486, 511, 547, 507, 0, 0, 0, 0, 0, 0, 0, 0, 497, 504, 530, 529, 508, 467, 549, 495, 48, 0, 0, 0, 0, 0, 0, 0, 518, 522, 484, 509, 523, 533, 530, 546, 0, 0, 0, 0, 0, 0, 0, 0, 538, 503, 495, 535, 539, 554, 503, 525, 0, 0, 0, 0, 0, 0, 0, 0, 478, 509, 514, 517, 473, 506, 520, 466, 0, 0, 0, 0, 0, 0, 0, 0, 535, 553, 506, 485, 481, 524, 512, 468, 1, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4052, 4070, 4113, 4127, 4165, 4192, 3983, 4066] avg=4096.0000 max=4192 max_rank=5 min=3983 min_rank=6 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16406 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[511, 457, 499, 515, 538, 521, 481, 496, 31, 0, 0, 0, 0, 0, 0, 0, 462, 493, 548, 529, 533, 467, 500, 522, 7, 0, 0, 0, 0, 0, 0, 0, 542, 479, 527, 506, 494, 519, 496, 543, 0, 0, 0, 0, 0, 0, 0, 0, 506, 490, 517, 499, 497, 500, 527, 516, 49, 0, 0, 0, 0, 0, 0, 0, 481, 531, 497, 509, 479, 511, 518, 541, 0, 0, 0, 0, 0, 0, 0, 0, 542, 531, 471, 505, 484, 487, 519, 498, 0, 0, 0, 0, 0, 0, 0, 0, 522, 478, 494, 543, 535, 554, 505, 477, 0, 0, 0, 0, 0, 0, 0, 0, 533, 525, 567, 517, 521, 500, 510, 564, 1, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4049, 4061, 4106, 4101, 4067, 4037, 4108, 4239] avg=4096.0000 max=4239 max_rank=7 min=4037 min_rank=5 imbalance_ratio=1.034912 max_load_reduction=28529 max_load_reduction_pct=0.870636 moved_to_echo=16451 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[484, 481, 500, 542, 497, 465, 547, 538, 32, 0, 0, 0, 0, 0, 0, 0, 465, 497, 539, 517, 526, 527, 509, 537, 7, 0, 0, 0, 0, 0, 0, 0, 511, 470, 524, 479, 538, 499, 500, 483, 0, 0, 0, 0, 0, 0, 0, 0, 499, 504, 502, 490, 470, 486, 521, 517, 43, 0, 0, 0, 0, 0, 0, 0, 492, 520, 432, 525, 503, 536, 502, 525, 0, 0, 0, 0, 0, 0, 0, 0, 491, 539, 515, 532, 505, 525, 546, 516, 0, 0, 0, 0, 0, 0, 0, 0, 540, 500, 499, 554, 538, 536, 494, 501, 0, 0, 0, 0, 0, 0, 0, 0, 555, 458, 513, 500, 564, 494, 539, 531, 1, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4124, 4004, 4032, 4035, 4169, 4162, 4156] avg=4096.0000 max=4169 max_rank=5 min=4004 min_rank=2 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16522 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[535, 546, 482, 503, 501, 480, 541, 537, 32, 0, 0, 0, 0, 0, 0, 0, 530, 528, 508, 543, 508, 470, 499, 487, 7, 0, 0, 0, 0, 0, 0, 0, 525, 480, 486, 497, 506, 532, 523, 538, 0, 0, 0, 0, 0, 0, 0, 0, 528, 478, 492, 490, 498, 541, 496, 446, 46, 0, 0, 0, 0, 0, 0, 0, 516, 535, 469, 517, 534, 497, 504, 530, 0, 0, 0, 0, 0, 0, 0, 0, 497, 533, 530, 525, 515, 507, 505, 503, 0, 0, 0, 0, 0, 0, 0, 0, 500, 495, 523, 510, 513, 535, 536, 504, 0, 0, 0, 0, 0, 0, 0, 0, 507, 525, 503, 502, 522, 481, 577, 477, 1, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4157, 4080, 4087, 4015, 4102, 4115, 4116, 4096] avg=4096.0000 max=4157 max_rank=0 min=4015 min_rank=3 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[471, 506, 495, 516, 493, 516, 515, 504, 33, 0, 0, 0, 0, 0, 0, 0, 524, 529, 488, 490, 530, 536, 474, 507, 8, 0, 0, 0, 0, 0, 0, 0, 501, 504, 494, 494, 493, 536, 547, 499, 0, 0, 0, 0, 0, 0, 0, 0, 509, 583, 481, 502, 543, 513, 512, 511, 48, 0, 0, 0, 0, 0, 0, 0, 533, 529, 482, 492, 554, 517, 545, 487, 0, 0, 0, 0, 0, 0, 0, 0, 489, 523, 511, 501, 512, 508, 528, 520, 0, 0, 0, 0, 0, 0, 0, 0, 485, 520, 492, 521, 498, 511, 535, 515, 0, 0, 0, 0, 0, 0, 0, 0, 535, 503, 534, 490, 489, 498, 458, 546, 1, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4049, 4086, 4068, 4202, 4139, 4092, 4077, 4055] avg=4096.0000 max=4202 max_rank=3 min=4049 min_rank=0 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[535, 529, 549, 492, 501, 528, 493, 519, 33, 6, 0, 0, 0, 0, 0, 0, 502, 522, 493, 520, 512, 501, 482, 511, 12, 0, 0, 0, 0, 0, 0, 0, 504, 510, 481, 512, 529, 466, 518, 508, 0, 0, 0, 0, 0, 0, 0, 0, 511, 514, 503, 477, 506, 517, 488, 517, 54, 5, 0, 0, 0, 0, 0, 0, 496, 529, 505, 540, 473, 504, 518, 543, 0, 0, 0, 0, 0, 0, 0, 0, 506, 524, 488, 520, 525, 501, 502, 492, 0, 0, 0, 0, 0, 0, 0, 0, 530, 457, 547, 555, 542, 498, 501, 510, 0, 0, 0, 0, 0, 0, 0, 0, 507, 545, 533, 474, 509, 515, 483, 525, 6, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4185, 4055, 4028, 4092, 4108, 4058, 4140, 4102] avg=4096.0000 max=4185 max_rank=0 min=4028 min_rank=2 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16408 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[482, 484, 543, 502, 499, 501, 519, 495, 33, 0, 0, 0, 0, 0, 0, 0, 507, 522, 496, 514, 496, 539, 522, 553, 8, 0, 0, 0, 0, 0, 0, 0, 541, 536, 533, 533, 555, 525, 528, 535, 0, 0, 0, 0, 0, 0, 0, 0, 514, 512, 470, 507, 519, 498, 505, 516, 46, 0, 0, 0, 0, 0, 0, 0, 506, 507, 470, 501, 507, 489, 473, 524, 0, 0, 0, 0, 0, 0, 0, 0, 471, 482, 512, 549, 521, 505, 503, 506, 0, 0, 0, 0, 0, 0, 0, 0, 515, 505, 529, 523, 508, 537, 514, 505, 0, 0, 0, 0, 0, 0, 0, 0, 519, 497, 520, 496, 474, 511, 494, 505, 1, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4058, 4157, 4286, 4087, 3977, 4049, 4136, 4018] avg=4096.0000 max=4286 max_rank=2 min=3977 min_rank=4 imbalance_ratio=1.046387 max_load_reduction=28482 max_load_reduction_pct=0.869202 moved_to_echo=16180 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 3840, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[539, 518, 526, 491, 539, 481, 488, 528, 487, 523, 486, 505, 492, 510, 482, 477, 527, 503, 526, 519, 515, 490, 500, 542, 536, 546, 525, 503, 482, 528, 549, 477, 503, 521, 498, 497, 495, 537, 516, 556, 503, 518, 523, 537, 471, 521, 547, 523, 472, 543, 494, 536, 514, 539, 487, 501, 548, 474, 476, 500, 485, 513, 535, 545] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[516, 513, 511, 484, 552, 473, 470, 527, 486, 539, 508, 501, 511, 547, 505, 492, 503, 496, 538, 512, 511, 518, 506, 484, 509, 521, 522, 506, 497, 511, 494, 491, 504, 540, 515, 483, 531, 473, 526, 497, 556, 514, 539, 497, 527, 529, 502, 540, 506, 486, 582, 492, 521, 531, 535, 466, 498, 535, 506, 509, 530, 494, 518, 532] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[509, 505, 501, 488, 469, 467, 530, 537, 534, 487, 541, 497, 513, 472, 521, 492, 468, 555, 490, 508, 517, 533, 508, 542, 523, 484, 511, 513, 514, 505, 527, 533, 546, 507, 490, 518, 522, 518, 488, 500, 473, 518, 525, 485, 511, 513, 525, 499, 535, 517, 508, 530, 505, 532, 541, 537, 509, 505, 505, 542, 526, 501, 526, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[504, 490, 510, 539, 497, 507, 506, 486, 511, 504, 548, 496, 480, 484, 501, 545, 502, 535, 507, 504, 559, 528, 510, 516, 526, 499, 485, 510, 537, 517, 508, 480, 481, 531, 519, 509, 548, 522, 518, 513, 482, 506, 499, 546, 563, 486, 537, 483, 512, 525, 511, 538, 490, 524, 530, 500, 498, 541, 521, 517, 488, 466, 509, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[512, 540, 546, 497, 519, 520, 511, 482, 514, 514, 489, 511, 517, 479, 525, 510, 532, 470, 543, 488, 483, 486, 515, 498, 544, 535, 575, 514, 510, 542, 500, 487, 521, 500, 490, 526, 488, 480, 490, 538, 505, 509, 532, 511, 532, 497, 527, 528, 516, 530, 502, 516, 536, 514, 506, 462, 489, 549, 507, 499, 533, 510, 544, 473] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[528, 475, 516, 544, 516, 533, 504, 507, 513, 500, 485, 522, 508, 476, 507, 498, 530, 515, 525, 503, 503, 493, 517, 532, 500, 513, 484, 530, 501, 492, 511, 503, 540, 482, 503, 496, 533, 543, 498, 518, 501, 540, 466, 527, 511, 517, 531, 498, 477, 539, 491, 502, 500, 547, 544, 547, 529, 524, 511, 487, 509, 506, 529, 538] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[542, 530, 488, 500, 474, 497, 519, 519, 514, 494, 504, 506, 529, 512, 501, 498, 519, 538, 555, 521, 564, 457, 508, 497, 485, 530, 527, 529, 508, 501, 537, 546, 501, 528, 546, 491, 529, 457, 506, 501, 529, 530, 531, 489, 504, 485, 509, 448, 518, 451, 490, 528, 513, 556, 527, 538, 524, 509, 489, 520, 496, 523, 530, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[564, 535, 466, 488, 490, 500, 482, 520, 538, 496, 509, 524, 495, 503, 524, 510, 517, 512, 520, 507, 546, 507, 478, 530, 520, 520, 521, 540, 492, 502, 512, 498, 474, 517, 545, 526, 540, 500, 500, 501, 503, 533, 523, 504, 516, 509, 538, 465, 515, 497, 536, 526, 514, 525, 527, 493, 509, 543, 527, 517, 500, 478, 487, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[539, 518, 526, 491, 539, 481, 488, 528, 13, 10, 1, 0, 0, 0, 0, 0, 487, 523, 486, 505, 492, 510, 482, 477, 32, 13, 0, 0, 0, 0, 0, 0, 527, 503, 513, 519, 515, 490, 500, 542, 0, 0, 0, 0, 0, 0, 0, 0, 536, 546, 512, 503, 482, 528, 549, 477, 0, 0, 0, 0, 0, 0, 0, 0, 503, 521, 498, 497, 495, 537, 516, 556, 4, 0, 0, 0, 0, 0, 0, 0, 503, 518, 523, 537, 471, 521, 537, 523, 0, 0, 0, 0, 0, 0, 0, 0, 472, 543, 494, 536, 514, 507, 487, 501, 0, 0, 0, 0, 0, 0, 0, 0, 548, 469, 476, 500, 485, 513, 535, 545, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4134, 4007, 4109, 4133, 4127, 4133, 4054, 4071] avg=4096.0000 max=4134 max_rank=0 min=4007 min_rank=1 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[516, 513, 511, 484, 552, 473, 470, 527, 13, 9, 1, 0, 0, 0, 0, 0, 486, 539, 508, 501, 511, 547, 505, 492, 31, 14, 0, 0, 0, 0, 0, 0, 503, 496, 524, 512, 511, 518, 506, 484, 0, 0, 0, 0, 0, 0, 0, 0, 509, 521, 509, 506, 497, 511, 494, 491, 0, 0, 0, 0, 0, 0, 0, 0, 504, 540, 515, 483, 531, 473, 526, 497, 4, 0, 0, 0, 0, 0, 0, 0, 556, 514, 539, 497, 527, 529, 493, 540, 0, 0, 0, 0, 0, 0, 0, 0, 506, 486, 582, 492, 521, 500, 535, 466, 0, 0, 0, 0, 0, 0, 0, 0, 498, 530, 506, 509, 530, 494, 518, 532, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4134, 4054, 4038, 4073, 4195, 4088, 4117] avg=4096.0000 max=4195 max_rank=5 min=4038 min_rank=3 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16473 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[528, 475, 516, 544, 516, 533, 504, 507, 12, 9, 1, 0, 0, 0, 0, 0, 513, 500, 485, 522, 508, 476, 507, 498, 32, 13, 0, 0, 0, 0, 0, 0, 530, 515, 512, 503, 503, 493, 517, 532, 0, 0, 0, 0, 0, 0, 0, 0, 500, 513, 472, 530, 501, 492, 511, 503, 0, 0, 0, 0, 0, 0, 0, 0, 540, 482, 503, 496, 533, 543, 498, 518, 4, 0, 0, 0, 0, 0, 0, 0, 501, 540, 466, 527, 511, 517, 522, 498, 0, 0, 0, 0, 0, 0, 0, 0, 477, 539, 491, 502, 500, 515, 544, 547, 0, 0, 0, 0, 0, 0, 0, 0, 529, 519, 511, 487, 509, 506, 529, 538, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4145, 4054, 4105, 4022, 4117, 4082, 4115, 4128] avg=4096.0000 max=4145 max_rank=0 min=4022 min_rank=3 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16442 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[564, 535, 466, 488, 490, 500, 482, 520, 13, 9, 1, 0, 0, 0, 0, 0, 538, 496, 509, 524, 495, 503, 524, 510, 31, 13, 0, 0, 0, 0, 0, 0, 517, 512, 507, 507, 546, 507, 478, 530, 0, 0, 0, 0, 0, 0, 0, 0, 520, 520, 508, 540, 492, 502, 512, 498, 0, 0, 0, 0, 0, 0, 0, 0, 474, 517, 545, 526, 540, 500, 500, 501, 4, 0, 0, 0, 0, 0, 0, 0, 503, 533, 523, 504, 516, 509, 529, 465, 0, 0, 0, 0, 0, 0, 0, 0, 515, 497, 536, 526, 514, 494, 527, 493, 0, 0, 0, 0, 0, 0, 0, 0, 509, 538, 527, 517, 500, 478, 487, 514, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4143, 4104, 4092, 4107, 4082, 4102, 4070] avg=4096.0000 max=4143 max_rank=1 min=4068 min_rank=0 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16361 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[504, 490, 510, 539, 497, 507, 506, 486, 18, 14, 6, 2, 0, 0, 0, 0, 511, 504, 548, 496, 480, 484, 501, 545, 35, 18, 0, 0, 0, 0, 0, 0, 502, 535, 487, 504, 559, 528, 510, 516, 0, 0, 0, 0, 0, 0, 0, 0, 526, 499, 467, 510, 537, 517, 508, 480, 0, 0, 0, 0, 0, 0, 0, 0, 481, 531, 519, 509, 548, 522, 518, 513, 10, 0, 0, 0, 0, 0, 0, 0, 482, 506, 499, 546, 563, 486, 523, 483, 0, 0, 0, 0, 0, 0, 0, 0, 512, 525, 511, 538, 490, 489, 530, 500, 0, 0, 0, 0, 0, 0, 0, 0, 498, 525, 521, 517, 488, 466, 509, 524, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4122, 4141, 4044, 4151, 4088, 4095, 4048] avg=4096.0000 max=4151 max_rank=4 min=4044 min_rank=3 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[512, 540, 546, 497, 519, 520, 511, 482, 15, 9, 1, 0, 0, 0, 0, 0, 514, 514, 489, 511, 517, 479, 525, 510, 30, 14, 0, 0, 0, 0, 0, 0, 532, 470, 529, 488, 483, 486, 515, 498, 0, 0, 0, 0, 0, 0, 0, 0, 544, 535, 560, 514, 510, 542, 500, 487, 0, 0, 0, 0, 0, 0, 0, 0, 521, 500, 490, 526, 488, 480, 490, 538, 4, 0, 0, 0, 0, 0, 0, 0, 505, 509, 532, 511, 532, 497, 518, 528, 0, 0, 0, 0, 0, 0, 0, 0, 516, 530, 502, 516, 536, 484, 506, 462, 0, 0, 0, 0, 0, 0, 0, 0, 489, 544, 507, 499, 533, 510, 544, 473, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4152, 4103, 4001, 4192, 4037, 4132, 4052, 4099] avg=4096.0000 max=4192 max_rank=3 min=4001 min_rank=2 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16320 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[509, 505, 501, 488, 469, 467, 530, 537, 13, 9, 1, 0, 0, 0, 0, 0, 534, 487, 541, 497, 513, 472, 521, 492, 31, 12, 0, 0, 0, 0, 0, 0, 468, 555, 478, 508, 517, 533, 508, 542, 0, 0, 0, 0, 0, 0, 0, 0, 523, 484, 498, 513, 514, 505, 527, 533, 0, 0, 0, 0, 0, 0, 0, 0, 546, 507, 490, 518, 522, 518, 488, 500, 4, 0, 0, 0, 0, 0, 0, 0, 473, 518, 525, 485, 511, 513, 516, 499, 0, 0, 0, 0, 0, 0, 0, 0, 535, 517, 508, 530, 505, 501, 541, 537, 0, 0, 0, 0, 0, 0, 0, 0, 509, 500, 505, 542, 526, 501, 526, 517, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4029, 4100, 4109, 4097, 4093, 4040, 4174, 4126] avg=4096.0000 max=4174 max_rank=6 min=4029 min_rank=0 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16433 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[542, 530, 488, 500, 474, 497, 519, 519, 13, 9, 1, 0, 0, 0, 0, 0, 514, 494, 504, 506, 529, 512, 501, 498, 33, 14, 0, 0, 0, 0, 0, 0, 519, 538, 541, 521, 564, 457, 508, 497, 0, 0, 0, 0, 0, 0, 0, 0, 485, 530, 514, 529, 508, 501, 537, 546, 0, 0, 0, 0, 0, 0, 0, 0, 501, 528, 546, 491, 529, 457, 506, 501, 4, 0, 0, 0, 0, 0, 0, 0, 529, 530, 531, 489, 504, 485, 500, 448, 0, 0, 0, 0, 0, 0, 0, 0, 518, 451, 490, 528, 513, 523, 527, 538, 0, 0, 0, 0, 0, 0, 0, 0, 524, 504, 489, 520, 496, 523, 530, 523, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4105, 4145, 4150, 4063, 4016, 4088, 4109] avg=4096.0000 max=4150 max_rank=3 min=4016 min_rank=5 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16276 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 128, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[542, 499, 513, 501, 493, 553, 488, 518, 483, 504, 534, 525, 508, 503, 495, 508, 503, 540, 549, 458, 501, 499, 557, 520, 546, 519, 522, 470, 483, 534, 518, 474, 535, 510, 531, 503, 520, 521, 506, 509, 529, 507, 512, 501, 560, 508, 534, 510, 510, 492, 503, 523, 502, 537, 520, 479, 488, 508, 494, 533, 524, 499, 490, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[520, 551, 508, 527, 512, 520, 484, 496, 496, 527, 490, 495, 491, 522, 477, 541, 498, 506, 528, 538, 481, 533, 500, 495, 509, 527, 497, 508, 550, 486, 537, 494, 537, 490, 513, 489, 516, 539, 519, 513, 540, 523, 478, 521, 474, 516, 541, 526, 543, 483, 508, 500, 493, 520, 524, 518, 527, 528, 483, 486, 549, 510, 496, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[506, 536, 494, 511, 522, 510, 526, 485, 512, 509, 520, 529, 541, 508, 504, 504, 525, 498, 474, 532, 485, 534, 511, 510, 492, 526, 510, 527, 501, 514, 503, 504, 530, 554, 488, 504, 488, 521, 522, 501, 520, 512, 507, 531, 476, 504, 518, 472, 530, 528, 517, 479, 509, 504, 551, 508, 536, 525, 464, 530, 527, 495, 541, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[543, 484, 517, 492, 541, 571, 502, 482, 514, 540, 505, 513, 524, 474, 495, 518, 493, 489, 509, 509, 487, 513, 544, 526, 466, 504, 496, 511, 471, 506, 523, 505, 525, 486, 504, 491, 493, 482, 514, 511, 536, 500, 510, 494, 499, 504, 540, 554, 540, 500, 534, 562, 491, 498, 549, 518, 545, 569, 510, 514, 515, 506, 493, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[518, 499, 506, 523, 528, 545, 500, 486, 500, 528, 506, 503, 492, 507, 495, 499, 509, 539, 510, 531, 525, 495, 540, 514, 524, 496, 498, 512, 509, 510, 506, 523, 539, 515, 483, 490, 557, 499, 534, 473, 500, 497, 492, 511, 522, 506, 505, 520, 520, 526, 537, 531, 495, 534, 509, 513, 534, 505, 528, 473, 504, 515, 484, 541] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[524, 517, 527, 536, 489, 465, 471, 516, 505, 486, 481, 514, 509, 526, 493, 512, 487, 559, 531, 516, 500, 517, 553, 522, 522, 490, 556, 536, 497, 466, 523, 513, 498, 520, 473, 518, 529, 544, 491, 541, 529, 526, 550, 477, 559, 551, 485, 474, 507, 544, 499, 506, 506, 515, 475, 506, 509, 482, 544, 504, 497, 510, 541, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[520, 508, 544, 500, 529, 526, 485, 517, 482, 523, 492, 500, 507, 543, 487, 481, 463, 499, 499, 495, 487, 530, 534, 520, 520, 539, 508, 525, 486, 499, 506, 501, 530, 521, 518, 488, 529, 505, 541, 530, 537, 526, 524, 514, 526, 477, 495, 520, 497, 497, 526, 520, 530, 501, 544, 503, 494, 493, 505, 520, 549, 530, 485, 538] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[520, 511, 544, 510, 552, 507, 506, 488, 538, 508, 518, 522, 503, 487, 483, 530, 502, 518, 517, 527, 463, 555, 512, 475, 449, 490, 504, 494, 488, 495, 507, 522, 512, 504, 538, 507, 520, 515, 543, 497, 510, 495, 540, 499, 484, 537, 476, 514, 530, 518, 524, 529, 528, 550, 532, 538, 499, 494, 538, 508, 485, 505, 517, 537] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[543, 484, 517, 492, 541, 558, 502, 482, 0, 0, 0, 0, 0, 0, 0, 0, 514, 540, 505, 513, 524, 474, 495, 518, 13, 9, 8, 5, 2, 0, 0, 0, 493, 489, 509, 509, 487, 513, 542, 526, 0, 0, 0, 0, 0, 0, 0, 0, 466, 504, 496, 511, 471, 506, 523, 505, 38, 3, 0, 0, 0, 0, 0, 0, 513, 486, 504, 491, 493, 482, 514, 511, 0, 0, 0, 0, 0, 0, 0, 0, 528, 500, 510, 494, 499, 504, 540, 554, 0, 0, 0, 0, 0, 0, 0, 0, 540, 500, 534, 562, 491, 498, 511, 518, 0, 0, 0, 0, 0, 0, 0, 0, 545, 569, 510, 514, 515, 506, 493, 509, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4120, 4068, 4023, 3994, 4129, 4154, 4161] avg=4096.0000 max=4161 max_rank=7 min=3994 min_rank=4 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16438 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[506, 536, 494, 511, 522, 499, 526, 485, 0, 0, 0, 0, 0, 0, 0, 0, 512, 509, 520, 529, 541, 508, 504, 504, 11, 9, 8, 5, 2, 0, 0, 0, 525, 498, 474, 532, 485, 534, 509, 510, 0, 0, 0, 0, 0, 0, 0, 0, 492, 526, 510, 527, 501, 514, 503, 504, 38, 3, 0, 0, 0, 0, 0, 0, 518, 554, 488, 504, 488, 521, 522, 501, 0, 0, 0, 0, 0, 0, 0, 0, 512, 512, 507, 531, 476, 504, 518, 472, 0, 0, 0, 0, 0, 0, 0, 0, 530, 528, 517, 479, 509, 504, 513, 508, 0, 0, 0, 0, 0, 0, 0, 0, 536, 525, 464, 530, 527, 495, 541, 508, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4162, 4067, 4118, 4096, 4032, 4088, 4126] avg=4096.0000 max=4162 max_rank=1 min=4032 min_rank=5 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16342 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[542, 499, 513, 501, 493, 541, 488, 518, 0, 0, 0, 0, 0, 0, 0, 0, 483, 504, 534, 525, 508, 503, 495, 508, 12, 9, 8, 5, 2, 0, 0, 0, 503, 540, 549, 458, 501, 499, 555, 520, 0, 0, 0, 0, 0, 0, 0, 0, 546, 519, 522, 470, 483, 534, 518, 474, 36, 3, 0, 0, 0, 0, 0, 0, 523, 510, 531, 503, 520, 521, 506, 509, 0, 0, 0, 0, 0, 0, 0, 0, 521, 507, 512, 501, 560, 508, 534, 510, 0, 0, 0, 0, 0, 0, 0, 0, 510, 492, 503, 523, 502, 537, 484, 479, 0, 0, 0, 0, 0, 0, 0, 0, 488, 508, 494, 533, 524, 499, 490, 505, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4125, 4105, 4123, 4153, 4030, 4041] avg=4096.0000 max=4153 max_rank=5 min=4030 min_rank=6 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[520, 511, 544, 510, 552, 492, 506, 488, 0, 0, 0, 0, 0, 0, 0, 0, 538, 508, 518, 522, 503, 487, 483, 530, 15, 11, 12, 7, 7, 0, 0, 0, 502, 518, 517, 527, 463, 555, 505, 475, 0, 0, 0, 0, 0, 0, 0, 0, 449, 490, 504, 494, 488, 495, 507, 522, 41, 5, 0, 0, 0, 0, 0, 0, 496, 504, 538, 507, 520, 515, 543, 497, 0, 0, 0, 0, 0, 0, 0, 0, 498, 495, 540, 499, 484, 537, 476, 514, 0, 0, 0, 0, 0, 0, 0, 0, 530, 518, 524, 529, 528, 550, 491, 538, 0, 0, 0, 0, 0, 0, 0, 0, 499, 494, 538, 508, 485, 505, 517, 530, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4123, 4141, 4062, 3995, 4120, 4043, 4208, 4076] avg=4096.0000 max=4208 max_rank=6 min=3995 min_rank=3 imbalance_ratio=1.027344 max_load_reduction=28560 max_load_reduction_pct=0.871582 moved_to_echo=16447 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[518, 499, 506, 523, 528, 533, 500, 486, 0, 0, 0, 0, 0, 0, 0, 0, 500, 528, 506, 503, 492, 507, 495, 499, 12, 9, 7, 5, 2, 0, 0, 0, 509, 539, 510, 531, 525, 495, 538, 514, 0, 0, 0, 0, 0, 0, 0, 0, 524, 496, 498, 512, 509, 510, 506, 523, 35, 3, 0, 0, 0, 0, 0, 0, 527, 515, 483, 490, 557, 499, 534, 473, 0, 0, 0, 0, 0, 0, 0, 0, 493, 497, 492, 511, 522, 506, 505, 520, 0, 0, 0, 0, 0, 0, 0, 0, 520, 526, 537, 531, 495, 534, 474, 513, 0, 0, 0, 0, 0, 0, 0, 0, 534, 505, 528, 473, 504, 515, 484, 536, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4065, 4161, 4116, 4078, 4046, 4130, 4079] avg=4096.0000 max=4161 max_rank=2 min=4046 min_rank=5 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16333 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[520, 551, 508, 527, 512, 509, 484, 496, 0, 0, 0, 0, 0, 0, 0, 0, 496, 527, 490, 495, 491, 522, 477, 541, 11, 9, 8, 5, 2, 0, 0, 0, 498, 506, 528, 538, 481, 533, 498, 495, 0, 0, 0, 0, 0, 0, 0, 0, 509, 527, 497, 508, 550, 486, 537, 494, 36, 3, 0, 0, 0, 0, 0, 0, 525, 490, 513, 489, 516, 539, 519, 513, 0, 0, 0, 0, 0, 0, 0, 0, 532, 523, 478, 521, 474, 516, 541, 526, 0, 0, 0, 0, 0, 0, 0, 0, 543, 483, 508, 500, 493, 520, 488, 518, 0, 0, 0, 0, 0, 0, 0, 0, 527, 528, 483, 486, 549, 510, 496, 516, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4074, 4077, 4147, 4104, 4111, 4053, 4095] avg=4096.0000 max=4147 max_rank=3 min=4053 min_rank=6 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[520, 508, 544, 500, 529, 514, 485, 517, 0, 0, 0, 0, 0, 0, 0, 0, 482, 523, 492, 500, 507, 543, 487, 481, 12, 9, 8, 5, 2, 0, 0, 0, 463, 499, 499, 495, 487, 530, 532, 520, 0, 0, 0, 0, 0, 0, 0, 0, 520, 539, 508, 525, 486, 499, 506, 501, 38, 3, 0, 0, 0, 0, 0, 0, 518, 521, 518, 488, 529, 505, 541, 530, 0, 0, 0, 0, 0, 0, 0, 0, 529, 526, 524, 514, 526, 477, 495, 520, 0, 0, 0, 0, 0, 0, 0, 0, 497, 497, 526, 520, 530, 501, 506, 503, 0, 0, 0, 0, 0, 0, 0, 0, 494, 493, 505, 520, 549, 530, 485, 533, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4117, 4051, 4025, 4125, 4150, 4111, 4080, 4109] avg=4096.0000 max=4150 max_rank=4 min=4025 min_rank=2 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16450 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[524, 517, 527, 536, 489, 455, 471, 516, 0, 0, 0, 0, 0, 0, 0, 0, 505, 486, 481, 514, 509, 526, 493, 512, 10, 8, 8, 5, 2, 0, 0, 0, 487, 559, 531, 516, 500, 517, 551, 522, 0, 0, 0, 0, 0, 0, 0, 0, 522, 490, 556, 536, 497, 466, 523, 513, 33, 3, 0, 0, 0, 0, 0, 0, 487, 520, 473, 518, 529, 544, 491, 541, 0, 0, 0, 0, 0, 0, 0, 0, 521, 526, 550, 477, 559, 551, 485, 474, 0, 0, 0, 0, 0, 0, 0, 0, 507, 544, 499, 506, 506, 515, 442, 506, 0, 0, 0, 0, 0, 0, 0, 0, 509, 482, 544, 504, 497, 510, 541, 494, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4035, 4059, 4183, 4139, 4103, 4143, 4025, 4081] avg=4096.0000 max=4183 max_rank=2 min=4025 min_rank=6 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16352 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 3968, 4096, 128, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4224, 3968, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 3968, 4224, 4352, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 3968, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[511, 531, 506, 508, 513, 511, 478, 502, 500, 541, 505, 518, 488, 513, 533, 510, 509, 522, 503, 510, 473, 492, 549, 518, 521, 536, 571, 506, 541, 505, 481, 497, 532, 505, 543, 534, 509, 492, 517, 504, 503, 543, 509, 484, 488, 494, 514, 504, 516, 535, 504, 497, 506, 512, 521, 556, 484, 501, 513, 507, 505, 498, 514, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[498, 484, 496, 517, 525, 532, 510, 492, 519, 526, 481, 521, 518, 506, 530, 478, 479, 517, 516, 528, 470, 522, 528, 521, 508, 463, 522, 505, 511, 534, 525, 514, 506, 482, 527, 489, 510, 481, 519, 515, 534, 503, 504, 508, 500, 493, 529, 525, 503, 511, 538, 525, 510, 532, 512, 514, 518, 489, 528, 515, 515, 533, 551, 553] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[508, 534, 529, 491, 518, 506, 516, 575, 506, 520, 483, 533, 512, 527, 529, 498, 478, 552, 483, 510, 511, 541, 530, 509, 516, 467, 524, 523, 476, 508, 498, 505, 513, 535, 508, 494, 505, 532, 543, 510, 520, 500, 501, 497, 521, 502, 516, 513, 486, 511, 547, 529, 491, 482, 497, 523, 538, 493, 502, 482, 494, 533, 513, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[507, 509, 471, 524, 549, 555, 486, 514, 496, 530, 545, 513, 510, 480, 511, 514, 466, 545, 543, 496, 521, 486, 505, 502, 485, 467, 502, 517, 501, 493, 512, 498, 504, 536, 531, 518, 481, 538, 513, 511, 513, 508, 505, 525, 520, 481, 514, 508, 576, 536, 492, 503, 535, 538, 521, 491, 491, 463, 542, 523, 501, 585, 513, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[497, 536, 520, 502, 552, 506, 540, 497, 519, 496, 535, 495, 476, 486, 476, 520, 512, 479, 536, 491, 523, 499, 499, 494, 506, 486, 507, 536, 526, 541, 472, 512, 510, 478, 543, 502, 528, 536, 519, 507, 523, 481, 519, 534, 536, 488, 521, 513, 531, 518, 527, 528, 502, 509, 553, 511, 495, 549, 463, 515, 489, 545, 517, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[531, 555, 484, 487, 525, 532, 523, 519, 554, 479, 523, 510, 514, 516, 529, 520, 517, 475, 496, 514, 517, 523, 516, 488, 479, 527, 540, 540, 482, 535, 495, 502, 495, 504, 478, 518, 515, 529, 510, 526, 499, 521, 518, 492, 496, 530, 530, 559, 511, 517, 506, 560, 519, 527, 491, 490, 487, 483, 499, 463, 505, 538, 517, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[480, 501, 533, 535, 522, 498, 506, 485, 522, 508, 518, 505, 553, 516, 498, 524, 517, 512, 541, 516, 538, 500, 555, 469, 508, 504, 520, 506, 542, 508, 471, 538, 507, 497, 478, 511, 488, 539, 522, 517, 495, 522, 506, 509, 506, 557, 487, 486, 525, 512, 509, 539, 510, 528, 502, 491, 487, 527, 495, 511, 512, 524, 488, 532] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[549, 494, 501, 490, 500, 494, 465, 514, 514, 488, 488, 539, 534, 523, 541, 506, 538, 557, 478, 506, 531, 502, 548, 516, 458, 530, 522, 520, 502, 528, 515, 527, 510, 486, 550, 499, 521, 525, 561, 527, 550, 525, 530, 493, 500, 498, 504, 532, 507, 514, 530, 500, 498, 497, 497, 502, 501, 484, 496, 525, 497, 513, 495, 483] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[511, 531, 506, 508, 512, 511, 478, 502, 0, 0, 0, 0, 0, 0, 0, 0, 500, 541, 505, 518, 488, 513, 528, 510, 0, 0, 0, 0, 0, 0, 0, 0, 509, 522, 503, 510, 473, 492, 549, 518, 3, 0, 0, 0, 0, 0, 0, 0, 521, 536, 571, 506, 541, 505, 481, 497, 12, 5, 0, 0, 0, 0, 0, 0, 532, 505, 543, 534, 509, 492, 505, 504, 0, 0, 0, 0, 0, 0, 0, 0, 503, 543, 509, 484, 488, 494, 514, 504, 2, 1, 0, 0, 0, 0, 0, 0, 516, 535, 504, 469, 506, 512, 521, 556, 0, 0, 0, 0, 0, 0, 0, 0, 484, 501, 513, 507, 505, 498, 514, 522, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4059, 4103, 4079, 4175, 4124, 4042, 4119, 4067] avg=4096.0000 max=4175 max_rank=3 min=4042 min_rank=5 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16352 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[498, 484, 496, 517, 524, 532, 510, 492, 0, 0, 0, 0, 0, 0, 0, 0, 519, 526, 481, 521, 518, 506, 525, 478, 0, 0, 0, 0, 0, 0, 0, 0, 479, 517, 516, 528, 470, 522, 528, 521, 3, 0, 0, 0, 0, 0, 0, 0, 508, 463, 522, 505, 511, 534, 525, 514, 12, 5, 0, 0, 0, 0, 0, 0, 506, 482, 527, 489, 510, 481, 507, 515, 0, 0, 0, 0, 0, 0, 0, 0, 534, 503, 504, 508, 500, 493, 529, 525, 2, 1, 0, 0, 0, 0, 0, 0, 503, 511, 538, 495, 510, 532, 512, 514, 0, 0, 0, 0, 0, 0, 0, 0, 518, 489, 528, 515, 515, 533, 551, 553, 25, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4053, 4074, 4084, 4099, 4017, 4099, 4115, 4227] avg=4096.0000 max=4227 max_rank=7 min=4017 min_rank=4 imbalance_ratio=1.031982 max_load_reduction=28541 max_load_reduction_pct=0.871002 moved_to_echo=16458 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[531, 555, 484, 487, 524, 532, 523, 519, 0, 0, 0, 0, 0, 0, 0, 0, 554, 479, 523, 510, 514, 516, 524, 520, 0, 0, 0, 0, 0, 0, 0, 0, 517, 475, 496, 514, 517, 523, 516, 488, 3, 0, 0, 0, 0, 0, 0, 0, 479, 527, 540, 540, 482, 535, 495, 502, 12, 5, 0, 0, 0, 0, 0, 0, 495, 504, 478, 518, 515, 529, 498, 526, 0, 0, 0, 0, 0, 0, 0, 0, 499, 521, 518, 492, 496, 530, 530, 559, 2, 1, 0, 0, 0, 0, 0, 0, 511, 517, 506, 529, 519, 527, 491, 490, 0, 0, 0, 0, 0, 0, 0, 0, 487, 483, 499, 463, 505, 538, 517, 488, 26, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4155, 4140, 4049, 4117, 4063, 4148, 4090, 4006] avg=4096.0000 max=4155 max_rank=0 min=4006 min_rank=7 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16307 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[497, 536, 520, 502, 551, 506, 540, 497, 0, 0, 0, 0, 0, 0, 0, 0, 519, 496, 535, 495, 476, 486, 471, 520, 0, 0, 0, 0, 0, 0, 0, 0, 512, 479, 536, 491, 523, 499, 499, 494, 3, 0, 0, 0, 0, 0, 0, 0, 506, 486, 507, 536, 526, 541, 472, 512, 12, 5, 0, 0, 0, 0, 0, 0, 510, 478, 543, 502, 528, 536, 507, 507, 0, 0, 0, 0, 0, 0, 0, 0, 523, 481, 519, 534, 536, 488, 521, 513, 2, 1, 0, 0, 0, 0, 0, 0, 531, 518, 527, 498, 502, 509, 553, 511, 0, 0, 0, 0, 0, 0, 0, 0, 495, 549, 463, 515, 489, 545, 517, 506, 25, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4149, 3998, 4036, 4103, 4111, 4118, 4149, 4104] avg=4096.0000 max=4149 max_rank=0 min=3998 min_rank=1 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16482 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[508, 534, 529, 491, 517, 506, 516, 575, 0, 0, 0, 0, 0, 0, 0, 0, 506, 520, 483, 533, 512, 527, 524, 498, 0, 0, 0, 0, 0, 0, 0, 0, 478, 552, 483, 510, 511, 541, 530, 509, 3, 0, 0, 0, 0, 0, 0, 0, 516, 467, 524, 523, 476, 508, 498, 505, 13, 5, 0, 0, 0, 0, 0, 0, 513, 535, 508, 494, 505, 532, 530, 510, 0, 0, 0, 0, 0, 0, 0, 0, 520, 500, 501, 497, 521, 502, 516, 513, 2, 1, 0, 0, 0, 0, 0, 0, 486, 511, 547, 499, 491, 482, 497, 523, 0, 0, 0, 0, 0, 0, 0, 0, 538, 493, 502, 482, 494, 533, 513, 521, 25, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4176, 4103, 4117, 4035, 4127, 4073, 4036, 4101] avg=4096.0000 max=4176 max_rank=0 min=4035 min_rank=3 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16337 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[507, 509, 471, 524, 548, 555, 486, 514, 0, 0, 0, 0, 0, 0, 0, 0, 496, 530, 545, 513, 510, 480, 506, 514, 0, 0, 0, 0, 0, 0, 0, 0, 466, 545, 543, 496, 521, 486, 505, 502, 3, 0, 0, 0, 0, 0, 0, 0, 485, 467, 502, 517, 501, 493, 512, 498, 12, 5, 0, 0, 0, 0, 0, 0, 504, 536, 531, 518, 481, 538, 501, 511, 0, 0, 0, 0, 0, 0, 0, 0, 513, 508, 505, 525, 520, 481, 514, 508, 2, 1, 0, 0, 0, 0, 0, 0, 576, 536, 492, 474, 535, 538, 521, 491, 0, 0, 0, 0, 0, 0, 0, 0, 491, 463, 542, 523, 501, 585, 513, 499, 24, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4094, 4067, 3992, 4120, 4077, 4163, 4141] avg=4096.0000 max=4163 max_rank=6 min=3992 min_rank=3 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16501 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[480, 501, 533, 535, 521, 498, 506, 485, 0, 0, 0, 0, 0, 0, 0, 0, 522, 508, 518, 505, 553, 516, 493, 524, 0, 0, 0, 0, 0, 0, 0, 0, 517, 512, 541, 516, 538, 500, 555, 469, 3, 0, 0, 0, 0, 0, 0, 0, 508, 504, 520, 506, 542, 508, 471, 538, 13, 5, 0, 0, 0, 0, 0, 0, 507, 497, 478, 511, 488, 539, 509, 517, 0, 0, 0, 0, 0, 0, 0, 0, 495, 522, 506, 509, 506, 557, 487, 486, 2, 1, 0, 0, 0, 0, 0, 0, 525, 512, 509, 509, 510, 528, 502, 491, 0, 0, 0, 0, 0, 0, 0, 0, 487, 527, 495, 511, 512, 524, 488, 532, 25, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4059, 4139, 4151, 4115, 4046, 4071, 4086, 4101] avg=4096.0000 max=4151 max_rank=2 min=4046 min_rank=4 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16304 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[549, 494, 501, 490, 496, 494, 465, 514, 0, 0, 0, 0, 0, 0, 0, 0, 514, 488, 488, 539, 534, 523, 525, 506, 0, 0, 0, 0, 0, 0, 0, 0, 538, 557, 478, 506, 531, 502, 548, 516, 9, 0, 0, 0, 0, 0, 0, 0, 458, 530, 522, 520, 502, 528, 515, 527, 19, 8, 3, 0, 0, 0, 0, 0, 510, 486, 550, 499, 521, 525, 542, 527, 0, 0, 0, 0, 0, 0, 0, 0, 550, 525, 530, 493, 500, 498, 504, 532, 4, 4, 0, 0, 0, 0, 0, 0, 507, 514, 530, 465, 498, 497, 497, 502, 0, 0, 0, 0, 0, 0, 0, 0, 501, 484, 496, 525, 497, 513, 495, 483, 27, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4003, 4117, 4185, 4132, 4160, 4140, 4010, 4021] avg=4096.0000 max=4185 max_rank=2 min=4003 min_rank=0 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16331 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4352, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4352, 4224, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4224, 3968, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[485, 537, 450, 544, 492, 514, 516, 495, 474, 488, 501, 508, 543, 512, 508, 510, 493, 505, 506, 486, 508, 556, 481, 502, 482, 506, 511, 494, 530, 540, 517, 534, 525, 513, 572, 517, 544, 470, 493, 496, 539, 533, 540, 523, 507, 525, 517, 531, 558, 520, 501, 466, 514, 502, 512, 503, 531, 497, 564, 466, 482, 502, 549, 528] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[490, 518, 498, 531, 509, 481, 493, 519, 521, 535, 505, 564, 506, 481, 462, 499, 512, 503, 485, 499, 520, 521, 504, 529, 525, 527, 490, 506, 536, 522, 509, 521, 504, 508, 508, 520, 491, 529, 507, 524, 529, 480, 504, 532, 513, 522, 541, 513, 535, 514, 515, 546, 493, 514, 526, 519, 500, 493, 492, 515, 521, 506, 510, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[512, 539, 496, 487, 483, 526, 546, 535, 540, 508, 491, 549, 508, 549, 519, 537, 541, 507, 522, 521, 509, 505, 497, 519, 493, 516, 531, 518, 502, 487, 511, 485, 471, 539, 503, 502, 506, 532, 485, 491, 514, 537, 507, 489, 535, 483, 508, 524, 505, 515, 506, 505, 526, 496, 503, 492, 534, 498, 530, 502, 525, 501, 494, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[506, 523, 549, 514, 524, 483, 549, 505, 531, 494, 503, 502, 529, 524, 508, 486, 505, 511, 527, 536, 512, 539, 491, 464, 494, 522, 540, 512, 557, 481, 508, 473, 495, 496, 531, 494, 521, 508, 546, 504, 519, 508, 536, 519, 507, 508, 528, 477, 505, 523, 492, 534, 540, 484, 513, 483, 518, 507, 554, 507, 493, 524, 510, 482] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[503, 563, 499, 530, 531, 493, 505, 488, 574, 514, 516, 461, 550, 518, 526, 494, 557, 492, 500, 514, 493, 544, 523, 498, 505, 530, 504, 514, 504, 488, 499, 481, 529, 519, 522, 529, 484, 512, 523, 527, 470, 508, 520, 538, 521, 514, 474, 487, 511, 501, 501, 508, 543, 504, 532, 457, 530, 532, 525, 512, 494, 497, 518, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[516, 508, 537, 505, 501, 476, 489, 520, 487, 514, 517, 536, 542, 526, 482, 546, 519, 544, 527, 531, 524, 511, 515, 530, 472, 520, 498, 498, 486, 456, 550, 527, 500, 523, 506, 502, 510, 523, 517, 493, 515, 506, 509, 536, 503, 509, 545, 523, 494, 491, 483, 523, 484, 492, 516, 511, 511, 495, 546, 540, 475, 518, 537, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[489, 522, 505, 481, 479, 503, 546, 503, 509, 478, 489, 510, 530, 571, 512, 532, 500, 494, 512, 499, 516, 519, 515, 515, 503, 525, 479, 489, 536, 497, 518, 520, 496, 484, 517, 532, 497, 514, 516, 516, 514, 486, 527, 539, 490, 525, 544, 531, 470, 513, 560, 538, 493, 511, 485, 520, 532, 525, 491, 521, 485, 548, 519, 533] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[538, 510, 530, 499, 572, 511, 537, 515, 462, 506, 529, 512, 563, 488, 522, 492, 528, 502, 519, 517, 510, 545, 484, 470, 529, 520, 521, 519, 535, 480, 476, 510, 482, 483, 538, 509, 493, 506, 517, 552, 504, 490, 497, 534, 501, 533, 529, 500, 530, 505, 535, 525, 509, 506, 501, 566, 512, 532, 476, 495, 510, 466, 496, 485] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[485, 537, 450, 544, 492, 514, 516, 495, 1, 0, 0, 0, 0, 0, 0, 0, 474, 488, 501, 508, 526, 512, 508, 510, 0, 0, 0, 0, 0, 0, 0, 0, 493, 505, 506, 486, 508, 555, 481, 502, 0, 0, 0, 0, 0, 0, 0, 0, 482, 506, 511, 494, 530, 540, 517, 534, 24, 0, 0, 0, 0, 0, 0, 0, 525, 513, 572, 517, 544, 470, 493, 496, 6, 0, 0, 0, 0, 0, 0, 0, 539, 533, 540, 499, 507, 525, 517, 531, 0, 0, 0, 0, 0, 0, 0, 0, 558, 520, 501, 466, 514, 502, 512, 503, 11, 0, 0, 0, 0, 0, 0, 0, 531, 497, 564, 466, 482, 502, 549, 528, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4034, 4027, 4036, 4138, 4136, 4191, 4087, 4119] avg=4096.0000 max=4191 max_rank=5 min=4027 min_rank=1 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16533 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[512, 539, 496, 487, 483, 526, 546, 535, 1, 0, 0, 0, 0, 0, 0, 0, 540, 508, 491, 549, 493, 549, 519, 537, 0, 0, 0, 0, 0, 0, 0, 0, 541, 507, 522, 521, 509, 504, 497, 519, 0, 0, 0, 0, 0, 0, 0, 0, 493, 516, 531, 518, 502, 487, 511, 485, 23, 0, 0, 0, 0, 0, 0, 0, 471, 539, 503, 502, 506, 532, 485, 491, 5, 0, 0, 0, 0, 0, 0, 0, 514, 537, 507, 466, 535, 483, 508, 524, 0, 0, 0, 0, 0, 0, 0, 0, 505, 515, 506, 505, 526, 496, 503, 492, 10, 0, 0, 0, 0, 0, 0, 0, 534, 498, 530, 502, 525, 501, 494, 521, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4186, 4120, 4066, 4034, 4074, 4058, 4105] avg=4096.0000 max=4186 max_rank=1 min=4034 min_rank=4 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16271 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[490, 518, 498, 531, 509, 481, 493, 519, 1, 0, 0, 0, 0, 0, 0, 0, 521, 535, 505, 564, 491, 481, 462, 499, 0, 0, 0, 0, 0, 0, 0, 0, 512, 503, 485, 499, 520, 520, 504, 529, 0, 0, 0, 0, 0, 0, 0, 0, 525, 527, 490, 506, 536, 522, 509, 521, 25, 0, 0, 0, 0, 0, 0, 0, 504, 508, 508, 520, 491, 529, 507, 524, 5, 0, 0, 0, 0, 0, 0, 0, 529, 480, 504, 507, 513, 522, 541, 513, 0, 0, 0, 0, 0, 0, 0, 0, 535, 514, 515, 546, 493, 514, 526, 519, 10, 0, 0, 0, 0, 0, 0, 0, 500, 493, 492, 515, 521, 506, 510, 523, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4040, 4058, 4072, 4161, 4096, 4109, 4172, 4060] avg=4096.0000 max=4172 max_rank=6 min=4040 min_rank=0 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16437 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[516, 508, 537, 505, 501, 476, 489, 520, 1, 0, 0, 0, 0, 0, 0, 0, 487, 514, 517, 536, 525, 526, 482, 546, 0, 0, 0, 0, 0, 0, 0, 0, 519, 544, 527, 531, 524, 510, 515, 530, 0, 0, 0, 0, 0, 0, 0, 0, 472, 520, 498, 498, 486, 456, 550, 527, 25, 0, 0, 0, 0, 0, 0, 0, 500, 523, 506, 502, 510, 523, 517, 493, 6, 0, 0, 0, 0, 0, 0, 0, 515, 506, 509, 511, 503, 509, 545, 523, 0, 0, 0, 0, 0, 0, 0, 0, 494, 491, 483, 523, 484, 492, 516, 511, 11, 0, 0, 0, 0, 0, 0, 0, 511, 495, 546, 540, 475, 518, 537, 522, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4053, 4133, 4200, 4032, 4080, 4121, 4005, 4144] avg=4096.0000 max=4200 max_rank=2 min=4005 min_rank=6 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16350 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[506, 523, 549, 514, 524, 483, 549, 505, 1, 0, 0, 0, 0, 0, 0, 0, 531, 494, 503, 502, 514, 524, 508, 486, 0, 0, 0, 0, 0, 0, 0, 0, 505, 511, 527, 536, 512, 538, 491, 464, 0, 0, 0, 0, 0, 0, 0, 0, 494, 522, 540, 512, 557, 481, 508, 473, 24, 0, 0, 0, 0, 0, 0, 0, 495, 496, 531, 494, 521, 508, 546, 504, 5, 0, 0, 0, 0, 0, 0, 0, 519, 508, 536, 495, 507, 508, 528, 477, 0, 0, 0, 0, 0, 0, 0, 0, 505, 523, 492, 534, 540, 484, 513, 483, 10, 0, 0, 0, 0, 0, 0, 0, 518, 507, 554, 507, 493, 524, 510, 482, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4154, 4062, 4084, 4111, 4100, 4078, 4084, 4095] avg=4096.0000 max=4154 max_rank=0 min=4062 min_rank=1 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16357 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[503, 563, 499, 530, 531, 493, 505, 488, 4, 4, 0, 0, 0, 0, 0, 0, 574, 514, 516, 461, 526, 518, 526, 494, 0, 0, 0, 0, 0, 0, 0, 0, 557, 492, 500, 514, 493, 536, 523, 498, 0, 0, 0, 0, 0, 0, 0, 0, 505, 530, 504, 514, 504, 488, 499, 481, 28, 0, 0, 0, 0, 0, 0, 0, 529, 519, 522, 529, 484, 512, 523, 527, 10, 4, 0, 0, 0, 0, 0, 0, 470, 508, 520, 507, 521, 514, 474, 487, 0, 0, 0, 0, 0, 0, 0, 0, 511, 501, 501, 508, 543, 504, 532, 457, 14, 3, 0, 0, 0, 0, 0, 0, 530, 532, 521, 512, 494, 497, 518, 515, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4129, 4113, 4053, 4159, 4001, 4074, 4119] avg=4096.0000 max=4159 max_rank=4 min=4001 min_rank=5 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[538, 510, 530, 499, 572, 511, 537, 515, 1, 0, 0, 0, 0, 0, 0, 0, 462, 506, 529, 512, 546, 488, 522, 492, 0, 0, 0, 0, 0, 0, 0, 0, 528, 502, 519, 517, 510, 544, 484, 470, 0, 0, 0, 0, 0, 0, 0, 0, 529, 520, 521, 519, 535, 480, 476, 510, 25, 0, 0, 0, 0, 0, 0, 0, 482, 483, 538, 509, 493, 506, 517, 552, 6, 0, 0, 0, 0, 0, 0, 0, 504, 490, 497, 509, 501, 533, 529, 500, 0, 0, 0, 0, 0, 0, 0, 0, 530, 505, 535, 525, 509, 506, 501, 566, 11, 0, 0, 0, 0, 0, 0, 0, 512, 532, 476, 495, 510, 466, 496, 485, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4213, 4057, 4074, 4115, 4086, 4063, 4188, 3972] avg=4096.0000 max=4213 max_rank=0 min=3972 min_rank=7 imbalance_ratio=1.028564 max_load_reduction=28555 max_load_reduction_pct=0.871429 moved_to_echo=16309 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[489, 522, 505, 481, 479, 503, 546, 503, 1, 0, 0, 0, 0, 0, 0, 0, 509, 478, 489, 510, 515, 571, 512, 532, 0, 0, 0, 0, 0, 0, 0, 0, 500, 494, 512, 499, 516, 518, 515, 515, 0, 0, 0, 0, 0, 0, 0, 0, 503, 525, 479, 489, 536, 497, 518, 520, 25, 0, 0, 0, 0, 0, 0, 0, 496, 484, 517, 532, 497, 514, 516, 516, 5, 0, 0, 0, 0, 0, 0, 0, 514, 486, 527, 514, 490, 525, 544, 531, 0, 0, 0, 0, 0, 0, 0, 0, 470, 513, 560, 538, 493, 511, 485, 520, 10, 0, 0, 0, 0, 0, 0, 0, 532, 525, 491, 521, 485, 548, 519, 533, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4029, 4116, 4069, 4092, 4077, 4131, 4100, 4154] avg=4096.0000 max=4154 max_rank=7 min=4029 min_rank=0 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16462 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4352, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 3968, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[480, 469, 502, 493, 515, 505, 485, 483, 482, 518, 532, 521, 549, 470, 513, 515, 512, 528, 493, 521, 513, 503, 498, 531, 505, 520, 521, 520, 522, 529, 561, 459, 518, 529, 520, 493, 516, 531, 479, 512, 508, 524, 531, 493, 556, 516, 540, 523, 510, 516, 526, 487, 525, 502, 502, 496, 485, 506, 538, 521, 507, 530, 543, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[490, 484, 483, 529, 516, 514, 519, 524, 486, 481, 516, 542, 523, 513, 488, 504, 503, 485, 475, 550, 526, 521, 515, 509, 490, 538, 501, 464, 523, 527, 515, 522, 482, 532, 529, 517, 540, 514, 508, 491, 523, 523, 515, 516, 527, 498, 534, 500, 535, 520, 522, 511, 513, 517, 528, 527, 507, 508, 516, 537, 483, 505, 505, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[500, 537, 508, 579, 520, 452, 522, 488, 511, 494, 532, 537, 510, 522, 512, 504, 547, 510, 493, 523, 502, 538, 531, 502, 514, 506, 468, 530, 536, 496, 514, 519, 499, 509, 512, 480, 528, 544, 526, 502, 496, 522, 514, 520, 506, 502, 523, 497, 511, 502, 503, 531, 495, 529, 521, 510, 522, 480, 510, 485, 523, 509, 488, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[528, 533, 512, 518, 512, 491, 494, 479, 503, 544, 502, 553, 522, 523, 479, 538, 520, 520, 504, 520, 508, 492, 527, 497, 520, 508, 513, 527, 527, 499, 557, 519, 464, 506, 477, 529, 524, 522, 535, 510, 516, 510, 526, 524, 498, 541, 521, 489, 482, 532, 496, 484, 532, 532, 523, 462, 513, 468, 518, 524, 494, 516, 500, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[489, 499, 532, 478, 515, 496, 533, 509, 517, 528, 531, 488, 515, 506, 540, 527, 506, 517, 504, 522, 494, 475, 520, 514, 520, 518, 516, 528, 507, 490, 482, 520, 531, 512, 503, 484, 528, 531, 482, 500, 520, 516, 522, 534, 542, 510, 501, 506, 518, 511, 542, 501, 537, 479, 489, 520, 499, 510, 527, 548, 487, 496, 526, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[503, 532, 531, 531, 502, 514, 527, 518, 519, 518, 531, 517, 502, 542, 494, 493, 502, 501, 524, 517, 541, 505, 509, 500, 465, 475, 535, 500, 517, 519, 540, 507, 520, 518, 542, 474, 518, 531, 504, 526, 504, 530, 475, 535, 516, 497, 491, 465, 515, 520, 496, 573, 512, 496, 522, 524, 501, 502, 476, 527, 537, 487, 521, 482] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[537, 497, 485, 573, 543, 538, 519, 506, 541, 539, 521, 511, 529, 549, 505, 501, 522, 471, 526, 549, 507, 519, 550, 551, 501, 493, 525, 459, 542, 483, 504, 499, 526, 497, 512, 524, 494, 485, 514, 474, 522, 519, 513, 462, 490, 531, 483, 494, 491, 498, 541, 480, 498, 516, 515, 505, 468, 538, 530, 509, 514, 494, 532, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[503, 510, 476, 489, 526, 489, 513, 500, 515, 525, 520, 494, 502, 539, 502, 505, 505, 458, 492, 528, 515, 518, 522, 511, 535, 538, 530, 519, 511, 477, 523, 499, 600, 511, 514, 523, 480, 502, 486, 549, 539, 498, 515, 521, 529, 508, 494, 524, 490, 494, 520, 534, 480, 526, 496, 479, 499, 562, 531, 486, 548, 519, 520, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[480, 469, 502, 493, 515, 505, 485, 483, 24, 0, 0, 0, 0, 0, 0, 0, 482, 518, 503, 521, 549, 470, 513, 515, 0, 0, 0, 0, 0, 0, 0, 0, 512, 528, 493, 517, 513, 503, 498, 531, 0, 0, 0, 0, 0, 0, 0, 0, 505, 520, 521, 520, 522, 529, 561, 459, 1, 0, 0, 0, 0, 0, 0, 0, 518, 529, 520, 493, 516, 531, 479, 512, 0, 0, 0, 0, 0, 0, 0, 0, 508, 524, 531, 493, 547, 516, 540, 523, 0, 0, 0, 0, 0, 0, 0, 0, 510, 516, 526, 487, 525, 502, 502, 496, 5, 3, 0, 0, 0, 0, 0, 0, 485, 506, 538, 521, 507, 530, 543, 517, 5, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3956, 4071, 4095, 4138, 4098, 4182, 4072, 4156] avg=4096.0000 max=4182 max_rank=5 min=3956 min_rank=0 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16508 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[500, 537, 508, 579, 520, 452, 522, 488, 24, 0, 0, 0, 0, 0, 0, 0, 511, 494, 503, 537, 510, 522, 512, 504, 0, 0, 0, 0, 0, 0, 0, 0, 547, 510, 493, 519, 502, 538, 531, 502, 0, 0, 0, 0, 0, 0, 0, 0, 514, 506, 468, 530, 536, 496, 514, 519, 1, 0, 0, 0, 0, 0, 0, 0, 499, 509, 512, 480, 528, 544, 526, 502, 0, 0, 0, 0, 0, 0, 0, 0, 496, 522, 514, 520, 499, 502, 523, 497, 0, 0, 0, 0, 0, 0, 0, 0, 511, 502, 503, 531, 495, 529, 521, 510, 4, 3, 0, 0, 0, 0, 0, 0, 522, 480, 510, 485, 523, 509, 488, 512, 5, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4130, 4093, 4142, 4084, 4100, 4073, 4109, 4037] avg=4096.0000 max=4142 max_rank=2 min=4037 min_rank=7 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16319 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[528, 533, 512, 518, 512, 491, 494, 479, 22, 0, 0, 0, 0, 0, 0, 0, 503, 544, 475, 553, 522, 523, 479, 538, 0, 0, 0, 0, 0, 0, 0, 0, 520, 520, 504, 516, 508, 492, 527, 497, 0, 0, 0, 0, 0, 0, 0, 0, 520, 508, 513, 527, 527, 499, 557, 519, 1, 0, 0, 0, 0, 0, 0, 0, 464, 506, 477, 529, 524, 522, 535, 510, 0, 0, 0, 0, 0, 0, 0, 0, 516, 510, 526, 524, 491, 541, 521, 489, 0, 0, 0, 0, 0, 0, 0, 0, 482, 532, 496, 484, 532, 532, 523, 462, 4, 3, 0, 0, 0, 0, 0, 0, 513, 468, 518, 524, 494, 516, 500, 511, 5, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4137, 4084, 4171, 4067, 4118, 4050, 4052] avg=4096.0000 max=4171 max_rank=3 min=4050 min_rank=6 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16287 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[489, 499, 532, 478, 515, 496, 533, 509, 24, 0, 0, 0, 0, 0, 0, 0, 517, 528, 502, 488, 515, 506, 540, 527, 0, 0, 0, 0, 0, 0, 0, 0, 506, 517, 504, 518, 494, 475, 520, 514, 0, 0, 0, 0, 0, 0, 0, 0, 520, 518, 516, 528, 507, 490, 482, 520, 1, 0, 0, 0, 0, 0, 0, 0, 531, 512, 503, 484, 528, 531, 482, 500, 0, 0, 0, 0, 0, 0, 0, 0, 520, 516, 522, 534, 534, 510, 501, 506, 0, 0, 0, 0, 0, 0, 0, 0, 518, 511, 542, 501, 537, 479, 489, 520, 5, 3, 0, 0, 0, 0, 0, 0, 499, 510, 527, 548, 487, 496, 526, 520, 5, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4123, 4048, 4082, 4071, 4143, 4105, 4121] avg=4096.0000 max=4143 max_rank=5 min=4048 min_rank=2 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16440 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[490, 484, 483, 529, 516, 514, 519, 524, 23, 0, 0, 0, 0, 0, 0, 0, 486, 481, 488, 542, 523, 513, 488, 504, 0, 0, 0, 0, 0, 0, 0, 0, 503, 485, 475, 545, 526, 521, 515, 509, 0, 0, 0, 0, 0, 0, 0, 0, 490, 538, 501, 464, 523, 527, 515, 522, 2, 0, 0, 0, 0, 0, 0, 0, 482, 532, 529, 517, 540, 514, 508, 491, 0, 0, 0, 0, 0, 0, 0, 0, 523, 523, 515, 516, 519, 498, 534, 500, 0, 0, 0, 0, 0, 0, 0, 0, 535, 520, 522, 511, 513, 517, 528, 527, 5, 3, 0, 0, 0, 0, 0, 0, 507, 508, 516, 537, 483, 505, 505, 509, 5, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4025, 4079, 4082, 4113, 4128, 4181, 4078] avg=4096.0000 max=4181 max_rank=6 min=4025 min_rank=1 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16500 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[503, 532, 531, 531, 502, 514, 527, 518, 28, 0, 0, 0, 0, 0, 0, 0, 519, 518, 492, 517, 502, 542, 494, 493, 0, 0, 0, 0, 0, 0, 0, 0, 502, 501, 524, 503, 541, 505, 509, 500, 0, 0, 0, 0, 0, 0, 0, 0, 465, 475, 535, 500, 517, 519, 540, 507, 7, 5, 0, 0, 0, 0, 0, 0, 520, 518, 542, 474, 518, 526, 504, 526, 0, 0, 0, 0, 0, 0, 0, 0, 504, 530, 475, 535, 500, 497, 491, 465, 0, 0, 0, 0, 0, 0, 0, 0, 515, 520, 496, 573, 512, 496, 522, 524, 8, 7, 0, 0, 0, 0, 0, 0, 501, 502, 476, 527, 537, 487, 521, 482, 11, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4186, 4077, 4085, 4070, 4128, 3997, 4173, 4052] avg=4096.0000 max=4186 max_rank=0 min=3997 min_rank=5 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16350 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[537, 497, 485, 573, 543, 538, 519, 506, 23, 0, 0, 0, 0, 0, 0, 0, 541, 539, 493, 511, 529, 549, 505, 501, 0, 0, 0, 0, 0, 0, 0, 0, 522, 471, 526, 544, 507, 519, 550, 551, 0, 0, 0, 0, 0, 0, 0, 0, 501, 493, 525, 459, 542, 483, 504, 499, 2, 0, 0, 0, 0, 0, 0, 0, 526, 497, 512, 524, 494, 485, 514, 474, 0, 0, 0, 0, 0, 0, 0, 0, 522, 519, 513, 462, 483, 531, 483, 494, 0, 0, 0, 0, 0, 0, 0, 0, 491, 498, 541, 480, 498, 516, 515, 505, 4, 3, 0, 0, 0, 0, 0, 0, 468, 538, 530, 509, 514, 494, 532, 504, 5, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4221, 4168, 4190, 4008, 4026, 4007, 4051, 4097] avg=4096.0000 max=4221 max_rank=0 min=4007 min_rank=5 imbalance_ratio=1.030518 max_load_reduction=28547 max_load_reduction_pct=0.871185 moved_to_echo=16181 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[503, 510, 476, 489, 526, 489, 513, 500, 23, 0, 0, 0, 0, 0, 0, 0, 515, 525, 492, 494, 502, 539, 502, 505, 0, 0, 0, 0, 0, 0, 0, 0, 505, 458, 492, 524, 515, 518, 522, 511, 0, 0, 0, 0, 0, 0, 0, 0, 535, 538, 530, 519, 511, 477, 523, 499, 1, 0, 0, 0, 0, 0, 0, 0, 600, 511, 514, 523, 480, 502, 486, 549, 0, 0, 0, 0, 0, 0, 0, 0, 539, 498, 515, 521, 521, 508, 494, 524, 0, 0, 0, 0, 0, 0, 0, 0, 490, 494, 520, 534, 480, 526, 496, 479, 5, 3, 0, 0, 0, 0, 0, 0, 499, 562, 531, 486, 548, 519, 520, 502, 5, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4029, 4074, 4045, 4133, 4165, 4120, 4027, 4175] avg=4096.0000 max=4175 max_rank=7 min=4027 min_rank=6 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16487 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 3968, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[527, 514, 528, 513, 504, 555, 517, 480, 524, 522, 511, 500, 512, 519, 514, 547, 535, 484, 526, 486, 494, 504, 494, 511, 522, 513, 530, 487, 502, 498, 503, 542, 564, 499, 488, 533, 510, 498, 494, 488, 508, 544, 537, 489, 519, 503, 511, 532, 511, 484, 471, 510, 508, 515, 496, 508, 516, 529, 484, 526, 515, 516, 560, 484] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[501, 494, 532, 516, 534, 518, 553, 485, 547, 527, 499, 523, 558, 502, 497, 517, 535, 517, 515, 514, 579, 521, 492, 497, 513, 490, 502, 552, 507, 506, 492, 524, 516, 530, 484, 484, 496, 475, 530, 494, 482, 517, 502, 508, 519, 508, 494, 486, 529, 507, 488, 505, 474, 521, 501, 532, 509, 526, 519, 521, 500, 523, 490, 539] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[523, 529, 507, 557, 528, 508, 492, 529, 483, 544, 523, 508, 493, 500, 513, 509, 511, 504, 494, 510, 501, 479, 505, 491, 483, 523, 562, 539, 524, 487, 527, 511, 493, 521, 505, 527, 554, 470, 547, 502, 517, 530, 491, 532, 532, 495, 504, 497, 508, 549, 538, 500, 541, 495, 506, 522, 486, 525, 463, 467, 516, 535, 510, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[486, 534, 536, 487, 535, 535, 512, 468, 484, 512, 552, 519, 497, 512, 517, 509, 526, 528, 530, 496, 459, 506, 515, 516, 501, 531, 511, 518, 494, 549, 508, 495, 543, 512, 493, 499, 548, 512, 508, 495, 521, 505, 517, 517, 477, 518, 515, 491, 553, 491, 510, 514, 492, 536, 521, 493, 497, 525, 517, 555, 548, 484, 478, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[523, 504, 500, 535, 538, 495, 502, 510, 516, 517, 480, 482, 537, 500, 511, 511, 486, 507, 539, 507, 522, 562, 512, 562, 507, 470, 505, 486, 483, 523, 484, 528, 541, 537, 476, 524, 530, 482, 516, 525, 489, 561, 524, 506, 508, 513, 513, 517, 495, 503, 516, 526, 534, 526, 480, 495, 521, 520, 483, 531, 515, 516, 505, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[486, 514, 545, 521, 511, 488, 519, 497, 549, 493, 498, 519, 498, 545, 538, 576, 525, 557, 499, 520, 501, 490, 533, 499, 491, 478, 486, 491, 514, 540, 511, 514, 515, 526, 495, 536, 514, 537, 509, 494, 525, 513, 517, 483, 501, 487, 528, 518, 499, 539, 505, 551, 487, 531, 458, 486, 506, 523, 513, 517, 519, 488, 495, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[509, 523, 535, 523, 514, 519, 514, 533, 497, 523, 522, 473, 473, 522, 524, 520, 481, 504, 514, 498, 497, 453, 493, 522, 449, 509, 478, 532, 498, 545, 526, 505, 528, 553, 510, 524, 536, 498, 482, 523, 523, 517, 519, 515, 496, 522, 572, 526, 533, 495, 483, 535, 505, 564, 512, 483, 510, 514, 549, 495, 500, 546, 470, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[577, 532, 503, 507, 524, 532, 522, 495, 518, 501, 473, 508, 473, 539, 533, 464, 529, 512, 542, 491, 506, 522, 524, 531, 531, 537, 553, 502, 514, 507, 509, 511, 542, 508, 514, 513, 501, 507, 531, 511, 512, 508, 507, 511, 480, 516, 515, 495, 523, 482, 532, 514, 506, 472, 500, 536, 501, 542, 504, 484, 523, 479, 467, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[527, 514, 528, 513, 467, 555, 517, 480, 0, 0, 0, 0, 0, 0, 0, 0, 524, 522, 511, 500, 512, 519, 514, 540, 0, 0, 0, 0, 0, 0, 0, 0, 535, 484, 526, 486, 494, 504, 494, 511, 6, 0, 0, 0, 0, 0, 0, 0, 522, 513, 530, 487, 502, 498, 503, 542, 9, 3, 0, 0, 0, 0, 0, 0, 555, 499, 488, 533, 510, 498, 494, 488, 0, 0, 0, 0, 0, 0, 0, 0, 508, 544, 537, 489, 519, 503, 511, 532, 1, 0, 0, 0, 0, 0, 0, 0, 511, 484, 471, 510, 508, 515, 496, 508, 16, 0, 0, 0, 0, 0, 0, 0, 516, 529, 484, 526, 515, 516, 560, 484, 18, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4142, 4040, 4109, 4065, 4144, 4019, 4148] avg=4096.0000 max=4148 max_rank=7 min=4019 min_rank=6 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16376 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[523, 504, 500, 535, 498, 495, 502, 510, 0, 0, 0, 0, 0, 0, 0, 0, 516, 517, 480, 482, 537, 500, 511, 505, 0, 0, 0, 0, 0, 0, 0, 0, 486, 507, 539, 507, 522, 562, 512, 562, 5, 0, 0, 0, 0, 0, 0, 0, 507, 470, 505, 486, 483, 523, 484, 528, 9, 3, 0, 0, 0, 0, 0, 0, 532, 537, 476, 524, 530, 482, 516, 525, 0, 0, 0, 0, 0, 0, 0, 0, 489, 561, 524, 506, 508, 513, 513, 517, 1, 0, 0, 0, 0, 0, 0, 0, 495, 503, 516, 526, 534, 526, 480, 495, 17, 0, 0, 0, 0, 0, 0, 0, 521, 520, 483, 531, 515, 516, 505, 496, 20, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4048, 4202, 3998, 4122, 4132, 4092, 4107] avg=4096.0000 max=4202 max_rank=2 min=3998 min_rank=3 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16453 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[501, 494, 532, 516, 496, 518, 553, 485, 0, 0, 0, 0, 0, 0, 0, 0, 547, 527, 499, 523, 558, 502, 497, 511, 0, 0, 0, 0, 0, 0, 0, 0, 535, 517, 515, 514, 579, 521, 492, 497, 5, 0, 0, 0, 0, 0, 0, 0, 513, 490, 502, 552, 507, 506, 492, 524, 9, 3, 0, 0, 0, 0, 0, 0, 507, 530, 484, 484, 496, 475, 530, 494, 0, 0, 0, 0, 0, 0, 0, 0, 482, 517, 502, 508, 519, 508, 494, 486, 1, 0, 0, 0, 0, 0, 0, 0, 529, 507, 488, 505, 474, 521, 501, 532, 16, 0, 0, 0, 0, 0, 0, 0, 509, 526, 519, 521, 500, 523, 490, 539, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4164, 4175, 4098, 4000, 4017, 4073, 4146] avg=4096.0000 max=4175 max_rank=2 min=4000 min_rank=4 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16236 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[486, 534, 536, 487, 497, 535, 512, 468, 0, 0, 0, 0, 0, 0, 0, 0, 484, 512, 552, 519, 497, 512, 517, 503, 0, 0, 0, 0, 0, 0, 0, 0, 526, 528, 530, 496, 459, 506, 515, 516, 5, 0, 0, 0, 0, 0, 0, 0, 501, 531, 511, 518, 494, 549, 508, 495, 9, 3, 0, 0, 0, 0, 0, 0, 534, 512, 493, 499, 548, 512, 508, 495, 0, 0, 0, 0, 0, 0, 0, 0, 521, 505, 517, 517, 477, 518, 515, 491, 1, 0, 0, 0, 0, 0, 0, 0, 553, 491, 510, 514, 492, 536, 521, 493, 16, 0, 0, 0, 0, 0, 0, 0, 497, 525, 517, 555, 548, 484, 478, 505, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4055, 4096, 4081, 4119, 4101, 4062, 4126, 4128] avg=4096.0000 max=4128 max_rank=7 min=4055 min_rank=0 imbalance_ratio=1.007812 max_load_reduction=28640 max_load_reduction_pct=0.874023 moved_to_echo=16417 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[523, 529, 507, 557, 490, 508, 492, 529, 0, 0, 0, 0, 0, 0, 0, 0, 483, 544, 523, 508, 493, 500, 513, 503, 0, 0, 0, 0, 0, 0, 0, 0, 511, 504, 494, 510, 501, 479, 505, 491, 5, 0, 0, 0, 0, 0, 0, 0, 483, 523, 562, 539, 524, 487, 527, 511, 8, 3, 0, 0, 0, 0, 0, 0, 485, 521, 505, 527, 554, 470, 547, 502, 0, 0, 0, 0, 0, 0, 0, 0, 517, 530, 491, 532, 532, 495, 504, 497, 1, 0, 0, 0, 0, 0, 0, 0, 508, 549, 538, 500, 541, 495, 506, 522, 16, 0, 0, 0, 0, 0, 0, 0, 486, 525, 463, 467, 516, 535, 510, 493, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4135, 4067, 4000, 4167, 4111, 4099, 4175, 4014] avg=4096.0000 max=4175 max_rank=6 min=4000 min_rank=2 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[577, 532, 503, 507, 486, 532, 522, 495, 0, 0, 0, 0, 0, 0, 0, 0, 518, 501, 473, 508, 473, 539, 533, 458, 0, 0, 0, 0, 0, 0, 0, 0, 529, 512, 542, 491, 506, 522, 524, 531, 5, 0, 0, 0, 0, 0, 0, 0, 531, 537, 553, 502, 514, 507, 509, 511, 9, 3, 0, 0, 0, 0, 0, 0, 533, 508, 514, 513, 501, 507, 531, 511, 0, 0, 0, 0, 0, 0, 0, 0, 512, 508, 507, 511, 480, 516, 515, 495, 1, 0, 0, 0, 0, 0, 0, 0, 523, 482, 532, 514, 506, 472, 500, 536, 16, 0, 0, 0, 0, 0, 0, 0, 501, 542, 504, 484, 523, 479, 467, 510, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4154, 4003, 4162, 4176, 4118, 4045, 4081, 4029] avg=4096.0000 max=4176 max_rank=3 min=4003 min_rank=1 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16273 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[509, 523, 535, 523, 462, 519, 514, 533, 0, 0, 0, 0, 0, 0, 0, 0, 497, 523, 522, 473, 473, 522, 524, 505, 0, 0, 0, 0, 0, 0, 0, 0, 481, 504, 514, 498, 497, 453, 493, 522, 9, 7, 0, 0, 0, 0, 0, 0, 449, 509, 478, 532, 498, 545, 526, 505, 13, 9, 0, 0, 0, 0, 0, 0, 508, 553, 510, 524, 536, 498, 482, 523, 0, 0, 0, 0, 0, 0, 0, 0, 523, 517, 519, 515, 496, 522, 572, 526, 6, 0, 0, 0, 0, 0, 0, 0, 533, 495, 483, 535, 505, 564, 512, 483, 20, 0, 0, 0, 0, 0, 0, 0, 510, 514, 549, 495, 500, 546, 470, 502, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4039, 3978, 4064, 4134, 4196, 4130, 4109] avg=4096.0000 max=4196 max_rank=5 min=3978 min_rank=2 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16569 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[486, 514, 545, 521, 473, 488, 519, 497, 0, 0, 0, 0, 0, 0, 0, 0, 549, 493, 498, 519, 498, 545, 538, 569, 0, 0, 0, 0, 0, 0, 0, 0, 525, 557, 499, 520, 501, 490, 533, 499, 6, 0, 0, 0, 0, 0, 0, 0, 491, 478, 486, 491, 514, 540, 511, 514, 9, 3, 0, 0, 0, 0, 0, 0, 506, 526, 495, 536, 514, 537, 509, 494, 0, 0, 0, 0, 0, 0, 0, 0, 525, 513, 517, 483, 501, 487, 528, 518, 1, 0, 0, 0, 0, 0, 0, 0, 499, 539, 505, 551, 487, 531, 458, 486, 16, 0, 0, 0, 0, 0, 0, 0, 506, 523, 513, 517, 519, 488, 495, 507, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4043, 4209, 4130, 4037, 4117, 4073, 4072, 4087] avg=4096.0000 max=4209 max_rank=1 min=4037 min_rank=3 imbalance_ratio=1.027588 max_load_reduction=28559 max_load_reduction_pct=0.871552 moved_to_echo=16349 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4224, 3968, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 3968, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[511, 515, 504, 500, 509, 513, 515, 535, 494, 527, 509, 508, 517, 503, 545, 516, 541, 488, 501, 488, 477, 524, 508, 528, 546, 565, 522, 543, 502, 491, 491, 518, 520, 532, 505, 509, 484, 494, 506, 514, 507, 487, 500, 484, 490, 561, 524, 524, 468, 503, 508, 537, 518, 503, 504, 469, 557, 503, 535, 541, 510, 524, 501, 492] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[511, 503, 480, 556, 509, 535, 531, 513, 516, 473, 509, 467, 495, 536, 516, 515, 487, 523, 545, 551, 536, 544, 508, 509, 522, 514, 477, 514, 482, 500, 536, 508, 505, 521, 509, 504, 469, 524, 500, 516, 482, 505, 545, 497, 514, 532, 533, 463, 535, 515, 511, 505, 530, 509, 513, 468, 502, 513, 463, 541, 508, 543, 565, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[517, 537, 486, 518, 526, 499, 495, 485, 543, 509, 503, 499, 487, 490, 525, 537, 501, 532, 548, 486, 527, 508, 499, 534, 514, 512, 514, 469, 528, 486, 522, 509, 509, 518, 508, 495, 525, 509, 512, 521, 510, 490, 507, 542, 486, 526, 512, 495, 485, 494, 493, 516, 525, 510, 549, 520, 472, 508, 530, 527, 522, 527, 537, 543] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[557, 467, 513, 516, 551, 506, 529, 499, 500, 541, 517, 534, 524, 533, 458, 494, 494, 548, 519, 511, 499, 535, 512, 491, 493, 542, 500, 501, 523, 509, 470, 526, 487, 549, 560, 519, 536, 479, 502, 510, 534, 524, 510, 501, 470, 493, 512, 527, 494, 543, 469, 508, 513, 510, 505, 526, 529, 513, 507, 465, 535, 515, 507, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[471, 516, 502, 528, 498, 488, 505, 502, 509, 539, 547, 546, 500, 539, 522, 523, 498, 475, 510, 521, 530, 494, 516, 527, 488, 507, 477, 519, 510, 516, 516, 512, 517, 533, 534, 522, 496, 524, 459, 538, 509, 506, 495, 531, 511, 527, 451, 520, 550, 509, 496, 508, 490, 517, 564, 478, 556, 546, 478, 497, 521, 535, 505, 494] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[486, 510, 512, 514, 494, 512, 540, 508, 465, 519, 494, 480, 487, 522, 484, 540, 515, 504, 528, 530, 489, 525, 518, 503, 522, 536, 493, 536, 514, 564, 495, 499, 488, 546, 493, 542, 518, 494, 508, 515, 510, 531, 527, 497, 505, 489, 536, 497, 529, 511, 507, 507, 529, 493, 494, 527, 494, 488, 521, 517, 496, 517, 578, 526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[502, 534, 506, 514, 536, 506, 497, 506, 496, 492, 499, 500, 542, 518, 509, 508, 531, 570, 512, 512, 492, 541, 505, 529, 485, 448, 544, 537, 536, 510, 522, 515, 503, 507, 498, 508, 523, 470, 481, 520, 531, 508, 507, 542, 492, 498, 538, 517, 507, 513, 517, 526, 496, 518, 501, 507, 511, 533, 523, 520, 502, 505, 521, 471] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[567, 525, 496, 543, 504, 487, 498, 491, 504, 547, 524, 526, 533, 523, 527, 486, 500, 504, 519, 526, 532, 486, 502, 496, 505, 513, 497, 488, 511, 513, 512, 502, 509, 519, 531, 519, 515, 527, 491, 510, 503, 512, 503, 517, 505, 509, 525, 513, 518, 501, 513, 520, 462, 507, 530, 495, 538, 530, 505, 497, 511, 496, 521, 529] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[511, 503, 480, 556, 509, 535, 531, 513, 2, 0, 0, 0, 0, 0, 0, 0, 516, 473, 509, 467, 495, 534, 516, 515, 0, 0, 0, 0, 0, 0, 0, 0, 487, 523, 524, 551, 536, 544, 508, 509, 0, 0, 0, 0, 0, 0, 0, 0, 522, 514, 477, 514, 482, 500, 536, 508, 10, 0, 0, 0, 0, 0, 0, 0, 505, 521, 509, 504, 469, 524, 500, 516, 7, 0, 0, 0, 0, 0, 0, 0, 482, 505, 545, 497, 514, 532, 533, 463, 11, 4, 0, 0, 0, 0, 0, 0, 535, 515, 511, 505, 530, 509, 513, 468, 22, 0, 0, 0, 0, 0, 0, 0, 502, 513, 463, 541, 508, 543, 532, 507, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4140, 4025, 4182, 4063, 4055, 4086, 4108, 4109] avg=4096.0000 max=4182 max_rank=2 min=4025 min_rank=1 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16358 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[471, 516, 502, 528, 498, 488, 505, 502, 2, 0, 0, 0, 0, 0, 0, 0, 509, 539, 547, 546, 500, 537, 522, 523, 0, 0, 0, 0, 0, 0, 0, 0, 498, 475, 491, 521, 530, 494, 516, 527, 0, 0, 0, 0, 0, 0, 0, 0, 488, 507, 477, 519, 510, 516, 516, 512, 9, 0, 0, 0, 0, 0, 0, 0, 517, 533, 534, 522, 496, 524, 459, 538, 7, 0, 0, 0, 0, 0, 0, 0, 509, 506, 495, 531, 511, 527, 451, 520, 10, 3, 0, 0, 0, 0, 0, 0, 550, 509, 496, 508, 490, 517, 564, 478, 20, 0, 0, 0, 0, 0, 0, 0, 556, 546, 478, 497, 521, 535, 475, 494, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4012, 4223, 4052, 4054, 4130, 4063, 4132, 4102] avg=4096.0000 max=4223 max_rank=1 min=4012 min_rank=0 imbalance_ratio=1.031006 max_load_reduction=28545 max_load_reduction_pct=0.871124 moved_to_echo=16427 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[557, 467, 513, 516, 551, 506, 529, 499, 2, 0, 0, 0, 0, 0, 0, 0, 500, 541, 517, 534, 524, 531, 458, 494, 0, 0, 0, 0, 0, 0, 0, 0, 494, 548, 500, 511, 499, 535, 512, 491, 0, 0, 0, 0, 0, 0, 0, 0, 493, 542, 500, 501, 523, 509, 470, 526, 9, 0, 0, 0, 0, 0, 0, 0, 487, 549, 560, 519, 536, 479, 502, 510, 7, 0, 0, 0, 0, 0, 0, 0, 534, 524, 510, 501, 470, 493, 512, 527, 10, 3, 0, 0, 0, 0, 0, 0, 494, 543, 469, 508, 513, 510, 505, 526, 20, 0, 0, 0, 0, 0, 0, 0, 529, 513, 507, 465, 535, 515, 477, 504, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4140, 4099, 4090, 4073, 4149, 4084, 4088, 4045] avg=4096.0000 max=4149 max_rank=4 min=4045 min_rank=7 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[511, 515, 504, 500, 509, 513, 515, 535, 8, 7, 0, 0, 0, 0, 0, 0, 494, 527, 509, 508, 517, 495, 545, 516, 0, 0, 0, 0, 0, 0, 0, 0, 541, 488, 464, 488, 477, 524, 508, 528, 0, 0, 0, 0, 0, 0, 0, 0, 546, 565, 522, 543, 502, 491, 491, 518, 12, 0, 0, 0, 0, 0, 0, 0, 520, 532, 505, 509, 484, 494, 506, 514, 10, 0, 0, 0, 0, 0, 0, 0, 507, 487, 500, 484, 490, 561, 524, 524, 14, 8, 0, 0, 0, 0, 0, 0, 468, 503, 508, 537, 518, 503, 504, 469, 24, 0, 0, 0, 0, 0, 0, 0, 557, 503, 535, 541, 510, 524, 463, 492, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4117, 4111, 4018, 4190, 4074, 4099, 4034, 4125] avg=4096.0000 max=4190 max_rank=3 min=4018 min_rank=2 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16332 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[517, 537, 486, 518, 526, 499, 495, 485, 2, 0, 0, 0, 0, 0, 0, 0, 543, 509, 503, 499, 487, 488, 525, 537, 0, 0, 0, 0, 0, 0, 0, 0, 501, 532, 527, 486, 527, 508, 499, 534, 0, 0, 0, 0, 0, 0, 0, 0, 514, 512, 514, 469, 528, 486, 522, 509, 10, 0, 0, 0, 0, 0, 0, 0, 509, 518, 508, 495, 525, 509, 512, 521, 7, 0, 0, 0, 0, 0, 0, 0, 510, 490, 507, 542, 486, 526, 512, 495, 11, 4, 0, 0, 0, 0, 0, 0, 485, 494, 493, 516, 525, 510, 549, 520, 21, 0, 0, 0, 0, 0, 0, 0, 472, 508, 530, 527, 522, 527, 505, 543, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4065, 4091, 4114, 4064, 4104, 4083, 4113, 4134] avg=4096.0000 max=4134 max_rank=7 min=4064 min_rank=3 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16434 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[486, 510, 512, 514, 494, 512, 540, 508, 2, 0, 0, 0, 0, 0, 0, 0, 465, 519, 494, 480, 487, 520, 484, 540, 0, 0, 0, 0, 0, 0, 0, 0, 515, 504, 509, 530, 489, 525, 518, 503, 0, 0, 0, 0, 0, 0, 0, 0, 522, 536, 493, 536, 514, 564, 495, 499, 9, 0, 0, 0, 0, 0, 0, 0, 488, 546, 493, 542, 518, 494, 508, 515, 7, 0, 0, 0, 0, 0, 0, 0, 510, 531, 527, 497, 505, 489, 536, 497, 12, 3, 0, 0, 0, 0, 0, 0, 529, 511, 507, 507, 529, 493, 494, 527, 23, 0, 0, 0, 0, 0, 0, 0, 494, 488, 521, 517, 496, 517, 543, 526, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 3989, 4093, 4168, 4111, 4107, 4120, 4102] avg=4096.0000 max=4168 max_rank=3 min=3989 min_rank=1 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16440 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[567, 525, 496, 543, 504, 487, 498, 491, 2, 0, 0, 0, 0, 0, 0, 0, 504, 547, 524, 526, 533, 521, 527, 486, 0, 0, 0, 0, 0, 0, 0, 0, 500, 504, 500, 526, 532, 486, 502, 496, 0, 0, 0, 0, 0, 0, 0, 0, 505, 513, 497, 488, 511, 513, 512, 502, 9, 0, 0, 0, 0, 0, 0, 0, 509, 519, 531, 519, 515, 527, 491, 510, 7, 0, 0, 0, 0, 0, 0, 0, 503, 512, 503, 517, 505, 509, 525, 513, 10, 3, 0, 0, 0, 0, 0, 0, 518, 501, 513, 520, 462, 507, 530, 495, 21, 0, 0, 0, 0, 0, 0, 0, 538, 530, 505, 497, 511, 496, 490, 529, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4168, 4046, 4050, 4128, 4100, 4067, 4096] avg=4096.0000 max=4168 max_rank=1 min=4046 min_rank=2 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[502, 534, 506, 514, 536, 506, 497, 506, 2, 0, 0, 0, 0, 0, 0, 0, 496, 492, 499, 500, 542, 516, 509, 508, 0, 0, 0, 0, 0, 0, 0, 0, 531, 570, 493, 512, 492, 541, 505, 529, 0, 0, 0, 0, 0, 0, 0, 0, 485, 448, 544, 537, 536, 510, 522, 515, 9, 0, 0, 0, 0, 0, 0, 0, 503, 507, 498, 508, 523, 470, 481, 520, 7, 0, 0, 0, 0, 0, 0, 0, 531, 508, 507, 542, 492, 498, 538, 517, 10, 3, 0, 0, 0, 0, 0, 0, 507, 513, 517, 526, 496, 518, 501, 507, 21, 0, 0, 0, 0, 0, 0, 0, 511, 533, 523, 520, 502, 505, 490, 471, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4062, 4173, 4106, 4017, 4146, 4106, 4055] avg=4096.0000 max=4173 max_rank=2 min=4017 min_rank=4 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16324 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4352, 4224, 4224, 4096, 4096, 3968, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[504, 481, 506, 491, 449, 521, 450, 482, 545, 505, 514, 523, 503, 514, 536, 511, 495, 527, 523, 533, 484, 542, 505, 501, 494, 552, 497, 555, 494, 507, 502, 502, 504, 501, 528, 553, 506, 496, 482, 488, 498, 516, 523, 521, 529, 517, 513, 490, 584, 516, 559, 521, 499, 468, 478, 527, 520, 544, 557, 517, 548, 546, 483, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[483, 503, 519, 500, 533, 528, 533, 523, 492, 503, 510, 519, 501, 518, 502, 515, 451, 503, 529, 485, 500, 531, 498, 516, 520, 532, 485, 471, 522, 521, 522, 462, 506, 518, 524, 523, 535, 493, 513, 466, 528, 484, 528, 534, 523, 509, 504, 526, 513, 506, 520, 550, 508, 551, 519, 511, 564, 495, 522, 490, 534, 514, 513, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[527, 473, 515, 498, 490, 547, 536, 550, 501, 484, 526, 511, 516, 541, 513, 484, 441, 499, 512, 523, 544, 536, 536, 541, 510, 527, 515, 529, 522, 503, 519, 513, 560, 506, 517, 535, 472, 514, 520, 509, 504, 467, 533, 477, 522, 484, 471, 487, 493, 524, 488, 502, 543, 496, 511, 508, 501, 530, 498, 533, 548, 502, 525, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[541, 525, 481, 533, 501, 511, 495, 520, 555, 518, 502, 503, 512, 481, 505, 528, 506, 505, 493, 512, 483, 508, 533, 522, 517, 502, 537, 514, 507, 548, 497, 514, 495, 500, 490, 523, 503, 521, 516, 527, 533, 522, 533, 498, 522, 517, 522, 502, 503, 523, 482, 519, 473, 519, 534, 490, 517, 518, 510, 537, 504, 482, 498, 526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[484, 502, 488, 510, 513, 489, 522, 549, 540, 526, 528, 507, 531, 495, 531, 507, 529, 504, 492, 495, 520, 517, 503, 546, 504, 565, 483, 507, 503, 534, 521, 506, 515, 500, 528, 514, 498, 535, 493, 492, 529, 493, 536, 497, 532, 511, 453, 543, 530, 518, 478, 507, 502, 478, 529, 501, 516, 528, 506, 503, 516, 490, 523, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[485, 496, 466, 504, 496, 503, 522, 536, 522, 534, 508, 506, 517, 506, 485, 507, 491, 529, 534, 510, 548, 540, 519, 457, 514, 513, 495, 511, 530, 548, 515, 493, 500, 538, 470, 516, 481, 547, 516, 523, 482, 531, 541, 522, 558, 477, 515, 515, 494, 508, 517, 518, 547, 531, 489, 522, 520, 485, 526, 497, 536, 497, 491, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[479, 458, 511, 516, 526, 488, 537, 496, 500, 513, 536, 503, 548, 516, 522, 534, 508, 530, 537, 570, 505, 513, 501, 505, 500, 493, 484, 487, 490, 511, 514, 500, 510, 501, 504, 522, 511, 497, 546, 534, 544, 503, 512, 515, 542, 498, 510, 520, 473, 504, 501, 519, 509, 524, 515, 568, 509, 508, 480, 490, 518, 565, 480, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[527, 549, 513, 522, 524, 522, 510, 474, 489, 512, 540, 557, 521, 545, 529, 511, 500, 497, 502, 539, 520, 505, 507, 485, 518, 542, 527, 482, 471, 524, 507, 496, 525, 501, 501, 516, 548, 501, 453, 543, 496, 479, 508, 488, 526, 512, 517, 522, 519, 512, 482, 500, 486, 524, 552, 500, 541, 510, 545, 508, 507, 512, 472, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[504, 481, 506, 491, 449, 521, 450, 482, 30, 6, 0, 0, 0, 0, 0, 0, 545, 505, 484, 523, 503, 514, 536, 511, 0, 0, 0, 0, 0, 0, 0, 0, 495, 527, 523, 533, 484, 542, 505, 501, 0, 0, 0, 0, 0, 0, 0, 0, 494, 552, 497, 555, 494, 507, 502, 502, 8, 0, 0, 0, 0, 0, 0, 0, 504, 501, 528, 553, 506, 496, 482, 488, 3, 1, 0, 0, 0, 0, 0, 0, 498, 516, 523, 521, 529, 517, 513, 490, 0, 0, 0, 0, 0, 0, 0, 0, 584, 516, 559, 518, 499, 468, 478, 527, 0, 0, 0, 0, 0, 0, 0, 0, 520, 544, 557, 517, 533, 546, 483, 488, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3920, 4121, 4110, 4111, 4062, 4107, 4149, 4188] avg=4096.0000 max=4188 max_rank=7 min=3920 min_rank=0 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16506 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[527, 549, 513, 522, 524, 522, 510, 474, 32, 6, 0, 0, 0, 0, 0, 0, 489, 512, 508, 557, 521, 545, 529, 511, 0, 0, 0, 0, 0, 0, 0, 0, 500, 497, 502, 539, 520, 505, 507, 485, 0, 0, 0, 0, 0, 0, 0, 0, 518, 542, 527, 482, 471, 524, 507, 496, 8, 0, 0, 0, 0, 0, 0, 0, 525, 501, 501, 516, 548, 501, 453, 543, 3, 1, 0, 0, 0, 0, 0, 0, 496, 479, 508, 488, 526, 512, 517, 522, 0, 0, 0, 0, 0, 0, 0, 0, 519, 512, 482, 497, 486, 524, 552, 500, 0, 0, 0, 0, 0, 0, 0, 0, 541, 510, 545, 508, 492, 512, 472, 495, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4179, 4172, 4055, 4075, 4092, 4048, 4072, 4075] avg=4096.0000 max=4179 max_rank=0 min=4048 min_rank=5 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16287 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[483, 503, 519, 500, 533, 528, 533, 523, 30, 6, 0, 0, 0, 0, 0, 0, 492, 503, 480, 519, 501, 518, 502, 515, 0, 0, 0, 0, 0, 0, 0, 0, 451, 503, 529, 485, 500, 531, 498, 516, 0, 0, 0, 0, 0, 0, 0, 0, 520, 532, 485, 471, 522, 521, 522, 462, 8, 0, 0, 0, 0, 0, 0, 0, 506, 518, 524, 523, 535, 493, 513, 466, 3, 1, 0, 0, 0, 0, 0, 0, 528, 484, 528, 534, 523, 509, 504, 526, 0, 0, 0, 0, 0, 0, 0, 0, 513, 506, 520, 547, 508, 551, 519, 511, 0, 0, 0, 0, 0, 0, 0, 0, 564, 495, 522, 490, 519, 514, 513, 514, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4158, 4030, 4013, 4043, 4082, 4136, 4175, 4131] avg=4096.0000 max=4175 max_rank=6 min=4013 min_rank=2 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16524 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[479, 458, 511, 516, 526, 488, 537, 496, 32, 6, 0, 0, 0, 0, 0, 0, 500, 513, 504, 503, 548, 516, 522, 534, 0, 0, 0, 0, 0, 0, 0, 0, 508, 530, 537, 570, 505, 513, 501, 505, 0, 0, 0, 0, 0, 0, 0, 0, 500, 493, 484, 487, 490, 511, 514, 500, 8, 0, 0, 0, 0, 0, 0, 0, 510, 501, 504, 522, 511, 497, 546, 534, 3, 1, 0, 0, 0, 0, 0, 0, 544, 503, 512, 515, 542, 498, 510, 520, 0, 0, 0, 0, 0, 0, 0, 0, 473, 504, 501, 516, 509, 524, 515, 568, 0, 0, 0, 0, 0, 0, 0, 0, 509, 508, 480, 490, 503, 565, 480, 505, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4049, 4140, 4169, 3987, 4129, 4144, 4110, 4040] avg=4096.0000 max=4169 max_rank=2 min=3987 min_rank=3 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16423 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[484, 502, 488, 510, 513, 489, 522, 549, 31, 6, 0, 0, 0, 0, 0, 0, 540, 526, 497, 507, 531, 495, 531, 507, 0, 0, 0, 0, 0, 0, 0, 0, 529, 504, 492, 495, 520, 517, 503, 546, 0, 0, 0, 0, 0, 0, 0, 0, 504, 565, 483, 507, 503, 534, 521, 506, 8, 0, 0, 0, 0, 0, 0, 0, 515, 500, 528, 514, 498, 535, 493, 492, 3, 1, 0, 0, 0, 0, 0, 0, 529, 493, 536, 497, 532, 511, 453, 543, 0, 0, 0, 0, 0, 0, 0, 0, 530, 518, 478, 504, 502, 478, 529, 501, 0, 0, 0, 0, 0, 0, 0, 0, 516, 528, 506, 503, 501, 490, 523, 523, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4134, 4106, 4131, 4079, 4094, 4040, 4090] avg=4096.0000 max=4134 max_rank=1 min=4040 min_rank=6 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16303 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[527, 473, 515, 498, 490, 547, 536, 550, 31, 6, 0, 0, 0, 0, 0, 0, 501, 484, 495, 511, 516, 541, 513, 484, 0, 0, 0, 0, 0, 0, 0, 0, 441, 499, 512, 523, 544, 536, 536, 541, 0, 0, 0, 0, 0, 0, 0, 0, 510, 527, 515, 529, 522, 503, 519, 513, 8, 0, 0, 0, 0, 0, 0, 0, 560, 506, 517, 535, 472, 514, 520, 509, 3, 1, 0, 0, 0, 0, 0, 0, 504, 467, 533, 477, 522, 484, 471, 487, 0, 0, 0, 0, 0, 0, 0, 0, 493, 524, 488, 499, 543, 496, 511, 508, 0, 0, 0, 0, 0, 0, 0, 0, 501, 530, 498, 533, 533, 502, 525, 506, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4173, 4045, 4132, 4146, 4137, 3945, 4062, 4128] avg=4096.0000 max=4173 max_rank=0 min=3945 min_rank=5 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16272 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[541, 525, 481, 533, 501, 511, 495, 520, 33, 11, 0, 0, 0, 0, 0, 0, 555, 518, 469, 503, 512, 481, 505, 528, 0, 0, 0, 0, 0, 0, 0, 0, 506, 505, 493, 512, 483, 501, 533, 522, 0, 0, 0, 0, 0, 0, 0, 0, 517, 502, 537, 514, 507, 548, 497, 514, 12, 0, 0, 0, 0, 0, 0, 0, 495, 500, 490, 523, 503, 521, 516, 527, 6, 8, 3, 0, 0, 0, 0, 0, 533, 522, 533, 498, 522, 517, 522, 502, 4, 0, 0, 0, 0, 0, 0, 0, 503, 523, 482, 513, 473, 519, 534, 490, 0, 0, 0, 0, 0, 0, 0, 0, 517, 518, 510, 537, 473, 482, 498, 526, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4151, 4071, 4055, 4148, 4092, 4153, 4037, 4061] avg=4096.0000 max=4153 max_rank=5 min=4037 min_rank=6 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16343 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[485, 496, 466, 504, 496, 503, 522, 536, 30, 6, 0, 0, 0, 0, 0, 0, 522, 534, 478, 506, 517, 506, 485, 507, 0, 0, 0, 0, 0, 0, 0, 0, 491, 529, 534, 510, 548, 540, 519, 457, 0, 0, 0, 0, 0, 0, 0, 0, 514, 513, 495, 511, 530, 548, 515, 493, 8, 0, 0, 0, 0, 0, 0, 0, 500, 538, 470, 516, 481, 547, 516, 523, 3, 1, 0, 0, 0, 0, 0, 0, 482, 531, 541, 522, 558, 477, 515, 515, 0, 0, 0, 0, 0, 0, 0, 0, 494, 508, 517, 515, 547, 531, 489, 522, 0, 0, 0, 0, 0, 0, 0, 0, 520, 485, 526, 497, 521, 497, 491, 518, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4044, 4055, 4128, 4127, 4095, 4141, 4123, 4055] avg=4096.0000 max=4141 max_rank=5 min=4044 min_rank=0 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16414 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 3968, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4096, 4352, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3968, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4352, 4096, 4096, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.rerun_state_machine:Result validation enabled +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") + [2026-09-21 08:26:47] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 37185.4 | throughput per GPU (TFLOP/s/GPU): 0.9 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.118878E+01 | loss scale: 1.0 | grad norm: 34.557 | number of skipped iterations: 0 | number of nan iterations: 0 | +Number of parameters in transformer block in billions: 6.71 +Number of parameters in embedding layers in billions: 0.21 +Total number of parameters in billions: 6.92 +Number of parameters in most loaded shard in billions: 6.9171 +Theoretical memory footprints: weight and optimizer=49474.72 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 19764.63671875 | max allocated: 23491.14599609375 | reserved: 24204.0 | max reserved: 24204.0 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[492, 515, 540, 546, 476, 521, 503, 464, 489, 493, 523, 480, 527, 548, 493, 495, 508, 529, 514, 488, 592, 505, 525, 513, 536, 500, 517, 498, 515, 498, 544, 489, 532, 539, 545, 553, 521, 472, 500, 525, 534, 517, 503, 487, 532, 532, 516, 494, 485, 489, 535, 484, 497, 545, 514, 494, 515, 487, 498, 503, 496, 519, 515, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[523, 541, 510, 515, 512, 504, 528, 527, 510, 536, 496, 492, 512, 515, 504, 498, 514, 519, 497, 506, 557, 510, 491, 526, 511, 530, 496, 511, 535, 522, 525, 529, 519, 516, 466, 556, 563, 517, 503, 531, 513, 525, 503, 489, 493, 484, 493, 487, 503, 521, 491, 519, 512, 520, 506, 555, 525, 491, 509, 466, 497, 487, 501, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[507, 511, 572, 519, 538, 526, 513, 530, 531, 461, 511, 499, 529, 478, 489, 541, 518, 554, 527, 516, 489, 519, 483, 537, 504, 481, 492, 510, 481, 509, 504, 534, 503, 504, 550, 495, 514, 501, 498, 487, 483, 513, 481, 528, 503, 509, 520, 536, 555, 571, 492, 554, 536, 526, 536, 518, 491, 485, 525, 487, 494, 464, 514, 482] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[549, 501, 521, 520, 556, 489, 505, 521, 494, 499, 496, 499, 480, 481, 533, 509, 539, 523, 525, 553, 501, 475, 520, 561, 522, 497, 502, 491, 542, 515, 538, 490, 492, 496, 508, 492, 565, 523, 521, 541, 536, 515, 498, 485, 517, 510, 560, 500, 501, 514, 483, 539, 488, 526, 520, 518, 524, 500, 547, 499, 494, 464, 482, 463] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[518, 529, 500, 504, 488, 517, 466, 511, 515, 533, 546, 514, 519, 488, 492, 510, 492, 523, 503, 504, 525, 468, 524, 491, 504, 513, 482, 529, 517, 490, 504, 543, 490, 479, 549, 545, 509, 554, 538, 513, 508, 536, 538, 549, 484, 554, 519, 487, 503, 533, 536, 480, 535, 482, 556, 508, 510, 513, 523, 467, 488, 528, 493, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[510, 506, 535, 512, 528, 511, 534, 545, 524, 519, 499, 502, 505, 507, 519, 512, 518, 520, 506, 544, 513, 508, 478, 535, 507, 527, 513, 529, 500, 504, 530, 502, 501, 507, 492, 531, 536, 486, 505, 496, 507, 538, 517, 476, 533, 504, 518, 510, 514, 491, 527, 507, 510, 525, 523, 474, 520, 496, 558, 454, 525, 495, 497, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[517, 528, 491, 520, 498, 531, 516, 493, 507, 525, 516, 525, 499, 476, 535, 490, 530, 522, 519, 472, 490, 520, 491, 534, 487, 596, 533, 514, 513, 522, 527, 500, 519, 499, 522, 549, 518, 521, 532, 503, 523, 480, 528, 518, 488, 486, 509, 491, 523, 490, 556, 481, 538, 465, 523, 509, 507, 485, 523, 508, 497, 518, 509, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[513, 550, 500, 503, 537, 532, 536, 503, 493, 487, 506, 484, 477, 505, 510, 509, 528, 503, 469, 489, 500, 500, 494, 507, 585, 524, 503, 510, 528, 555, 523, 542, 510, 528, 519, 544, 486, 506, 545, 489, 524, 489, 498, 535, 520, 519, 520, 500, 542, 509, 509, 529, 504, 519, 516, 494, 480, 484, 523, 511, 504, 498, 490, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[523, 502, 510, 515, 512, 504, 528, 527, 0, 0, 0, 0, 0, 0, 0, 0, 510, 536, 496, 492, 512, 515, 504, 498, 22, 21, 2, 0, 0, 0, 0, 0, 514, 519, 497, 506, 557, 510, 491, 516, 0, 0, 0, 0, 0, 0, 0, 0, 511, 530, 496, 511, 535, 522, 494, 529, 0, 0, 0, 0, 0, 0, 0, 0, 519, 516, 466, 517, 563, 517, 503, 531, 0, 0, 0, 0, 0, 0, 0, 0, 513, 525, 503, 489, 493, 484, 493, 487, 8, 0, 0, 0, 0, 0, 0, 0, 503, 521, 491, 519, 512, 520, 484, 555, 0, 0, 0, 0, 0, 0, 0, 0, 525, 491, 509, 466, 497, 487, 501, 505, 39, 39, 10, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4108, 4110, 4128, 4132, 3995, 4105, 4069] avg=4096.0000 max=4132 max_rank=4 min=3995 min_rank=5 imbalance_ratio=1.008789 max_load_reduction=28636 max_load_reduction_pct=0.873901 moved_to_echo=16301 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[507, 474, 572, 519, 538, 526, 513, 530, 0, 0, 0, 0, 0, 0, 0, 0, 531, 461, 511, 499, 529, 478, 489, 541, 24, 20, 2, 0, 0, 0, 0, 0, 518, 554, 527, 516, 489, 519, 483, 527, 0, 0, 0, 0, 0, 0, 0, 0, 504, 481, 492, 510, 481, 509, 474, 534, 0, 0, 0, 0, 0, 0, 0, 0, 503, 504, 550, 461, 514, 501, 498, 487, 0, 0, 0, 0, 0, 0, 0, 0, 483, 513, 481, 528, 503, 509, 520, 536, 8, 0, 0, 0, 0, 0, 0, 0, 555, 571, 492, 554, 536, 526, 512, 518, 0, 0, 0, 0, 0, 0, 0, 0, 491, 485, 525, 487, 494, 464, 514, 482, 37, 34, 10, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4179, 4085, 4133, 3985, 4018, 4081, 4264, 4023] avg=4096.0000 max=4264 max_rank=6 min=3985 min_rank=3 imbalance_ratio=1.041016 max_load_reduction=28504 max_load_reduction_pct=0.869873 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[492, 474, 540, 546, 476, 521, 503, 464, 0, 0, 0, 0, 0, 0, 0, 0, 489, 493, 523, 480, 527, 548, 493, 495, 27, 24, 6, 0, 0, 0, 0, 0, 508, 529, 514, 488, 592, 505, 525, 495, 0, 0, 0, 0, 0, 0, 0, 0, 536, 500, 517, 498, 515, 498, 504, 489, 0, 0, 0, 0, 0, 0, 0, 0, 532, 539, 545, 510, 521, 472, 500, 525, 0, 0, 0, 0, 0, 0, 0, 0, 534, 517, 503, 487, 532, 532, 516, 494, 12, 0, 0, 0, 0, 0, 0, 0, 485, 489, 535, 484, 497, 545, 487, 494, 0, 0, 0, 0, 0, 0, 0, 0, 515, 487, 498, 503, 496, 519, 515, 514, 41, 43, 16, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4016, 4105, 4156, 4057, 4144, 4127, 4016, 4147] avg=4096.0000 max=4156 max_rank=2 min=4016 min_rank=0 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16434 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[549, 464, 521, 520, 556, 489, 505, 521, 0, 0, 0, 0, 0, 0, 0, 0, 494, 499, 496, 499, 480, 481, 533, 509, 23, 21, 2, 0, 0, 0, 0, 0, 539, 523, 525, 553, 501, 475, 520, 551, 0, 0, 0, 0, 0, 0, 0, 0, 522, 497, 502, 491, 542, 515, 506, 490, 0, 0, 0, 0, 0, 0, 0, 0, 492, 496, 508, 458, 565, 523, 521, 541, 0, 0, 0, 0, 0, 0, 0, 0, 536, 515, 498, 485, 517, 510, 560, 500, 8, 0, 0, 0, 0, 0, 0, 0, 501, 514, 483, 539, 488, 526, 497, 518, 0, 0, 0, 0, 0, 0, 0, 0, 524, 500, 547, 499, 494, 464, 482, 463, 37, 34, 11, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4037, 4187, 4065, 4104, 4129, 4066, 4055] avg=4096.0000 max=4187 max_rank=2 min=4037 min_rank=1 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16354 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[510, 469, 535, 512, 528, 511, 534, 545, 0, 0, 0, 0, 0, 0, 0, 0, 524, 519, 499, 502, 505, 507, 519, 512, 23, 21, 2, 0, 0, 0, 0, 0, 518, 520, 506, 544, 513, 508, 478, 525, 0, 0, 0, 0, 0, 0, 0, 0, 507, 527, 513, 529, 500, 504, 499, 502, 0, 0, 0, 0, 0, 0, 0, 0, 501, 507, 492, 494, 536, 486, 505, 496, 0, 0, 0, 0, 0, 0, 0, 0, 507, 538, 517, 476, 533, 504, 518, 510, 8, 0, 0, 0, 0, 0, 0, 0, 514, 491, 527, 507, 510, 525, 500, 474, 0, 0, 0, 0, 0, 0, 0, 0, 520, 496, 558, 454, 525, 495, 497, 493, 37, 37, 10, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4144, 4133, 4112, 4081, 4017, 4111, 4048, 4122] avg=4096.0000 max=4144 max_rank=0 min=4017 min_rank=4 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16298 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[518, 490, 500, 504, 488, 517, 466, 511, 0, 0, 0, 0, 0, 0, 0, 0, 515, 533, 546, 514, 519, 488, 492, 510, 25, 20, 2, 0, 0, 0, 0, 0, 492, 523, 503, 504, 525, 468, 524, 482, 0, 0, 0, 0, 0, 0, 0, 0, 504, 513, 482, 529, 517, 490, 474, 543, 0, 0, 0, 0, 0, 0, 0, 0, 490, 479, 549, 507, 509, 554, 538, 513, 0, 0, 0, 0, 0, 0, 0, 0, 508, 536, 538, 549, 484, 554, 519, 487, 7, 0, 0, 0, 0, 0, 0, 0, 503, 533, 536, 480, 535, 482, 531, 508, 0, 0, 0, 0, 0, 0, 0, 0, 510, 513, 523, 467, 488, 528, 493, 499, 39, 38, 10, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3994, 4164, 4021, 4052, 4139, 4182, 4108, 4108] avg=4096.0000 max=4182 max_rank=5 min=3994 min_rank=0 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16537 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[517, 489, 491, 520, 498, 531, 516, 493, 0, 0, 0, 0, 0, 0, 0, 0, 507, 525, 516, 525, 499, 476, 535, 490, 23, 21, 2, 0, 0, 0, 0, 0, 530, 522, 519, 472, 490, 520, 491, 524, 0, 0, 0, 0, 0, 0, 0, 0, 487, 596, 533, 514, 513, 522, 496, 500, 0, 0, 0, 0, 0, 0, 0, 0, 519, 499, 522, 511, 518, 521, 532, 503, 0, 0, 0, 0, 0, 0, 0, 0, 523, 480, 528, 518, 488, 486, 509, 491, 8, 0, 0, 0, 0, 0, 0, 0, 523, 490, 556, 481, 538, 465, 500, 509, 0, 0, 0, 0, 0, 0, 0, 0, 507, 485, 523, 508, 497, 518, 509, 513, 39, 38, 10, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4055, 4119, 4068, 4161, 4125, 4031, 4062, 4147] avg=4096.0000 max=4161 max_rank=3 min=4031 min_rank=5 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16365 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[513, 510, 500, 503, 537, 532, 536, 503, 0, 0, 0, 0, 0, 0, 0, 0, 493, 487, 506, 484, 477, 505, 510, 509, 23, 21, 2, 0, 0, 0, 0, 0, 528, 503, 469, 489, 500, 500, 494, 498, 0, 0, 0, 0, 0, 0, 0, 0, 585, 524, 503, 510, 528, 555, 492, 542, 0, 0, 0, 0, 0, 0, 0, 0, 510, 528, 519, 506, 486, 506, 545, 489, 0, 0, 0, 0, 0, 0, 0, 0, 524, 489, 498, 535, 520, 519, 520, 500, 7, 0, 0, 0, 0, 0, 0, 0, 542, 509, 509, 529, 504, 519, 493, 494, 0, 0, 0, 0, 0, 0, 0, 0, 480, 484, 523, 511, 504, 498, 490, 519, 40, 38, 10, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4134, 4017, 3981, 4239, 4089, 4112, 4099, 4097] avg=4096.0000 max=4239 max_rank=3 min=3981 min_rank=2 imbalance_ratio=1.034912 max_load_reduction=28529 max_load_reduction_pct=0.870636 moved_to_echo=16397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 256, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4096, 4096, 4224, 3968, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 3968, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 3968, 4224, 3968, 4096, 4096, 4096, 4096, 384, 384, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[515, 535, 531, 479, 547, 537, 514, 496, 508, 528, 492, 496, 500, 473, 497, 501, 530, 511, 480, 483, 514, 518, 502, 526, 523, 493, 526, 499, 553, 517, 502, 529, 498, 504, 494, 495, 556, 524, 525, 484, 517, 533, 500, 521, 506, 529, 552, 521, 537, 491, 501, 526, 499, 495, 529, 526, 534, 511, 532, 501, 457, 479, 512, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[530, 507, 504, 569, 490, 517, 519, 517, 524, 479, 516, 471, 504, 518, 487, 519, 550, 482, 532, 567, 486, 527, 494, 518, 513, 483, 499, 525, 517, 502, 517, 515, 493, 513, 500, 495, 546, 488, 492, 475, 532, 555, 504, 527, 554, 504, 496, 507, 480, 486, 527, 546, 531, 512, 506, 521, 517, 508, 465, 544, 527, 500, 502, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[505, 506, 543, 507, 556, 501, 517, 497, 541, 479, 496, 535, 524, 536, 496, 541, 496, 514, 499, 477, 518, 494, 481, 472, 536, 509, 502, 549, 546, 508, 519, 475, 512, 533, 471, 536, 507, 539, 485, 535, 490, 507, 512, 526, 555, 514, 497, 499, 485, 488, 534, 526, 519, 510, 504, 507, 496, 531, 502, 527, 524, 525, 481, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[493, 534, 517, 489, 498, 512, 523, 512, 510, 506, 491, 494, 486, 549, 477, 473, 521, 526, 523, 541, 514, 488, 516, 523, 549, 546, 497, 495, 502, 531, 477, 514, 501, 490, 496, 504, 545, 482, 496, 512, 544, 526, 450, 490, 510, 545, 507, 515, 485, 564, 504, 514, 534, 545, 538, 516, 534, 501, 509, 526, 520, 523, 502, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[487, 532, 504, 528, 521, 517, 518, 511, 460, 557, 515, 507, 494, 512, 530, 537, 506, 506, 500, 584, 543, 516, 517, 472, 487, 507, 489, 519, 544, 510, 484, 523, 516, 541, 521, 483, 466, 528, 494, 527, 480, 528, 533, 517, 527, 507, 484, 502, 501, 492, 523, 522, 506, 503, 556, 515, 536, 522, 510, 514, 494, 477, 520, 486] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[544, 545, 538, 530, 527, 518, 541, 470, 488, 527, 518, 552, 517, 497, 475, 543, 543, 508, 496, 555, 507, 522, 487, 495, 511, 485, 490, 519, 518, 531, 536, 518, 534, 511, 520, 526, 511, 511, 504, 533, 511, 490, 518, 523, 494, 528, 529, 488, 503, 450, 522, 497, 536, 493, 482, 494, 492, 474, 536, 534, 481, 476, 522, 494] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[493, 507, 498, 509, 515, 539, 500, 548, 501, 515, 545, 555, 521, 478, 534, 472, 519, 482, 494, 521, 552, 509, 524, 500, 487, 507, 518, 543, 522, 515, 512, 493, 520, 476, 494, 493, 490, 501, 517, 496, 529, 513, 522, 528, 492, 514, 461, 533, 510, 520, 519, 531, 523, 495, 516, 552, 505, 542, 495, 475, 537, 499, 523, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[458, 506, 505, 533, 510, 511, 476, 509, 523, 532, 515, 506, 538, 513, 548, 527, 538, 527, 472, 499, 503, 525, 514, 502, 492, 521, 541, 522, 512, 498, 516, 533, 516, 524, 482, 525, 526, 500, 471, 519, 537, 511, 546, 473, 534, 527, 540, 526, 521, 535, 477, 538, 520, 496, 523, 535, 480, 495, 461, 512, 510, 501, 494, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[505, 484, 543, 507, 556, 501, 517, 497, 0, 0, 0, 0, 0, 0, 0, 0, 541, 479, 496, 535, 524, 536, 496, 541, 8, 0, 0, 0, 0, 0, 0, 0, 496, 514, 499, 477, 518, 494, 481, 472, 0, 0, 0, 0, 0, 0, 0, 0, 536, 509, 502, 549, 535, 508, 519, 475, 0, 0, 0, 0, 0, 0, 0, 0, 512, 533, 471, 536, 507, 539, 485, 535, 15, 13, 3, 0, 0, 0, 0, 0, 490, 507, 512, 526, 526, 514, 497, 499, 0, 0, 0, 0, 0, 0, 0, 0, 485, 488, 534, 511, 519, 510, 504, 507, 0, 0, 0, 0, 0, 0, 0, 0, 496, 531, 502, 527, 524, 525, 481, 516, 29, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4156, 3951, 4133, 4149, 4071, 4058, 4140] avg=4096.0000 max=4156 max_rank=1 min=3951 min_rank=2 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16418 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[515, 511, 531, 479, 547, 537, 514, 496, 0, 0, 0, 0, 0, 0, 0, 0, 508, 528, 492, 496, 500, 473, 497, 501, 9, 0, 0, 0, 0, 0, 0, 0, 530, 511, 480, 483, 514, 518, 502, 526, 0, 0, 0, 0, 0, 0, 0, 0, 523, 493, 526, 499, 541, 517, 502, 529, 0, 0, 0, 0, 0, 0, 0, 0, 498, 504, 494, 495, 556, 524, 525, 484, 15, 14, 3, 0, 0, 0, 0, 0, 517, 533, 500, 521, 480, 529, 552, 521, 0, 0, 0, 0, 0, 0, 0, 0, 537, 491, 501, 511, 499, 495, 529, 526, 0, 0, 0, 0, 0, 0, 0, 0, 534, 511, 532, 501, 457, 479, 512, 524, 26, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4130, 4004, 4064, 4130, 4112, 4153, 4089, 4086] avg=4096.0000 max=4153 max_rank=5 min=4004 min_rank=1 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16440 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[487, 508, 504, 528, 521, 517, 518, 511, 0, 0, 0, 0, 0, 0, 0, 0, 460, 557, 515, 507, 494, 512, 530, 537, 8, 0, 0, 0, 0, 0, 0, 0, 506, 506, 500, 584, 543, 516, 517, 472, 0, 0, 0, 0, 0, 0, 0, 0, 487, 507, 489, 519, 533, 510, 484, 523, 0, 0, 0, 0, 0, 0, 0, 0, 516, 541, 521, 483, 466, 528, 494, 527, 15, 14, 3, 0, 0, 0, 0, 0, 480, 528, 533, 517, 499, 507, 484, 502, 0, 0, 0, 0, 0, 0, 0, 0, 501, 492, 523, 507, 506, 503, 556, 515, 0, 0, 0, 0, 0, 0, 0, 0, 536, 522, 510, 514, 494, 477, 520, 486, 28, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4120, 4144, 4052, 4108, 4050, 4103, 4097] avg=4096.0000 max=4144 max_rank=2 min=4050 min_rank=5 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16358 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[493, 510, 517, 489, 498, 512, 523, 512, 0, 0, 0, 0, 0, 0, 0, 0, 510, 506, 491, 494, 486, 549, 477, 473, 8, 0, 0, 0, 0, 0, 0, 0, 521, 526, 523, 541, 514, 488, 516, 523, 0, 0, 0, 0, 0, 0, 0, 0, 549, 546, 497, 495, 491, 531, 477, 514, 0, 0, 0, 0, 0, 0, 0, 0, 501, 490, 496, 504, 545, 482, 496, 512, 15, 14, 3, 0, 0, 0, 0, 0, 544, 526, 450, 490, 483, 545, 507, 515, 0, 0, 0, 0, 0, 0, 0, 0, 485, 564, 504, 499, 534, 545, 538, 516, 0, 0, 0, 0, 0, 0, 0, 0, 534, 501, 509, 526, 520, 523, 502, 513, 27, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4054, 3994, 4152, 4100, 4058, 4060, 4185, 4165] avg=4096.0000 max=4185 max_rank=6 min=3994 min_rank=1 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16468 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[493, 485, 498, 509, 515, 539, 500, 548, 0, 0, 0, 0, 0, 0, 0, 0, 501, 515, 545, 555, 521, 478, 534, 472, 8, 0, 0, 0, 0, 0, 0, 0, 519, 482, 494, 521, 552, 509, 524, 500, 0, 0, 0, 0, 0, 0, 0, 0, 487, 507, 518, 543, 511, 515, 512, 493, 0, 0, 0, 0, 0, 0, 0, 0, 520, 476, 494, 493, 490, 501, 517, 496, 15, 13, 3, 0, 0, 0, 0, 0, 529, 513, 522, 528, 466, 514, 461, 533, 0, 0, 0, 0, 0, 0, 0, 0, 510, 520, 519, 516, 523, 495, 516, 552, 0, 0, 0, 0, 0, 0, 0, 0, 505, 542, 495, 475, 537, 499, 523, 519, 26, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4129, 4101, 4086, 4018, 4066, 4151, 4130] avg=4096.0000 max=4151 max_rank=6 min=4018 min_rank=4 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16365 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[530, 485, 504, 569, 490, 517, 519, 517, 0, 0, 0, 0, 0, 0, 0, 0, 524, 479, 516, 471, 504, 518, 487, 519, 8, 0, 0, 0, 0, 0, 0, 0, 550, 482, 532, 567, 486, 527, 494, 518, 0, 0, 0, 0, 0, 0, 0, 0, 513, 483, 499, 525, 506, 502, 517, 515, 0, 0, 0, 0, 0, 0, 0, 0, 493, 513, 500, 495, 546, 488, 492, 475, 16, 13, 3, 0, 0, 0, 0, 0, 532, 555, 504, 527, 525, 504, 496, 507, 0, 0, 0, 0, 0, 0, 0, 0, 480, 486, 527, 530, 531, 512, 506, 521, 0, 0, 0, 0, 0, 0, 0, 0, 517, 508, 465, 544, 527, 500, 502, 517, 29, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4131, 4026, 4156, 4060, 4034, 4150, 4093, 4118] avg=4096.0000 max=4156 max_rank=2 min=4026 min_rank=1 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16395 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[544, 508, 538, 530, 527, 518, 541, 470, 0, 0, 0, 0, 0, 0, 0, 0, 488, 527, 518, 552, 517, 497, 475, 543, 12, 0, 0, 0, 0, 0, 0, 0, 543, 508, 496, 555, 507, 522, 487, 495, 7, 0, 0, 0, 0, 0, 0, 0, 511, 485, 490, 519, 493, 531, 536, 518, 0, 0, 0, 0, 0, 0, 0, 0, 534, 511, 520, 526, 511, 511, 504, 533, 18, 21, 6, 0, 0, 0, 0, 0, 511, 490, 518, 523, 465, 528, 529, 488, 0, 0, 0, 0, 0, 0, 0, 0, 503, 450, 522, 479, 536, 493, 482, 494, 0, 0, 0, 0, 0, 0, 0, 0, 492, 474, 536, 534, 481, 476, 522, 494, 29, 16, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4176, 4129, 4120, 4083, 4195, 4052, 3959, 4054] avg=4096.0000 max=4195 max_rank=4 min=3959 min_rank=6 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16260 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[458, 484, 505, 533, 510, 511, 476, 509, 0, 0, 0, 0, 0, 0, 0, 0, 523, 532, 515, 506, 538, 513, 548, 527, 8, 0, 0, 0, 0, 0, 0, 0, 538, 527, 472, 499, 503, 525, 514, 502, 0, 0, 0, 0, 0, 0, 0, 0, 492, 521, 541, 522, 501, 498, 516, 533, 0, 0, 0, 0, 0, 0, 0, 0, 516, 524, 482, 525, 526, 500, 471, 519, 15, 13, 3, 0, 0, 0, 0, 0, 537, 511, 546, 473, 506, 527, 540, 526, 0, 0, 0, 0, 0, 0, 0, 0, 521, 535, 477, 523, 520, 496, 523, 535, 0, 0, 0, 0, 0, 0, 0, 0, 480, 495, 461, 512, 510, 501, 494, 488, 28, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3986, 4210, 4080, 4124, 4094, 4166, 4130, 3978] avg=4096.0000 max=4210 max_rank=1 min=3978 min_rank=7 imbalance_ratio=1.027832 max_load_reduction=28558 max_load_reduction_pct=0.871521 moved_to_echo=16368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4352, 4224, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4096, 4224, 3968, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[569, 472, 522, 500, 529, 509, 528, 525, 531, 493, 527, 474, 538, 510, 497, 496, 496, 492, 511, 536, 520, 525, 537, 509, 514, 515, 526, 538, 547, 503, 521, 524, 524, 513, 479, 511, 514, 475, 500, 465, 493, 497, 490, 515, 519, 522, 500, 517, 519, 511, 528, 523, 525, 525, 518, 516, 485, 502, 519, 491, 518, 526, 492, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[501, 517, 504, 487, 513, 519, 493, 529, 487, 552, 527, 504, 504, 516, 494, 476, 502, 532, 511, 464, 496, 496, 529, 517, 525, 503, 522, 565, 538, 505, 496, 526, 488, 523, 506, 479, 496, 498, 527, 509, 530, 517, 541, 502, 513, 511, 534, 508, 533, 538, 537, 494, 492, 537, 512, 540, 503, 498, 495, 522, 521, 497, 495, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[523, 536, 488, 517, 532, 496, 485, 498, 500, 502, 529, 520, 528, 517, 461, 518, 543, 543, 523, 541, 500, 523, 485, 528, 473, 516, 541, 521, 504, 484, 556, 491, 518, 539, 506, 477, 547, 526, 442, 515, 502, 515, 516, 462, 490, 554, 517, 534, 514, 514, 516, 528, 484, 507, 455, 521, 512, 535, 497, 507, 508, 538, 541, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[509, 529, 491, 529, 526, 528, 475, 503, 495, 516, 523, 482, 533, 495, 487, 505, 500, 494, 507, 548, 534, 488, 561, 502, 497, 569, 543, 520, 507, 523, 517, 508, 537, 492, 499, 531, 533, 519, 505, 509, 501, 550, 493, 521, 477, 482, 503, 509, 502, 499, 524, 554, 501, 492, 489, 495, 530, 525, 538, 490, 503, 484, 500, 537] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[495, 492, 508, 490, 499, 487, 486, 504, 516, 548, 510, 517, 559, 527, 527, 511, 477, 511, 494, 505, 540, 540, 517, 510, 531, 550, 482, 502, 518, 507, 536, 497, 518, 489, 481, 533, 476, 523, 514, 502, 555, 528, 523, 505, 537, 492, 502, 496, 508, 512, 525, 509, 500, 547, 501, 491, 503, 499, 529, 512, 494, 537, 511, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[492, 527, 458, 522, 538, 513, 520, 532, 489, 521, 547, 530, 532, 542, 491, 513, 532, 496, 485, 475, 498, 521, 514, 473, 506, 531, 468, 509, 492, 510, 529, 526, 529, 513, 507, 526, 550, 493, 548, 494, 558, 515, 520, 537, 504, 518, 526, 492, 515, 477, 509, 523, 479, 490, 534, 543, 486, 501, 527, 530, 502, 482, 508, 500] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[512, 475, 499, 508, 489, 509, 537, 531, 519, 496, 508, 522, 496, 505, 525, 496, 533, 496, 499, 502, 518, 526, 524, 487, 540, 511, 515, 511, 484, 534, 517, 476, 530, 521, 517, 538, 496, 523, 523, 506, 504, 502, 515, 515, 519, 493, 509, 469, 555, 520, 543, 533, 503, 515, 515, 512, 495, 508, 529, 542, 494, 531, 497, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[501, 501, 496, 530, 537, 482, 525, 517, 495, 501, 517, 490, 490, 521, 547, 461, 500, 504, 533, 546, 543, 528, 491, 498, 494, 504, 512, 488, 486, 507, 519, 481, 507, 514, 543, 546, 463, 509, 507, 538, 550, 538, 505, 528, 507, 506, 524, 499, 528, 499, 557, 494, 527, 504, 486, 531, 499, 530, 514, 540, 485, 543, 493, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[569, 472, 522, 500, 529, 509, 528, 525, 19, 1, 0, 0, 0, 0, 0, 0, 531, 493, 527, 474, 538, 510, 497, 496, 5, 2, 0, 0, 0, 0, 0, 0, 496, 492, 511, 536, 520, 525, 532, 509, 0, 0, 0, 0, 0, 0, 0, 0, 514, 500, 526, 538, 547, 503, 521, 524, 0, 0, 0, 0, 0, 0, 0, 0, 524, 513, 479, 511, 514, 475, 500, 465, 10, 0, 0, 0, 0, 0, 0, 0, 487, 497, 490, 515, 519, 522, 500, 517, 0, 0, 0, 0, 0, 0, 0, 0, 519, 511, 509, 523, 525, 525, 518, 516, 0, 0, 0, 0, 0, 0, 0, 0, 485, 502, 519, 491, 518, 526, 492, 502, 4, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4174, 4073, 4121, 4173, 3991, 4047, 4146, 4043] avg=4096.0000 max=4174 max_rank=0 min=3991 min_rank=4 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16227 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[523, 536, 488, 517, 532, 496, 485, 498, 19, 1, 0, 0, 0, 0, 0, 0, 500, 502, 529, 520, 528, 517, 461, 518, 4, 2, 0, 0, 0, 0, 0, 0, 543, 543, 523, 541, 500, 523, 481, 528, 0, 0, 0, 0, 0, 0, 0, 0, 473, 501, 541, 521, 504, 484, 556, 491, 0, 0, 0, 0, 0, 0, 0, 0, 518, 539, 506, 477, 547, 526, 442, 515, 10, 0, 0, 0, 0, 0, 0, 0, 496, 515, 516, 462, 490, 554, 517, 534, 0, 0, 0, 0, 0, 0, 0, 0, 514, 514, 497, 528, 484, 507, 455, 521, 0, 0, 0, 0, 0, 0, 0, 0, 512, 535, 497, 507, 508, 538, 541, 509, 4, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4081, 4182, 4071, 4080, 4084, 4020, 4155] avg=4096.0000 max=4182 max_rank=2 min=4020 min_rank=6 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16339 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[509, 529, 491, 529, 526, 528, 475, 503, 19, 1, 0, 0, 0, 0, 0, 0, 495, 516, 523, 482, 533, 495, 487, 505, 5, 2, 0, 0, 0, 0, 0, 0, 500, 494, 507, 548, 534, 488, 556, 502, 0, 0, 0, 0, 0, 0, 0, 0, 497, 551, 543, 520, 507, 523, 517, 508, 0, 0, 0, 0, 0, 0, 0, 0, 537, 492, 499, 531, 533, 519, 505, 509, 12, 0, 0, 0, 0, 0, 0, 0, 495, 550, 493, 521, 477, 482, 503, 509, 0, 0, 0, 0, 0, 0, 0, 0, 502, 499, 505, 554, 501, 492, 489, 495, 0, 0, 0, 0, 0, 0, 0, 0, 530, 525, 538, 490, 503, 484, 500, 537, 5, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4043, 4129, 4166, 4137, 4030, 4037, 4116] avg=4096.0000 max=4166 max_rank=3 min=4030 min_rank=5 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16320 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[501, 517, 504, 487, 513, 519, 493, 529, 24, 7, 0, 0, 0, 0, 0, 0, 487, 552, 527, 504, 504, 516, 494, 476, 8, 7, 0, 0, 0, 0, 0, 0, 502, 532, 511, 464, 496, 496, 521, 517, 0, 0, 0, 0, 0, 0, 0, 0, 525, 472, 522, 565, 538, 505, 496, 526, 0, 0, 0, 0, 0, 0, 0, 0, 488, 523, 506, 479, 496, 498, 527, 509, 15, 0, 0, 0, 0, 0, 0, 0, 514, 517, 541, 502, 513, 511, 534, 508, 0, 0, 0, 0, 0, 0, 0, 0, 533, 538, 513, 494, 492, 537, 512, 540, 0, 0, 0, 0, 0, 0, 0, 0, 503, 498, 495, 522, 521, 497, 495, 522, 9, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4075, 4039, 4149, 4041, 4140, 4159, 4071] avg=4096.0000 max=4159 max_rank=6 min=4039 min_rank=2 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16411 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[492, 527, 458, 522, 538, 513, 520, 532, 19, 1, 0, 0, 0, 0, 0, 0, 489, 521, 547, 530, 532, 542, 491, 513, 5, 2, 0, 0, 0, 0, 0, 0, 532, 496, 485, 475, 498, 521, 509, 473, 0, 0, 0, 0, 0, 0, 0, 0, 506, 514, 468, 509, 492, 510, 529, 526, 0, 0, 0, 0, 0, 0, 0, 0, 529, 513, 507, 526, 550, 493, 548, 494, 11, 0, 0, 0, 0, 0, 0, 0, 552, 515, 520, 537, 504, 518, 526, 492, 0, 0, 0, 0, 0, 0, 0, 0, 515, 477, 490, 523, 479, 490, 534, 543, 0, 0, 0, 0, 0, 0, 0, 0, 486, 501, 527, 530, 502, 482, 508, 500, 5, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4122, 4172, 3989, 4054, 4171, 4164, 4051, 4045] avg=4096.0000 max=4172 max_rank=1 min=3989 min_rank=2 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[495, 492, 508, 490, 499, 487, 486, 504, 19, 1, 0, 0, 0, 0, 0, 0, 516, 548, 510, 517, 559, 527, 527, 511, 5, 2, 0, 0, 0, 0, 0, 0, 477, 511, 494, 505, 540, 540, 512, 510, 0, 0, 0, 0, 0, 0, 0, 0, 531, 533, 482, 502, 518, 507, 536, 497, 0, 0, 0, 0, 0, 0, 0, 0, 518, 489, 481, 533, 476, 523, 514, 502, 11, 0, 0, 0, 0, 0, 0, 0, 549, 528, 523, 505, 537, 492, 502, 496, 0, 0, 0, 0, 0, 0, 0, 0, 508, 512, 506, 509, 500, 547, 501, 491, 0, 0, 0, 0, 0, 0, 0, 0, 503, 499, 529, 512, 494, 537, 511, 523, 5, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3981, 4222, 4089, 4106, 4047, 4132, 4074, 4117] avg=4096.0000 max=4222 max_rank=1 min=3981 min_rank=0 imbalance_ratio=1.030762 max_load_reduction=28546 max_load_reduction_pct=0.871155 moved_to_echo=16370 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[512, 475, 499, 508, 489, 509, 537, 531, 20, 1, 0, 0, 0, 0, 0, 0, 519, 496, 508, 522, 496, 505, 525, 496, 5, 2, 0, 0, 0, 0, 0, 0, 533, 496, 499, 502, 518, 526, 519, 487, 0, 0, 0, 0, 0, 0, 0, 0, 540, 496, 515, 511, 484, 534, 517, 476, 0, 0, 0, 0, 0, 0, 0, 0, 530, 521, 517, 538, 496, 523, 523, 506, 10, 0, 0, 0, 0, 0, 0, 0, 498, 502, 515, 515, 519, 493, 509, 469, 0, 0, 0, 0, 0, 0, 0, 0, 555, 520, 523, 533, 503, 515, 515, 512, 0, 0, 0, 0, 0, 0, 0, 0, 495, 508, 529, 542, 494, 531, 497, 496, 4, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4074, 4080, 4073, 4164, 4020, 4176, 4100] avg=4096.0000 max=4176 max_rank=6 min=4020 min_rank=5 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16460 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[501, 501, 496, 530, 537, 482, 525, 517, 21, 1, 0, 0, 0, 0, 0, 0, 495, 501, 517, 490, 490, 521, 547, 461, 4, 2, 0, 0, 0, 0, 0, 0, 500, 504, 533, 546, 543, 528, 487, 498, 0, 0, 0, 0, 0, 0, 0, 0, 494, 489, 512, 488, 486, 507, 519, 481, 0, 0, 0, 0, 0, 0, 0, 0, 507, 514, 543, 546, 463, 509, 507, 538, 10, 0, 0, 0, 0, 0, 0, 0, 544, 538, 505, 528, 507, 506, 524, 499, 0, 0, 0, 0, 0, 0, 0, 0, 528, 499, 536, 494, 527, 504, 486, 531, 0, 0, 0, 0, 0, 0, 0, 0, 499, 530, 514, 540, 485, 543, 493, 509, 4, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4028, 4139, 3976, 4137, 4151, 4105, 4121] avg=4096.0000 max=4151 max_rank=5 min=3976 min_rank=3 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16514 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 3968, 4096, 4224, 4096, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[512, 494, 557, 521, 511, 543, 525, 504, 502, 522, 497, 522, 483, 526, 484, 506, 522, 493, 479, 486, 545, 541, 513, 508, 520, 501, 477, 525, 530, 487, 507, 496, 508, 539, 516, 503, 535, 511, 517, 543, 506, 528, 490, 504, 490, 533, 538, 497, 516, 487, 500, 490, 491, 524, 546, 500, 493, 479, 503, 518, 532, 516, 543, 533] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[523, 525, 518, 479, 488, 497, 489, 512, 499, 501, 476, 530, 490, 520, 511, 530, 486, 498, 530, 487, 504, 508, 544, 546, 504, 524, 526, 508, 515, 566, 504, 506, 529, 507, 488, 536, 528, 505, 503, 538, 514, 529, 509, 468, 514, 475, 500, 526, 518, 546, 521, 485, 517, 514, 508, 526, 519, 471, 501, 569, 477, 536, 537, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[493, 534, 510, 539, 501, 527, 520, 553, 564, 506, 504, 503, 483, 510, 539, 473, 527, 496, 491, 492, 513, 542, 529, 509, 498, 508, 510, 505, 510, 483, 497, 512, 514, 486, 520, 519, 518, 493, 499, 480, 510, 492, 496, 522, 503, 496, 528, 509, 521, 563, 518, 537, 480, 486, 520, 511, 524, 488, 530, 515, 583, 526, 497, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[528, 528, 503, 520, 544, 499, 512, 486, 519, 507, 490, 471, 533, 503, 515, 513, 521, 517, 493, 520, 511, 510, 491, 529, 527, 540, 513, 550, 513, 534, 521, 499, 501, 524, 504, 489, 545, 525, 521, 516, 517, 471, 517, 526, 530, 531, 545, 505, 494, 504, 471, 500, 525, 454, 514, 499, 518, 502, 504, 498, 486, 546, 501, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[568, 496, 517, 553, 496, 510, 508, 515, 540, 517, 549, 494, 522, 478, 537, 507, 514, 537, 511, 519, 495, 526, 502, 492, 495, 502, 541, 489, 539, 528, 530, 483, 469, 524, 510, 527, 493, 470, 520, 524, 525, 516, 517, 504, 487, 482, 500, 493, 525, 482, 522, 513, 510, 541, 529, 491, 549, 471, 539, 511, 493, 527, 497, 497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[542, 547, 485, 496, 494, 548, 508, 507, 507, 481, 489, 482, 532, 479, 495, 508, 494, 508, 492, 530, 548, 514, 535, 511, 502, 479, 505, 524, 524, 492, 491, 530, 464, 520, 495, 534, 516, 563, 527, 530, 501, 495, 518, 513, 520, 530, 521, 478, 535, 505, 528, 532, 497, 526, 517, 502, 521, 505, 526, 520, 523, 523, 520, 484] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[493, 506, 517, 510, 531, 491, 493, 511, 502, 494, 528, 535, 514, 496, 554, 482, 527, 508, 528, 522, 529, 495, 508, 520, 493, 532, 533, 510, 526, 535, 478, 490, 512, 510, 515, 515, 519, 514, 497, 538, 523, 507, 531, 498, 507, 454, 525, 514, 510, 496, 560, 503, 513, 509, 505, 527, 491, 527, 517, 504, 491, 529, 514, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[510, 478, 527, 496, 477, 508, 525, 509, 507, 519, 555, 483, 547, 538, 491, 524, 497, 498, 546, 476, 549, 507, 540, 528, 525, 497, 531, 506, 523, 515, 528, 523, 508, 517, 505, 504, 492, 512, 509, 528, 501, 510, 497, 508, 533, 486, 501, 509, 525, 504, 529, 487, 495, 535, 512, 546, 496, 503, 487, 489, 511, 545, 485, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[500, 494, 557, 521, 511, 543, 525, 504, 0, 0, 0, 0, 0, 0, 0, 0, 502, 522, 497, 522, 483, 526, 484, 506, 5, 5, 3, 3, 0, 0, 0, 0, 522, 493, 479, 486, 530, 541, 513, 508, 0, 0, 0, 0, 0, 0, 0, 0, 520, 501, 477, 525, 525, 487, 507, 496, 0, 0, 0, 0, 0, 0, 0, 0, 508, 539, 516, 503, 535, 511, 517, 535, 0, 0, 0, 0, 0, 0, 0, 0, 506, 528, 490, 504, 490, 533, 538, 497, 15, 12, 3, 0, 0, 0, 0, 0, 516, 487, 500, 490, 491, 524, 543, 500, 0, 0, 0, 0, 0, 0, 0, 0, 493, 479, 503, 518, 532, 513, 543, 533, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4155, 4058, 4072, 4038, 4164, 4116, 4051, 4114] avg=4096.0000 max=4164 max_rank=4 min=4038 min_rank=3 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16445 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[482, 534, 510, 539, 501, 527, 520, 553, 0, 0, 0, 0, 0, 0, 0, 0, 564, 506, 504, 503, 483, 510, 539, 473, 5, 5, 3, 3, 0, 0, 0, 0, 527, 496, 491, 492, 499, 542, 529, 509, 0, 0, 0, 0, 0, 0, 0, 0, 498, 508, 510, 505, 505, 483, 497, 512, 0, 0, 0, 0, 0, 0, 0, 0, 514, 486, 520, 519, 518, 493, 499, 472, 0, 0, 0, 0, 0, 0, 0, 0, 510, 492, 496, 522, 503, 496, 528, 509, 14, 11, 3, 0, 0, 0, 0, 0, 521, 563, 518, 537, 480, 486, 517, 511, 0, 0, 0, 0, 0, 0, 0, 0, 524, 488, 530, 515, 583, 523, 497, 503, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4166, 4098, 4085, 4018, 4021, 4084, 4133, 4163] avg=4096.0000 max=4166 max_rank=0 min=4018 min_rank=3 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16401 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[511, 525, 518, 479, 488, 497, 489, 512, 0, 0, 0, 0, 0, 0, 0, 0, 499, 501, 476, 530, 490, 520, 511, 530, 5, 5, 3, 3, 0, 0, 0, 0, 486, 498, 530, 487, 490, 508, 544, 546, 0, 0, 0, 0, 0, 0, 0, 0, 504, 524, 526, 508, 510, 566, 504, 506, 0, 0, 0, 0, 0, 0, 0, 0, 529, 507, 488, 536, 528, 505, 503, 530, 0, 0, 0, 0, 0, 0, 0, 0, 514, 529, 509, 468, 514, 475, 500, 526, 14, 12, 3, 0, 0, 0, 0, 0, 518, 546, 521, 485, 517, 514, 505, 526, 0, 0, 0, 0, 0, 0, 0, 0, 519, 471, 501, 569, 477, 533, 537, 510, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4019, 4073, 4089, 4148, 4126, 4064, 4132, 4117] avg=4096.0000 max=4148 max_rank=3 min=4019 min_rank=0 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16439 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[516, 528, 503, 520, 544, 499, 512, 486, 0, 0, 0, 0, 0, 0, 0, 0, 519, 507, 490, 471, 533, 503, 515, 513, 5, 5, 3, 3, 0, 0, 0, 0, 521, 517, 493, 520, 497, 510, 491, 529, 0, 0, 0, 0, 0, 0, 0, 0, 527, 540, 513, 550, 508, 534, 521, 499, 0, 0, 0, 0, 0, 0, 0, 0, 501, 524, 504, 489, 545, 525, 521, 508, 0, 0, 0, 0, 0, 0, 0, 0, 517, 471, 517, 526, 530, 531, 545, 505, 14, 12, 3, 0, 0, 0, 0, 0, 494, 504, 471, 500, 525, 454, 511, 499, 0, 0, 0, 0, 0, 0, 0, 0, 518, 502, 504, 498, 486, 543, 501, 525, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4067, 4078, 4192, 4117, 4171, 3958, 4077] avg=4096.0000 max=4192 max_rank=3 min=3958 min_rank=6 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16323 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[555, 496, 517, 553, 496, 510, 508, 515, 0, 0, 0, 0, 0, 0, 0, 0, 540, 517, 549, 494, 522, 478, 537, 507, 6, 5, 3, 3, 0, 0, 0, 0, 514, 537, 511, 519, 481, 526, 502, 492, 0, 0, 0, 0, 0, 0, 0, 0, 495, 502, 541, 489, 533, 528, 530, 483, 0, 0, 0, 0, 0, 0, 0, 0, 469, 524, 510, 527, 493, 470, 520, 516, 0, 0, 0, 0, 0, 0, 0, 0, 525, 516, 517, 504, 487, 482, 500, 493, 14, 13, 3, 0, 0, 0, 0, 0, 525, 482, 522, 513, 510, 541, 526, 491, 0, 0, 0, 0, 0, 0, 0, 0, 549, 471, 539, 511, 493, 524, 497, 497, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4150, 4161, 4082, 4101, 4029, 4054, 4110, 4081] avg=4096.0000 max=4161 max_rank=1 min=4029 min_rank=4 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16274 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[478, 506, 517, 510, 531, 491, 493, 511, 0, 0, 0, 0, 0, 0, 0, 0, 502, 494, 528, 535, 514, 496, 554, 482, 11, 11, 8, 7, 0, 0, 0, 0, 527, 508, 528, 522, 511, 495, 508, 520, 0, 0, 0, 0, 0, 0, 0, 0, 493, 532, 533, 510, 515, 535, 478, 490, 0, 0, 0, 0, 0, 0, 0, 0, 512, 510, 515, 515, 519, 514, 497, 521, 0, 0, 0, 0, 0, 0, 0, 0, 523, 507, 531, 498, 507, 454, 525, 514, 18, 15, 6, 0, 0, 0, 0, 0, 510, 496, 560, 503, 513, 509, 497, 527, 0, 0, 0, 0, 0, 0, 0, 0, 491, 527, 517, 504, 491, 522, 514, 502, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4037, 4142, 4119, 4086, 4103, 4098, 4115, 4068] avg=4096.0000 max=4142 max_rank=1 min=4037 min_rank=0 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[530, 547, 485, 496, 494, 548, 508, 507, 0, 0, 0, 0, 0, 0, 0, 0, 507, 481, 489, 482, 532, 479, 495, 508, 5, 5, 3, 3, 0, 0, 0, 0, 494, 508, 492, 530, 533, 514, 535, 511, 0, 0, 0, 0, 0, 0, 0, 0, 502, 479, 505, 524, 519, 492, 491, 530, 0, 0, 0, 0, 0, 0, 0, 0, 464, 520, 495, 534, 516, 563, 527, 522, 0, 0, 0, 0, 0, 0, 0, 0, 501, 495, 518, 513, 520, 530, 521, 478, 15, 12, 3, 0, 0, 0, 0, 0, 535, 505, 528, 532, 497, 526, 514, 502, 0, 0, 0, 0, 0, 0, 0, 0, 521, 505, 526, 520, 523, 520, 520, 484, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 3989, 4117, 4042, 4141, 4106, 4139, 4119] avg=4096.0000 max=4141 max_rank=4 min=3989 min_rank=1 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16505 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[498, 478, 527, 496, 477, 508, 525, 509, 0, 0, 0, 0, 0, 0, 0, 0, 507, 519, 555, 483, 547, 538, 491, 524, 5, 5, 3, 3, 0, 0, 0, 0, 497, 498, 546, 476, 534, 507, 540, 528, 0, 0, 0, 0, 0, 0, 0, 0, 525, 497, 531, 506, 518, 515, 528, 523, 0, 0, 0, 0, 0, 0, 0, 0, 508, 517, 505, 504, 492, 512, 509, 520, 0, 0, 0, 0, 0, 0, 0, 0, 501, 510, 497, 508, 533, 486, 501, 509, 15, 12, 3, 0, 0, 0, 0, 0, 525, 504, 529, 487, 495, 535, 509, 546, 0, 0, 0, 0, 0, 0, 0, 0, 496, 503, 487, 489, 511, 542, 485, 516, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4018, 4180, 4126, 4143, 4067, 4075, 4130, 4029] avg=4096.0000 max=4180 max_rank=1 min=4018 min_rank=0 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16301 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 3968, 4224, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 128, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[493, 476, 508, 502, 505, 493, 489, 525, 491, 482, 513, 527, 514, 519, 517, 519, 503, 564, 525, 520, 529, 483, 530, 505, 476, 518, 517, 521, 493, 519, 525, 540, 514, 510, 526, 499, 505, 523, 515, 492, 510, 525, 530, 521, 547, 513, 525, 513, 531, 482, 527, 513, 502, 509, 505, 487, 511, 484, 497, 524, 513, 527, 541, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[460, 523, 495, 549, 527, 473, 502, 488, 493, 518, 510, 491, 551, 499, 489, 519, 510, 501, 526, 505, 539, 470, 479, 521, 509, 514, 484, 535, 511, 507, 522, 591, 496, 502, 493, 487, 550, 510, 531, 501, 540, 514, 472, 562, 520, 543, 487, 517, 461, 494, 475, 526, 519, 519, 558, 528, 572, 496, 481, 500, 523, 519, 530, 531] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[483, 512, 499, 513, 538, 496, 559, 520, 502, 521, 501, 531, 508, 522, 537, 508, 511, 502, 498, 541, 529, 528, 499, 502, 512, 536, 501, 515, 524, 521, 517, 469, 476, 562, 498, 496, 494, 497, 497, 550, 492, 527, 493, 522, 528, 514, 526, 491, 503, 483, 531, 507, 476, 496, 524, 501, 523, 515, 557, 511, 484, 521, 498, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[524, 456, 538, 530, 533, 553, 486, 544, 540, 490, 540, 509, 519, 506, 548, 498, 492, 528, 528, 503, 508, 496, 539, 511, 517, 504, 505, 522, 513, 552, 511, 488, 486, 477, 516, 512, 518, 501, 482, 502, 535, 488, 505, 503, 521, 533, 502, 557, 491, 481, 540, 488, 494, 509, 463, 553, 471, 502, 486, 509, 515, 523, 538, 536] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[498, 478, 496, 493, 542, 512, 510, 464, 509, 513, 517, 542, 454, 553, 527, 512, 522, 507, 508, 549, 493, 522, 545, 513, 498, 544, 473, 517, 527, 539, 509, 528, 517, 519, 520, 477, 488, 523, 522, 496, 536, 494, 501, 523, 535, 501, 517, 553, 506, 507, 544, 492, 504, 490, 514, 478, 491, 513, 513, 479, 549, 528, 517, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[510, 516, 504, 545, 510, 487, 524, 513, 546, 510, 506, 512, 503, 490, 491, 521, 501, 513, 487, 493, 543, 519, 524, 491, 519, 520, 516, 493, 484, 510, 513, 494, 517, 531, 560, 504, 486, 510, 503, 493, 476, 545, 523, 509, 528, 519, 531, 505, 515, 527, 520, 513, 485, 523, 506, 480, 543, 513, 539, 539, 514, 484, 517, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[487, 493, 509, 515, 497, 507, 511, 517, 522, 524, 462, 509, 498, 492, 512, 500, 503, 504, 485, 518, 493, 525, 561, 506, 481, 477, 467, 574, 502, 465, 519, 501, 481, 508, 504, 493, 557, 509, 525, 526, 498, 512, 488, 503, 549, 542, 544, 506, 533, 531, 515, 499, 553, 534, 496, 514, 463, 553, 498, 527, 563, 531, 547, 530] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[503, 544, 506, 539, 511, 552, 514, 511, 487, 496, 517, 508, 525, 510, 505, 496, 516, 482, 506, 482, 498, 509, 523, 523, 485, 490, 505, 476, 509, 511, 531, 523, 508, 531, 550, 518, 541, 523, 549, 539, 518, 496, 490, 484, 502, 536, 546, 535, 513, 523, 494, 537, 498, 477, 481, 497, 525, 544, 498, 504, 494, 525, 480, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[493, 476, 508, 502, 505, 493, 489, 525, 12, 7, 0, 0, 0, 0, 0, 0, 491, 482, 513, 527, 514, 519, 517, 519, 8, 0, 0, 0, 0, 0, 0, 0, 503, 564, 525, 520, 529, 483, 528, 505, 0, 0, 0, 0, 0, 0, 0, 0, 476, 518, 517, 521, 493, 519, 525, 540, 10, 0, 0, 0, 0, 0, 0, 0, 514, 510, 526, 499, 505, 523, 515, 492, 2, 0, 0, 0, 0, 0, 0, 0, 510, 525, 530, 521, 503, 513, 525, 513, 0, 0, 0, 0, 0, 0, 0, 0, 531, 482, 527, 513, 502, 509, 505, 487, 37, 0, 0, 0, 0, 0, 0, 0, 511, 484, 497, 524, 513, 527, 511, 506, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4010, 4090, 4157, 4119, 4086, 4140, 4093, 4073] avg=4096.0000 max=4157 max_rank=2 min=4010 min_rank=0 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[460, 523, 495, 549, 527, 473, 502, 488, 12, 7, 0, 0, 0, 0, 0, 0, 493, 518, 510, 491, 551, 499, 489, 519, 8, 0, 0, 0, 0, 0, 0, 0, 510, 501, 526, 505, 539, 470, 477, 521, 0, 0, 0, 0, 0, 0, 0, 0, 509, 514, 484, 535, 511, 507, 522, 591, 10, 0, 0, 0, 0, 0, 0, 0, 496, 502, 493, 487, 550, 510, 531, 501, 2, 0, 0, 0, 0, 0, 0, 0, 540, 514, 472, 562, 477, 543, 487, 517, 0, 0, 0, 0, 0, 0, 0, 0, 461, 494, 475, 526, 519, 519, 558, 528, 36, 0, 0, 0, 0, 0, 0, 0, 572, 496, 481, 500, 523, 519, 500, 531, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4036, 4078, 4049, 4183, 4072, 4112, 4116, 4122] avg=4096.0000 max=4183 max_rank=3 min=4036 min_rank=0 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16422 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[483, 512, 499, 513, 538, 496, 559, 520, 11, 7, 0, 0, 0, 0, 0, 0, 502, 521, 501, 531, 508, 522, 537, 508, 8, 0, 0, 0, 0, 0, 0, 0, 511, 502, 498, 541, 529, 528, 497, 502, 0, 0, 0, 0, 0, 0, 0, 0, 512, 536, 501, 515, 524, 521, 517, 469, 9, 0, 0, 0, 0, 0, 0, 0, 476, 562, 498, 496, 494, 497, 497, 550, 2, 0, 0, 0, 0, 0, 0, 0, 492, 527, 493, 522, 485, 514, 526, 491, 0, 0, 0, 0, 0, 0, 0, 0, 503, 483, 531, 507, 476, 496, 524, 501, 36, 0, 0, 0, 0, 0, 0, 0, 523, 515, 557, 511, 484, 521, 470, 520, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4138, 4138, 4108, 4104, 4072, 4050, 4057, 4101] avg=4096.0000 max=4138 max_rank=0 min=4050 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16280 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[487, 493, 509, 515, 497, 507, 511, 517, 12, 7, 0, 0, 0, 0, 0, 0, 522, 524, 462, 509, 498, 492, 512, 500, 8, 0, 0, 0, 0, 0, 0, 0, 503, 504, 485, 518, 493, 525, 559, 506, 0, 0, 0, 0, 0, 0, 0, 0, 481, 477, 467, 574, 502, 465, 519, 501, 10, 0, 0, 0, 0, 0, 0, 0, 481, 508, 504, 493, 557, 509, 525, 526, 2, 0, 0, 0, 0, 0, 0, 0, 498, 512, 488, 503, 504, 542, 544, 506, 0, 0, 0, 0, 0, 0, 0, 0, 533, 531, 515, 499, 553, 534, 496, 514, 38, 0, 0, 0, 0, 0, 0, 0, 463, 553, 498, 527, 563, 531, 517, 530, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4055, 4027, 4093, 3996, 4105, 4097, 4213, 4182] avg=4096.0000 max=4213 max_rank=6 min=3996 min_rank=3 imbalance_ratio=1.028564 max_load_reduction=28555 max_load_reduction_pct=0.871429 moved_to_echo=16597 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[498, 478, 496, 493, 542, 512, 510, 464, 12, 7, 0, 0, 0, 0, 0, 0, 509, 513, 517, 542, 454, 553, 527, 512, 8, 0, 0, 0, 0, 0, 0, 0, 522, 507, 508, 549, 493, 522, 543, 513, 0, 0, 0, 0, 0, 0, 0, 0, 498, 544, 473, 517, 527, 539, 509, 528, 9, 0, 0, 0, 0, 0, 0, 0, 517, 519, 520, 477, 488, 523, 522, 496, 2, 0, 0, 0, 0, 0, 0, 0, 536, 494, 501, 523, 491, 501, 517, 553, 0, 0, 0, 0, 0, 0, 0, 0, 506, 507, 544, 492, 504, 490, 514, 478, 37, 0, 0, 0, 0, 0, 0, 0, 491, 513, 513, 479, 549, 528, 488, 507, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4012, 4135, 4157, 4144, 4064, 4116, 4072, 4068] avg=4096.0000 max=4157 max_rank=2 min=4012 min_rank=0 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16320 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[524, 456, 538, 530, 533, 553, 486, 544, 16, 11, 0, 0, 0, 0, 0, 0, 540, 490, 540, 509, 519, 506, 548, 498, 12, 0, 0, 0, 0, 0, 0, 0, 492, 528, 528, 503, 508, 496, 532, 511, 0, 0, 0, 0, 0, 0, 0, 0, 517, 504, 505, 522, 513, 552, 511, 488, 13, 0, 0, 0, 0, 0, 0, 0, 486, 477, 516, 512, 518, 501, 482, 502, 7, 1, 0, 0, 0, 0, 0, 0, 535, 488, 505, 503, 471, 533, 502, 557, 0, 0, 0, 0, 0, 0, 0, 0, 491, 481, 540, 488, 494, 509, 463, 553, 39, 0, 0, 0, 0, 0, 0, 0, 471, 502, 486, 509, 515, 523, 496, 536, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4191, 4162, 4098, 4125, 4002, 4094, 4058, 4038] avg=4096.0000 max=4191 max_rank=0 min=4002 min_rank=4 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16192 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[510, 516, 504, 545, 510, 487, 524, 513, 12, 7, 0, 0, 0, 0, 0, 0, 546, 510, 506, 512, 503, 490, 491, 521, 8, 0, 0, 0, 0, 0, 0, 0, 501, 513, 487, 493, 543, 519, 522, 491, 0, 0, 0, 0, 0, 0, 0, 0, 519, 520, 516, 493, 484, 510, 513, 494, 9, 0, 0, 0, 0, 0, 0, 0, 517, 531, 560, 504, 486, 510, 503, 493, 2, 0, 0, 0, 0, 0, 0, 0, 476, 545, 523, 509, 485, 519, 531, 505, 0, 0, 0, 0, 0, 0, 0, 0, 515, 527, 520, 513, 485, 523, 506, 480, 36, 0, 0, 0, 0, 0, 0, 0, 543, 513, 539, 539, 514, 484, 488, 502, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4087, 4069, 4058, 4106, 4093, 4105, 4122] avg=4096.0000 max=4128 max_rank=0 min=4058 min_rank=3 imbalance_ratio=1.007812 max_load_reduction=28640 max_load_reduction_pct=0.874023 moved_to_echo=16426 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[503, 544, 506, 539, 511, 552, 514, 511, 11, 7, 0, 0, 0, 0, 0, 0, 487, 496, 517, 508, 525, 510, 505, 496, 7, 0, 0, 0, 0, 0, 0, 0, 516, 482, 506, 482, 498, 509, 521, 523, 0, 0, 0, 0, 0, 0, 0, 0, 485, 490, 505, 476, 509, 511, 531, 523, 9, 0, 0, 0, 0, 0, 0, 0, 508, 531, 550, 518, 541, 523, 549, 539, 2, 0, 0, 0, 0, 0, 0, 0, 518, 496, 490, 484, 461, 536, 546, 535, 0, 0, 0, 0, 0, 0, 0, 0, 513, 523, 494, 537, 498, 477, 481, 497, 34, 0, 0, 0, 0, 0, 0, 0, 525, 544, 498, 504, 494, 525, 453, 519, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4198, 4051, 4037, 4039, 4261, 4066, 4054, 4062] avg=4096.0000 max=4261 max_rank=4 min=4037 min_rank=2 imbalance_ratio=1.040283 max_load_reduction=28507 max_load_reduction_pct=0.869965 moved_to_echo=16443 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 3968, 4224, 4096, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3968, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 3968, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[508, 542, 531, 526, 493, 496, 530, 491, 516, 468, 493, 512, 530, 526, 506, 495, 506, 506, 524, 474, 524, 547, 559, 485, 525, 536, 513, 522, 495, 543, 514, 521, 530, 489, 495, 545, 544, 493, 521, 509, 538, 487, 522, 521, 501, 529, 547, 514, 498, 500, 523, 476, 502, 524, 522, 509, 504, 501, 492, 464, 504, 488, 493, 526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[515, 538, 517, 498, 511, 525, 523, 495, 519, 482, 516, 515, 523, 517, 517, 514, 508, 473, 535, 520, 481, 513, 494, 461, 519, 481, 482, 515, 496, 533, 504, 509, 510, 496, 562, 540, 555, 527, 503, 511, 522, 515, 509, 508, 548, 486, 503, 485, 519, 544, 513, 513, 536, 492, 492, 487, 483, 511, 570, 533, 521, 527, 483, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[469, 495, 548, 503, 503, 510, 528, 535, 500, 508, 526, 488, 524, 523, 544, 484, 515, 514, 524, 518, 491, 528, 492, 508, 561, 524, 521, 520, 516, 503, 510, 526, 504, 482, 544, 511, 520, 516, 451, 519, 503, 500, 498, 483, 515, 480, 514, 528, 539, 518, 558, 527, 527, 500, 473, 497, 496, 543, 487, 544, 509, 471, 511, 541] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[527, 529, 527, 564, 515, 484, 480, 511, 552, 506, 496, 484, 537, 491, 496, 497, 538, 477, 491, 507, 502, 472, 519, 492, 480, 532, 515, 539, 547, 519, 539, 515, 488, 440, 497, 474, 509, 519, 494, 523, 511, 515, 548, 543, 539, 493, 509, 500, 514, 526, 485, 482, 530, 531, 534, 522, 553, 507, 512, 491, 486, 523, 527, 563] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[517, 516, 486, 481, 519, 541, 539, 490, 515, 521, 501, 511, 542, 510, 491, 499, 509, 514, 512, 507, 523, 501, 516, 488, 506, 525, 514, 511, 500, 490, 527, 494, 505, 495, 521, 494, 518, 527, 518, 519, 523, 499, 554, 535, 498, 495, 529, 553, 541, 513, 519, 493, 488, 489, 483, 504, 511, 518, 539, 529, 503, 510, 528, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[516, 496, 494, 515, 550, 476, 512, 571, 512, 508, 504, 479, 479, 529, 526, 490, 498, 486, 509, 526, 537, 504, 499, 522, 499, 516, 515, 507, 504, 497, 525, 525, 516, 549, 496, 494, 519, 532, 492, 481, 525, 506, 522, 512, 508, 520, 536, 491, 513, 509, 538, 559, 458, 551, 560, 518, 516, 492, 541, 486, 463, 525, 481, 533] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[482, 499, 512, 552, 532, 519, 501, 483, 524, 515, 506, 514, 481, 485, 499, 509, 540, 514, 504, 541, 510, 495, 509, 452, 501, 545, 531, 503, 496, 500, 520, 513, 515, 504, 526, 495, 490, 522, 514, 493, 523, 550, 503, 512, 542, 500, 517, 502, 503, 503, 518, 478, 550, 502, 534, 494, 516, 514, 537, 510, 537, 524, 534, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[523, 536, 562, 509, 499, 474, 478, 479, 563, 507, 510, 508, 534, 519, 530, 516, 521, 552, 517, 505, 518, 501, 499, 469, 526, 479, 497, 518, 508, 543, 539, 530, 525, 502, 442, 545, 514, 535, 499, 502, 505, 512, 555, 531, 497, 531, 469, 499, 484, 510, 523, 500, 554, 531, 475, 495, 504, 552, 476, 517, 515, 502, 491, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[515, 538, 510, 498, 511, 525, 523, 495, 0, 0, 0, 0, 0, 0, 0, 0, 519, 482, 516, 515, 523, 517, 517, 514, 6, 5, 4, 0, 0, 0, 0, 0, 508, 473, 535, 520, 481, 513, 494, 461, 25, 7, 0, 0, 0, 0, 0, 0, 519, 481, 482, 515, 496, 533, 479, 509, 0, 0, 0, 0, 0, 0, 0, 0, 510, 496, 562, 540, 555, 527, 503, 511, 16, 1, 0, 0, 0, 0, 0, 0, 522, 515, 486, 508, 548, 486, 503, 485, 0, 0, 0, 0, 0, 0, 0, 0, 519, 544, 509, 513, 536, 492, 492, 487, 0, 0, 0, 0, 0, 0, 0, 0, 483, 511, 570, 533, 521, 527, 483, 510, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4118, 4017, 4014, 4221, 4053, 4092, 4138] avg=4096.0000 max=4221 max_rank=4 min=4014 min_rank=3 imbalance_ratio=1.030518 max_load_reduction=28547 max_load_reduction_pct=0.871185 moved_to_echo=16504 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[508, 542, 524, 526, 493, 496, 530, 491, 0, 0, 0, 0, 0, 0, 0, 0, 516, 468, 493, 512, 530, 526, 506, 495, 6, 5, 4, 0, 0, 0, 0, 0, 506, 506, 524, 474, 524, 547, 559, 485, 25, 7, 0, 0, 0, 0, 0, 0, 525, 536, 513, 522, 495, 543, 489, 521, 0, 0, 0, 0, 0, 0, 0, 0, 530, 489, 495, 545, 544, 493, 521, 509, 17, 1, 0, 0, 0, 0, 0, 0, 538, 487, 498, 521, 501, 529, 547, 514, 0, 0, 0, 0, 0, 0, 0, 0, 498, 500, 519, 476, 502, 524, 522, 509, 0, 0, 0, 0, 0, 0, 0, 0, 504, 501, 492, 464, 504, 488, 493, 521, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4061, 4157, 4144, 4144, 4135, 4050, 3967] avg=4096.0000 max=4157 max_rank=2 min=3967 min_rank=7 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16296 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[527, 529, 520, 564, 515, 484, 480, 511, 0, 0, 0, 0, 0, 0, 0, 0, 552, 506, 496, 484, 537, 491, 496, 497, 6, 5, 4, 0, 0, 0, 0, 0, 538, 477, 491, 507, 502, 472, 519, 492, 27, 7, 0, 0, 0, 0, 0, 0, 480, 532, 515, 539, 547, 519, 512, 515, 0, 0, 0, 0, 0, 0, 0, 0, 488, 440, 497, 474, 509, 519, 494, 523, 18, 1, 0, 0, 0, 0, 0, 0, 511, 515, 523, 543, 539, 493, 509, 500, 0, 0, 0, 0, 0, 0, 0, 0, 514, 526, 481, 482, 530, 531, 534, 522, 0, 0, 0, 0, 0, 0, 0, 0, 553, 507, 512, 491, 486, 523, 527, 558, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4130, 4074, 4032, 4159, 3963, 4133, 4120, 4157] avg=4096.0000 max=4159 max_rank=3 min=3963 min_rank=4 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[469, 495, 541, 503, 503, 510, 528, 535, 0, 0, 0, 0, 0, 0, 0, 0, 500, 508, 526, 488, 524, 523, 544, 484, 6, 5, 4, 0, 0, 0, 0, 0, 515, 514, 524, 518, 491, 528, 492, 508, 25, 7, 0, 0, 0, 0, 0, 0, 561, 524, 521, 520, 516, 503, 485, 526, 0, 0, 0, 0, 0, 0, 0, 0, 504, 482, 544, 511, 520, 516, 451, 519, 16, 1, 0, 0, 0, 0, 0, 0, 503, 500, 475, 483, 515, 480, 514, 528, 0, 0, 0, 0, 0, 0, 0, 0, 539, 518, 554, 527, 527, 500, 473, 497, 0, 0, 0, 0, 0, 0, 0, 0, 496, 543, 487, 544, 509, 471, 511, 536, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4112, 4122, 4156, 4064, 3998, 4135, 4097] avg=4096.0000 max=4156 max_rank=3 min=3998 min_rank=5 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16294 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[482, 499, 505, 552, 532, 519, 501, 483, 0, 0, 0, 0, 0, 0, 0, 0, 524, 515, 506, 514, 481, 485, 499, 509, 6, 5, 4, 0, 0, 0, 0, 0, 540, 514, 504, 541, 510, 495, 509, 452, 26, 7, 0, 0, 0, 0, 0, 0, 501, 545, 531, 503, 496, 500, 494, 513, 0, 0, 0, 0, 0, 0, 0, 0, 515, 504, 526, 495, 490, 522, 514, 493, 16, 1, 0, 0, 0, 0, 0, 0, 523, 550, 480, 512, 542, 500, 517, 502, 0, 0, 0, 0, 0, 0, 0, 0, 503, 503, 514, 478, 550, 502, 534, 494, 0, 0, 0, 0, 0, 0, 0, 0, 516, 514, 537, 510, 537, 524, 534, 514, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4048, 4098, 4083, 4076, 4126, 4078, 4186] avg=4096.0000 max=4186 max_rank=7 min=4048 min_rank=1 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16466 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[516, 496, 484, 515, 550, 476, 512, 571, 0, 0, 0, 0, 0, 0, 0, 0, 512, 508, 504, 479, 479, 529, 526, 490, 8, 7, 7, 0, 0, 0, 0, 0, 498, 486, 509, 526, 537, 504, 499, 522, 30, 10, 0, 0, 0, 0, 0, 0, 499, 516, 515, 507, 504, 497, 495, 525, 0, 0, 0, 0, 0, 0, 0, 0, 516, 549, 496, 494, 519, 532, 492, 481, 20, 2, 0, 0, 0, 0, 0, 0, 525, 506, 492, 512, 508, 520, 536, 491, 0, 0, 0, 0, 0, 0, 0, 0, 513, 509, 531, 559, 458, 551, 560, 518, 0, 0, 0, 0, 0, 0, 0, 0, 516, 492, 541, 486, 463, 525, 481, 526, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4049, 4121, 4058, 4101, 4090, 4199, 4030] avg=4096.0000 max=4199 max_rank=6 min=4030 min_rank=7 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16420 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[517, 516, 480, 481, 519, 541, 539, 490, 0, 0, 0, 0, 0, 0, 0, 0, 515, 521, 501, 511, 542, 510, 491, 499, 5, 5, 4, 0, 0, 0, 0, 0, 509, 514, 512, 507, 523, 501, 516, 488, 26, 8, 0, 0, 0, 0, 0, 0, 506, 525, 514, 511, 500, 490, 501, 494, 0, 0, 0, 0, 0, 0, 0, 0, 505, 495, 521, 494, 518, 527, 518, 519, 18, 1, 0, 0, 0, 0, 0, 0, 523, 499, 528, 535, 498, 495, 529, 553, 0, 0, 0, 0, 0, 0, 0, 0, 541, 513, 515, 493, 488, 489, 483, 504, 0, 0, 0, 0, 0, 0, 0, 0, 511, 518, 539, 529, 503, 510, 528, 496, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4104, 4104, 4041, 4116, 4160, 4026, 4134] avg=4096.0000 max=4160 max_rank=5 min=4026 min_rank=6 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16436 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[523, 536, 555, 509, 499, 474, 478, 479, 0, 0, 0, 0, 0, 0, 0, 0, 563, 507, 510, 508, 534, 519, 530, 516, 6, 5, 4, 0, 0, 0, 0, 0, 521, 552, 517, 505, 518, 501, 499, 469, 27, 8, 0, 0, 0, 0, 0, 0, 526, 479, 497, 518, 508, 543, 512, 530, 0, 0, 0, 0, 0, 0, 0, 0, 525, 502, 442, 545, 514, 535, 499, 502, 18, 1, 0, 0, 0, 0, 0, 0, 505, 512, 529, 531, 497, 531, 469, 499, 0, 0, 0, 0, 0, 0, 0, 0, 484, 510, 519, 500, 554, 531, 475, 495, 0, 0, 0, 0, 0, 0, 0, 0, 504, 552, 476, 517, 515, 502, 491, 502, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4053, 4202, 4117, 4113, 4083, 4073, 4068, 4059] avg=4096.0000 max=4202 max_rank=1 min=4053 min_rank=0 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16283 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 3968, 4096, 4224, 4224, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4096, 3968, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[513, 466, 566, 529, 519, 502, 516, 513, 522, 512, 510, 497, 510, 510, 484, 512, 484, 570, 499, 511, 500, 512, 529, 507, 513, 548, 512, 509, 520, 535, 490, 495, 493, 527, 527, 511, 532, 536, 506, 524, 510, 509, 550, 492, 464, 515, 504, 495, 489, 485, 548, 513, 521, 493, 567, 537, 457, 503, 494, 530, 496, 515, 508, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[478, 515, 466, 536, 493, 523, 510, 479, 497, 564, 541, 527, 455, 521, 559, 493, 551, 519, 525, 535, 519, 492, 542, 508, 486, 554, 529, 494, 482, 559, 497, 493, 523, 494, 491, 512, 485, 492, 512, 525, 531, 464, 504, 504, 490, 479, 562, 500, 511, 505, 519, 532, 489, 519, 519, 535, 505, 538, 504, 543, 494, 511, 531, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[494, 531, 544, 524, 527, 473, 540, 568, 484, 497, 499, 545, 511, 545, 490, 527, 511, 519, 489, 534, 518, 505, 521, 516, 494, 496, 516, 535, 496, 506, 536, 544, 476, 497, 527, 501, 516, 514, 491, 538, 490, 519, 518, 498, 512, 534, 533, 517, 502, 534, 482, 528, 453, 514, 517, 513, 481, 485, 493, 534, 505, 510, 498, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[532, 527, 524, 503, 533, 489, 508, 518, 548, 510, 474, 511, 487, 509, 495, 546, 497, 499, 533, 514, 525, 493, 503, 505, 511, 502, 503, 528, 524, 482, 499, 499, 491, 536, 515, 471, 512, 544, 515, 496, 516, 518, 550, 515, 519, 501, 509, 525, 508, 520, 485, 517, 518, 520, 512, 505, 532, 491, 521, 541, 487, 530, 504, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[529, 510, 521, 502, 508, 522, 504, 492, 506, 507, 496, 476, 535, 534, 515, 515, 560, 548, 483, 532, 513, 547, 512, 517, 510, 525, 512, 475, 504, 472, 527, 532, 539, 478, 526, 483, 495, 523, 519, 514, 514, 517, 518, 527, 545, 484, 501, 518, 497, 502, 538, 509, 492, 500, 515, 517, 505, 560, 526, 484, 504, 508, 498, 471] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[494, 514, 502, 526, 500, 476, 475, 534, 504, 503, 523, 502, 480, 499, 544, 518, 511, 506, 534, 496, 503, 540, 505, 520, 496, 581, 517, 521, 510, 494, 496, 512, 520, 518, 496, 512, 549, 511, 525, 522, 502, 496, 512, 514, 502, 545, 505, 511, 513, 507, 521, 511, 493, 493, 515, 493, 500, 542, 519, 541, 519, 538, 508, 479] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[509, 515, 524, 523, 508, 498, 506, 527, 540, 518, 542, 505, 518, 499, 536, 473, 490, 512, 539, 516, 507, 521, 550, 540, 512, 503, 472, 507, 533, 552, 527, 499, 528, 482, 496, 535, 529, 504, 523, 473, 479, 508, 516, 490, 536, 505, 514, 540, 517, 508, 491, 483, 485, 514, 522, 507, 511, 538, 524, 527, 511, 495, 481, 475] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[463, 493, 473, 500, 489, 524, 537, 530, 537, 485, 486, 470, 527, 550, 519, 516, 499, 467, 479, 529, 540, 492, 512, 526, 500, 525, 482, 542, 548, 533, 539, 518, 525, 470, 517, 476, 524, 531, 502, 521, 535, 515, 518, 482, 546, 502, 500, 485, 532, 498, 533, 505, 500, 499, 542, 544, 526, 513, 497, 511, 520, 537, 521, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[478, 515, 466, 536, 493, 523, 510, 479, 10, 0, 0, 0, 0, 0, 0, 0, 497, 564, 541, 527, 455, 521, 559, 493, 0, 0, 0, 0, 0, 0, 0, 0, 551, 519, 525, 535, 519, 492, 510, 508, 0, 0, 0, 0, 0, 0, 0, 0, 486, 543, 529, 494, 482, 559, 497, 493, 0, 0, 0, 0, 0, 0, 0, 0, 523, 494, 491, 512, 485, 492, 512, 525, 9, 0, 0, 0, 0, 0, 0, 0, 531, 464, 504, 504, 490, 479, 562, 500, 4, 0, 0, 0, 0, 0, 0, 0, 511, 505, 519, 532, 489, 519, 519, 535, 7, 0, 0, 0, 0, 0, 0, 0, 505, 538, 504, 543, 494, 511, 531, 503, 13, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4010, 4157, 4159, 4083, 4043, 4038, 4136, 4142] avg=4096.0000 max=4159 max_rank=2 min=4010 min_rank=0 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16359 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[513, 466, 566, 529, 519, 502, 516, 513, 10, 0, 0, 0, 0, 0, 0, 0, 522, 512, 510, 497, 510, 510, 484, 512, 0, 0, 0, 0, 0, 0, 0, 0, 484, 570, 499, 511, 500, 512, 497, 507, 0, 0, 0, 0, 0, 0, 0, 0, 513, 537, 512, 509, 520, 535, 490, 495, 0, 0, 0, 0, 0, 0, 0, 0, 493, 527, 527, 511, 532, 536, 506, 524, 9, 0, 0, 0, 0, 0, 0, 0, 510, 509, 550, 492, 464, 515, 504, 495, 4, 0, 0, 0, 0, 0, 0, 0, 489, 485, 548, 513, 521, 493, 567, 537, 7, 0, 0, 0, 0, 0, 0, 0, 457, 503, 494, 530, 496, 515, 508, 502, 13, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4134, 4057, 4080, 4111, 4165, 4043, 4160, 4018] avg=4096.0000 max=4165 max_rank=4 min=4018 min_rank=7 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[532, 527, 524, 503, 533, 489, 508, 518, 9, 0, 0, 0, 0, 0, 0, 0, 548, 510, 474, 511, 487, 509, 495, 546, 0, 0, 0, 0, 0, 0, 0, 0, 497, 499, 533, 514, 525, 493, 474, 505, 0, 0, 0, 0, 0, 0, 0, 0, 511, 492, 503, 528, 524, 482, 499, 499, 0, 0, 0, 0, 0, 0, 0, 0, 491, 536, 515, 471, 512, 544, 515, 496, 8, 0, 0, 0, 0, 0, 0, 0, 516, 518, 550, 515, 519, 501, 509, 525, 4, 0, 0, 0, 0, 0, 0, 0, 508, 520, 485, 517, 518, 520, 512, 505, 6, 0, 0, 0, 0, 0, 0, 0, 532, 491, 521, 541, 487, 530, 504, 513, 12, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4143, 4080, 4040, 4038, 4088, 4157, 4091, 4131] avg=4096.0000 max=4157 max_rank=5 min=4038 min_rank=3 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16467 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[494, 531, 544, 524, 527, 473, 540, 568, 10, 0, 0, 0, 0, 0, 0, 0, 484, 497, 499, 545, 511, 545, 490, 527, 0, 0, 0, 0, 0, 0, 0, 0, 511, 519, 489, 534, 518, 505, 490, 516, 0, 0, 0, 0, 0, 0, 0, 0, 494, 486, 516, 535, 496, 506, 536, 544, 0, 0, 0, 0, 0, 0, 0, 0, 476, 497, 527, 501, 516, 514, 491, 538, 8, 0, 0, 0, 0, 0, 0, 0, 490, 519, 518, 498, 512, 534, 533, 517, 4, 0, 0, 0, 0, 0, 0, 0, 502, 534, 482, 528, 453, 514, 517, 513, 6, 0, 0, 0, 0, 0, 0, 0, 481, 485, 493, 534, 505, 510, 498, 503, 13, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4211, 4098, 4082, 4113, 4068, 4125, 4049, 4022] avg=4096.0000 max=4211 max_rank=0 min=4022 min_rank=7 imbalance_ratio=1.028076 max_load_reduction=28557 max_load_reduction_pct=0.871490 moved_to_echo=16264 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[494, 514, 502, 526, 500, 476, 475, 534, 9, 0, 0, 0, 0, 0, 0, 0, 504, 503, 523, 502, 480, 499, 544, 518, 0, 0, 0, 0, 0, 0, 0, 0, 511, 506, 534, 496, 503, 540, 476, 520, 0, 0, 0, 0, 0, 0, 0, 0, 496, 569, 517, 521, 510, 494, 496, 512, 0, 0, 0, 0, 0, 0, 0, 0, 520, 518, 496, 512, 549, 511, 525, 522, 8, 0, 0, 0, 0, 0, 0, 0, 502, 496, 512, 514, 502, 545, 505, 511, 5, 0, 0, 0, 0, 0, 0, 0, 513, 507, 521, 511, 493, 493, 515, 493, 7, 0, 0, 0, 0, 0, 0, 0, 500, 542, 519, 541, 519, 538, 508, 479, 12, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4030, 4073, 4086, 4115, 4161, 4092, 4053, 4158] avg=4096.0000 max=4161 max_rank=4 min=4030 min_rank=0 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16464 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[529, 510, 521, 502, 508, 522, 504, 492, 14, 0, 0, 0, 0, 0, 0, 0, 506, 507, 496, 476, 535, 532, 515, 515, 0, 0, 0, 0, 0, 0, 0, 0, 560, 548, 483, 532, 513, 547, 464, 517, 0, 0, 0, 0, 0, 0, 0, 0, 510, 505, 512, 475, 504, 472, 527, 532, 0, 0, 0, 0, 0, 0, 0, 0, 539, 478, 526, 483, 495, 523, 519, 514, 13, 0, 0, 0, 0, 0, 0, 0, 514, 517, 518, 527, 545, 484, 501, 518, 8, 2, 0, 0, 0, 0, 0, 0, 497, 502, 538, 509, 492, 500, 515, 517, 12, 4, 0, 0, 0, 0, 0, 0, 505, 560, 526, 484, 504, 508, 498, 471, 17, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4082, 4164, 4037, 4090, 4134, 4086, 4073] avg=4096.0000 max=4164 max_rank=2 min=4037 min_rank=3 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[509, 515, 524, 523, 508, 498, 506, 527, 10, 0, 0, 0, 0, 0, 0, 0, 540, 518, 542, 505, 518, 499, 536, 473, 0, 0, 0, 0, 0, 0, 0, 0, 490, 512, 539, 516, 507, 521, 518, 540, 0, 0, 0, 0, 0, 0, 0, 0, 512, 493, 472, 507, 533, 552, 527, 499, 0, 0, 0, 0, 0, 0, 0, 0, 528, 482, 496, 535, 529, 504, 523, 473, 9, 0, 0, 0, 0, 0, 0, 0, 479, 508, 516, 490, 536, 505, 514, 540, 4, 0, 0, 0, 0, 0, 0, 0, 517, 508, 491, 483, 485, 514, 522, 507, 6, 0, 0, 0, 0, 0, 0, 0, 511, 538, 524, 527, 511, 495, 481, 475, 13, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4131, 4143, 4095, 4079, 4092, 4033, 4075] avg=4096.0000 max=4143 max_rank=2 min=4033 min_rank=6 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16279 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[463, 493, 473, 500, 489, 524, 537, 530, 9, 0, 0, 0, 0, 0, 0, 0, 537, 485, 486, 470, 527, 550, 519, 516, 0, 0, 0, 0, 0, 0, 0, 0, 499, 467, 479, 529, 540, 492, 482, 526, 0, 0, 0, 0, 0, 0, 0, 0, 500, 514, 482, 542, 548, 533, 539, 518, 0, 0, 0, 0, 0, 0, 0, 0, 525, 470, 517, 476, 524, 531, 502, 521, 8, 0, 0, 0, 0, 0, 0, 0, 535, 515, 518, 482, 546, 502, 500, 485, 4, 0, 0, 0, 0, 0, 0, 0, 532, 498, 533, 505, 500, 499, 542, 544, 7, 0, 0, 0, 0, 0, 0, 0, 526, 513, 497, 511, 520, 537, 521, 511, 13, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4018, 4090, 4014, 4176, 4074, 4087, 4160, 4149] avg=4096.0000 max=4176 max_rank=3 min=4014 min_rank=2 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16470 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4224, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 3968, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4096, 3968, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[496, 497, 532, 508, 501, 503, 502, 527, 555, 523, 485, 475, 518, 487, 556, 499, 503, 527, 519, 522, 512, 499, 504, 496, 506, 527, 512, 515, 547, 511, 495, 537, 491, 529, 502, 505, 546, 523, 451, 554, 547, 504, 529, 534, 524, 480, 504, 540, 501, 537, 498, 524, 494, 497, 554, 500, 489, 517, 517, 485, 529, 494, 487, 486] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[513, 525, 514, 509, 528, 542, 526, 526, 517, 488, 540, 515, 503, 537, 505, 526, 472, 507, 487, 543, 524, 505, 533, 495, 478, 477, 464, 515, 504, 535, 519, 500, 536, 520, 497, 505, 486, 517, 497, 523, 492, 505, 521, 514, 551, 532, 539, 523, 542, 491, 485, 523, 507, 468, 542, 496, 478, 548, 484, 533, 483, 483, 562, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[522, 493, 494, 519, 548, 493, 511, 471, 513, 517, 520, 519, 483, 523, 500, 517, 523, 510, 515, 500, 565, 513, 504, 504, 490, 524, 513, 519, 486, 523, 498, 506, 520, 502, 529, 524, 510, 511, 478, 500, 481, 537, 509, 504, 514, 509, 516, 517, 522, 481, 543, 525, 503, 504, 542, 497, 510, 533, 527, 530, 552, 537, 482, 483] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[517, 520, 540, 501, 487, 444, 526, 545, 525, 504, 535, 518, 504, 490, 496, 495, 510, 515, 513, 515, 540, 464, 522, 559, 469, 518, 473, 549, 539, 500, 473, 520, 487, 528, 530, 503, 538, 509, 521, 482, 490, 521, 514, 534, 568, 546, 493, 530, 510, 526, 505, 519, 534, 484, 513, 504, 459, 507, 518, 498, 528, 510, 525, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[524, 468, 540, 543, 526, 525, 519, 499, 483, 518, 509, 505, 523, 544, 530, 526, 514, 547, 511, 517, 489, 494, 533, 514, 521, 536, 527, 534, 526, 470, 532, 481, 519, 505, 496, 538, 535, 478, 522, 488, 523, 522, 530, 540, 478, 495, 507, 502, 486, 530, 512, 493, 492, 525, 488, 487, 513, 492, 483, 499, 484, 563, 499, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[501, 530, 517, 500, 521, 474, 495, 497, 462, 554, 495, 520, 500, 517, 516, 502, 547, 513, 513, 492, 487, 502, 500, 539, 541, 523, 524, 504, 491, 536, 521, 502, 533, 506, 501, 499, 534, 489, 506, 474, 527, 517, 548, 528, 515, 480, 490, 548, 523, 508, 517, 559, 531, 531, 509, 540, 482, 496, 506, 527, 508, 501, 511, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[527, 520, 533, 524, 518, 494, 534, 514, 486, 538, 551, 473, 496, 508, 533, 498, 485, 501, 524, 508, 485, 489, 519, 497, 518, 496, 521, 474, 501, 509, 529, 538, 545, 506, 511, 494, 536, 510, 499, 555, 524, 518, 520, 523, 498, 516, 485, 512, 509, 502, 534, 493, 455, 515, 553, 515, 499, 521, 527, 542, 522, 490, 523, 475] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[511, 523, 539, 520, 468, 529, 557, 539, 525, 478, 539, 500, 511, 487, 550, 511, 512, 508, 508, 519, 506, 469, 517, 528, 542, 486, 487, 505, 509, 522, 490, 506, 499, 542, 518, 505, 527, 529, 482, 533, 504, 480, 499, 517, 526, 483, 505, 500, 496, 536, 507, 504, 498, 531, 525, 496, 516, 478, 519, 519, 518, 525, 531, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[496, 497, 515, 508, 501, 503, 502, 527, 0, 0, 0, 0, 0, 0, 0, 0, 555, 523, 485, 475, 518, 487, 555, 499, 0, 0, 0, 0, 0, 0, 0, 0, 503, 527, 519, 522, 512, 499, 504, 496, 6, 1, 0, 0, 0, 0, 0, 0, 506, 527, 512, 515, 547, 511, 495, 537, 11, 4, 0, 0, 0, 0, 0, 0, 491, 529, 502, 505, 546, 523, 451, 554, 0, 0, 0, 0, 0, 0, 0, 0, 547, 504, 529, 507, 524, 480, 504, 540, 0, 0, 0, 0, 0, 0, 0, 0, 501, 537, 498, 524, 494, 497, 554, 500, 0, 0, 0, 0, 0, 0, 0, 0, 489, 517, 517, 485, 529, 494, 487, 486, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4049, 4097, 4089, 4165, 4101, 4135, 4105, 4027] avg=4096.0000 max=4165 max_rank=3 min=4027 min_rank=7 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16368 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[522, 493, 479, 519, 548, 493, 511, 471, 0, 0, 0, 0, 0, 0, 0, 0, 513, 517, 520, 519, 483, 523, 500, 517, 0, 0, 0, 0, 0, 0, 0, 0, 523, 510, 515, 500, 565, 513, 504, 504, 5, 0, 0, 0, 0, 0, 0, 0, 490, 524, 513, 519, 486, 523, 498, 506, 10, 3, 0, 0, 0, 0, 0, 0, 520, 502, 529, 524, 510, 511, 478, 500, 0, 0, 0, 0, 0, 0, 0, 0, 481, 537, 509, 480, 514, 509, 516, 517, 0, 0, 0, 0, 0, 0, 0, 0, 522, 481, 543, 525, 503, 504, 542, 497, 0, 0, 0, 0, 0, 0, 0, 0, 510, 533, 527, 530, 552, 537, 482, 483, 21, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4036, 4092, 4139, 4072, 4074, 4063, 4117, 4175] avg=4096.0000 max=4175 max_rank=7 min=4036 min_rank=0 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16429 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[511, 523, 522, 520, 468, 529, 557, 539, 0, 0, 0, 0, 0, 0, 0, 0, 525, 478, 539, 500, 511, 487, 549, 511, 0, 0, 0, 0, 0, 0, 0, 0, 512, 508, 508, 519, 506, 469, 517, 528, 6, 1, 0, 0, 0, 0, 0, 0, 542, 486, 487, 505, 509, 522, 490, 506, 11, 4, 0, 0, 0, 0, 0, 0, 499, 542, 518, 505, 527, 529, 482, 533, 0, 0, 0, 0, 0, 0, 0, 0, 504, 480, 499, 491, 526, 483, 505, 500, 0, 0, 0, 0, 0, 0, 0, 0, 496, 536, 507, 504, 498, 531, 525, 496, 0, 0, 0, 0, 0, 0, 0, 0, 516, 478, 519, 519, 518, 525, 531, 519, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4169, 4100, 4074, 4062, 4135, 3988, 4093, 4147] avg=4096.0000 max=4169 max_rank=0 min=3988 min_rank=5 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16363 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[524, 468, 523, 543, 526, 525, 519, 499, 0, 0, 0, 0, 0, 0, 0, 0, 483, 518, 509, 505, 523, 544, 529, 526, 0, 0, 0, 0, 0, 0, 0, 0, 514, 547, 511, 517, 489, 494, 533, 514, 6, 1, 0, 0, 0, 0, 0, 0, 521, 536, 527, 534, 526, 470, 532, 481, 11, 4, 0, 0, 0, 0, 0, 0, 519, 505, 496, 538, 535, 478, 522, 488, 0, 0, 0, 0, 0, 0, 0, 0, 523, 522, 530, 513, 478, 495, 507, 502, 0, 0, 0, 0, 0, 0, 0, 0, 486, 530, 512, 493, 492, 525, 488, 487, 0, 0, 0, 0, 0, 0, 0, 0, 513, 492, 483, 499, 484, 563, 499, 516, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4127, 4137, 4126, 4142, 4081, 4070, 4013, 4072] avg=4096.0000 max=4142 max_rank=3 min=4013 min_rank=6 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16236 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[513, 525, 490, 509, 528, 542, 526, 526, 0, 0, 0, 0, 0, 0, 0, 0, 517, 488, 540, 515, 503, 537, 501, 526, 0, 0, 0, 0, 0, 0, 0, 0, 472, 507, 487, 543, 524, 505, 533, 495, 9, 4, 3, 0, 0, 0, 0, 0, 478, 477, 464, 515, 504, 535, 519, 500, 15, 6, 0, 0, 0, 0, 0, 0, 536, 520, 497, 505, 486, 517, 497, 523, 0, 0, 0, 0, 0, 0, 0, 0, 492, 505, 521, 483, 551, 532, 539, 523, 0, 0, 0, 0, 0, 0, 0, 0, 542, 491, 485, 523, 507, 468, 539, 496, 0, 0, 0, 0, 0, 0, 0, 0, 478, 548, 484, 533, 483, 483, 562, 513, 25, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4159, 4127, 4082, 4013, 4081, 4146, 4051, 4109] avg=4096.0000 max=4159 max_rank=0 min=4013 min_rank=3 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16387 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[517, 520, 523, 501, 487, 444, 526, 545, 0, 0, 0, 0, 0, 0, 0, 0, 525, 504, 535, 518, 504, 490, 496, 495, 0, 0, 0, 0, 0, 0, 0, 0, 510, 515, 513, 515, 540, 464, 522, 559, 6, 0, 0, 0, 0, 0, 0, 0, 469, 518, 473, 549, 539, 500, 473, 520, 11, 4, 0, 0, 0, 0, 0, 0, 487, 528, 530, 503, 538, 509, 521, 482, 0, 0, 0, 0, 0, 0, 0, 0, 490, 521, 514, 507, 568, 546, 493, 530, 0, 0, 0, 0, 0, 0, 0, 0, 510, 526, 505, 519, 534, 484, 513, 504, 0, 0, 0, 0, 0, 0, 0, 0, 459, 507, 518, 498, 528, 510, 525, 508, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4067, 4144, 4056, 4098, 4169, 4095, 4076] avg=4096.0000 max=4169 max_rank=5 min=4056 min_rank=3 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16438 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[527, 520, 516, 524, 518, 494, 534, 514, 0, 0, 0, 0, 0, 0, 0, 0, 486, 538, 551, 473, 496, 508, 532, 498, 0, 0, 0, 0, 0, 0, 0, 0, 485, 501, 524, 508, 485, 489, 519, 497, 6, 1, 0, 0, 0, 0, 0, 0, 518, 496, 521, 474, 501, 509, 529, 538, 11, 4, 0, 0, 0, 0, 0, 0, 545, 506, 511, 494, 536, 510, 499, 555, 0, 0, 0, 0, 0, 0, 0, 0, 524, 518, 520, 497, 498, 516, 485, 512, 0, 0, 0, 0, 0, 0, 0, 0, 509, 502, 534, 493, 455, 515, 553, 515, 0, 0, 0, 0, 0, 0, 0, 0, 499, 521, 527, 542, 522, 490, 523, 475, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4147, 4082, 4015, 4101, 4156, 4070, 4076, 4121] avg=4096.0000 max=4156 max_rank=4 min=4015 min_rank=2 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16423 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[501, 530, 500, 500, 521, 474, 495, 497, 0, 0, 0, 0, 0, 0, 0, 0, 462, 554, 495, 520, 500, 517, 516, 502, 0, 0, 0, 0, 0, 0, 0, 0, 547, 513, 513, 492, 487, 502, 500, 539, 6, 0, 0, 0, 0, 0, 0, 0, 541, 523, 524, 504, 491, 536, 521, 502, 11, 4, 0, 0, 0, 0, 0, 0, 533, 506, 501, 499, 534, 489, 506, 474, 0, 0, 0, 0, 0, 0, 0, 0, 527, 517, 548, 502, 515, 480, 490, 548, 0, 0, 0, 0, 0, 0, 0, 0, 523, 508, 517, 559, 531, 531, 509, 540, 0, 0, 0, 0, 0, 0, 0, 0, 482, 496, 506, 527, 508, 501, 511, 488, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4018, 4066, 4099, 4157, 4042, 4127, 4218, 4041] avg=4096.0000 max=4218 max_rank=6 min=4018 min_rank=0 imbalance_ratio=1.029785 max_load_reduction=28550 max_load_reduction_pct=0.871277 moved_to_echo=16428 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 3968, 4224, 4224, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3968, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[510, 490, 557, 540, 526, 488, 493, 479, 517, 513, 483, 483, 515, 520, 491, 516, 538, 497, 517, 490, 503, 529, 546, 510, 519, 491, 546, 518, 539, 524, 511, 502, 530, 509, 486, 530, 511, 497, 516, 507, 515, 472, 559, 527, 495, 508, 509, 489, 476, 531, 488, 502, 467, 505, 543, 526, 552, 463, 541, 498, 523, 535, 489, 568] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[499, 541, 478, 485, 505, 527, 482, 477, 509, 511, 509, 497, 530, 528, 513, 523, 453, 500, 512, 560, 532, 474, 538, 483, 522, 528, 482, 528, 523, 526, 451, 500, 545, 485, 504, 535, 517, 489, 524, 500, 514, 563, 540, 550, 495, 491, 524, 507, 536, 523, 523, 525, 477, 528, 514, 514, 532, 504, 461, 499, 530, 545, 523, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[492, 475, 534, 494, 503, 521, 530, 494, 538, 541, 481, 507, 490, 475, 493, 503, 499, 492, 506, 517, 540, 494, 499, 551, 504, 483, 523, 539, 524, 510, 537, 555, 499, 510, 479, 501, 505, 478, 532, 555, 514, 506, 500, 535, 499, 453, 494, 514, 501, 534, 507, 490, 543, 532, 525, 536, 464, 536, 501, 544, 566, 548, 511, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[507, 501, 495, 488, 514, 519, 503, 546, 479, 514, 495, 495, 497, 516, 487, 500, 486, 503, 501, 540, 528, 535, 533, 521, 512, 498, 533, 500, 509, 515, 538, 495, 525, 525, 517, 532, 551, 520, 496, 518, 503, 503, 493, 522, 550, 518, 484, 506, 544, 458, 524, 527, 493, 520, 481, 491, 530, 513, 495, 532, 514, 520, 510, 550] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[530, 532, 533, 517, 514, 534, 470, 476, 533, 490, 510, 481, 474, 532, 514, 522, 522, 509, 510, 486, 487, 506, 526, 545, 512, 501, 523, 540, 485, 503, 526, 511, 497, 523, 501, 490, 514, 491, 496, 524, 496, 509, 506, 513, 493, 558, 506, 478, 532, 506, 544, 501, 525, 514, 524, 517, 535, 553, 523, 516, 470, 514, 531, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[513, 513, 495, 496, 562, 527, 513, 565, 496, 524, 504, 483, 503, 504, 462, 507, 555, 518, 500, 485, 492, 485, 516, 539, 479, 471, 538, 532, 542, 504, 511, 502, 493, 502, 505, 555, 513, 532, 550, 509, 522, 485, 529, 506, 486, 520, 533, 500, 509, 546, 515, 516, 518, 513, 522, 560, 519, 500, 502, 477, 473, 510, 485, 527] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[509, 533, 525, 515, 530, 501, 538, 495, 524, 520, 485, 515, 523, 527, 490, 494, 524, 496, 500, 494, 522, 508, 506, 503, 510, 521, 485, 496, 514, 544, 513, 518, 501, 525, 499, 525, 528, 489, 515, 503, 513, 506, 487, 516, 510, 500, 551, 534, 492, 517, 536, 512, 512, 503, 552, 518, 515, 478, 524, 536, 478, 483, 489, 543] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[516, 497, 494, 535, 532, 507, 505, 474, 484, 527, 509, 522, 491, 518, 445, 517, 557, 535, 530, 520, 535, 542, 517, 504, 535, 489, 514, 526, 518, 511, 538, 515, 529, 567, 548, 493, 520, 496, 507, 495, 509, 508, 523, 508, 541, 483, 522, 491, 487, 533, 496, 517, 513, 494, 486, 510, 518, 540, 492, 519, 514, 510, 454, 486] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[510, 490, 557, 540, 526, 488, 493, 479, 7, 2, 0, 0, 0, 0, 0, 0, 517, 513, 483, 483, 515, 520, 491, 516, 19, 17, 17, 4, 0, 0, 0, 0, 538, 497, 517, 490, 503, 529, 529, 510, 0, 0, 0, 0, 0, 0, 0, 0, 519, 491, 546, 501, 539, 524, 511, 502, 0, 0, 0, 0, 0, 0, 0, 0, 530, 509, 486, 519, 511, 497, 516, 507, 0, 0, 0, 0, 0, 0, 0, 0, 515, 472, 559, 527, 495, 508, 509, 489, 9, 0, 0, 0, 0, 0, 0, 0, 476, 531, 488, 502, 467, 505, 543, 507, 0, 0, 0, 0, 0, 0, 0, 0, 552, 463, 541, 498, 523, 535, 489, 557, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4095, 4113, 4133, 4075, 4083, 4019, 4158] avg=4096.0000 max=4158 max_rank=7 min=4019 min_rank=6 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16335 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[492, 475, 534, 494, 503, 521, 530, 494, 7, 2, 0, 0, 0, 0, 0, 0, 538, 541, 481, 507, 490, 475, 493, 503, 20, 18, 15, 4, 0, 0, 0, 0, 499, 492, 506, 517, 540, 494, 484, 551, 0, 0, 0, 0, 0, 0, 0, 0, 504, 483, 523, 521, 524, 510, 537, 555, 0, 0, 0, 0, 0, 0, 0, 0, 499, 510, 479, 490, 505, 478, 532, 555, 0, 0, 0, 0, 0, 0, 0, 0, 514, 506, 500, 535, 499, 453, 494, 514, 8, 0, 0, 0, 0, 0, 0, 0, 501, 534, 507, 490, 543, 532, 525, 516, 0, 0, 0, 0, 0, 0, 0, 0, 464, 536, 501, 544, 566, 548, 511, 502, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4052, 4085, 4083, 4157, 4048, 4023, 4148, 4172] avg=4096.0000 max=4172 max_rank=7 min=4023 min_rank=5 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[499, 541, 478, 485, 505, 527, 482, 477, 7, 2, 0, 0, 0, 0, 0, 0, 509, 511, 509, 497, 530, 528, 513, 523, 19, 18, 17, 4, 0, 0, 0, 0, 453, 500, 512, 560, 532, 474, 521, 483, 0, 0, 0, 0, 0, 0, 0, 0, 522, 528, 482, 510, 523, 526, 451, 500, 0, 0, 0, 0, 0, 0, 0, 0, 545, 485, 504, 524, 517, 489, 524, 500, 0, 0, 0, 0, 0, 0, 0, 0, 514, 563, 540, 550, 495, 491, 524, 507, 9, 0, 0, 0, 0, 0, 0, 0, 536, 523, 523, 525, 477, 528, 514, 495, 0, 0, 0, 0, 0, 0, 0, 0, 532, 504, 461, 499, 530, 545, 523, 514, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4003, 4178, 4035, 4042, 4088, 4193, 4121, 4108] avg=4096.0000 max=4193 max_rank=5 min=4003 min_rank=0 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16510 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[509, 533, 525, 515, 530, 501, 538, 495, 7, 2, 0, 0, 0, 0, 0, 0, 524, 520, 485, 515, 523, 527, 490, 494, 19, 17, 16, 4, 0, 0, 0, 0, 524, 496, 500, 494, 522, 508, 490, 503, 0, 0, 0, 0, 0, 0, 0, 0, 510, 521, 485, 479, 514, 544, 513, 518, 0, 0, 0, 0, 0, 0, 0, 0, 501, 525, 499, 514, 528, 489, 515, 503, 0, 0, 0, 0, 0, 0, 0, 0, 513, 506, 487, 516, 510, 500, 551, 534, 9, 0, 0, 0, 0, 0, 0, 0, 492, 517, 536, 512, 512, 503, 552, 499, 0, 0, 0, 0, 0, 0, 0, 0, 515, 478, 524, 536, 478, 483, 489, 532, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4155, 4134, 4037, 4084, 4074, 4126, 4123, 4035] avg=4096.0000 max=4155 max_rank=0 min=4035 min_rank=7 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16358 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[513, 513, 495, 496, 562, 527, 513, 565, 12, 6, 0, 0, 0, 0, 0, 0, 496, 524, 504, 483, 503, 504, 462, 507, 23, 21, 20, 8, 0, 0, 0, 0, 555, 518, 500, 485, 492, 485, 496, 539, 0, 0, 0, 0, 0, 0, 0, 0, 479, 471, 538, 511, 542, 504, 511, 502, 0, 0, 0, 0, 0, 0, 0, 0, 493, 502, 505, 535, 513, 532, 550, 509, 0, 0, 0, 0, 0, 0, 0, 0, 522, 485, 529, 506, 486, 520, 533, 500, 13, 0, 0, 0, 0, 0, 0, 0, 509, 546, 515, 516, 518, 513, 522, 537, 0, 0, 0, 0, 0, 0, 0, 0, 519, 500, 502, 477, 473, 510, 485, 508, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4202, 4055, 4070, 4058, 4139, 4094, 4176, 3974] avg=4096.0000 max=4202 max_rank=0 min=3974 min_rank=7 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[530, 532, 533, 517, 514, 534, 470, 476, 6, 2, 0, 0, 0, 0, 0, 0, 533, 490, 510, 481, 474, 532, 514, 522, 19, 18, 16, 4, 0, 0, 0, 0, 522, 509, 510, 486, 487, 506, 510, 545, 0, 0, 0, 0, 0, 0, 0, 0, 512, 501, 523, 522, 485, 503, 526, 511, 0, 0, 0, 0, 0, 0, 0, 0, 497, 523, 501, 480, 514, 491, 496, 524, 0, 0, 0, 0, 0, 0, 0, 0, 496, 509, 506, 513, 493, 558, 506, 478, 9, 0, 0, 0, 0, 0, 0, 0, 532, 506, 544, 501, 525, 514, 524, 498, 0, 0, 0, 0, 0, 0, 0, 0, 535, 553, 523, 516, 470, 514, 531, 503, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4113, 4075, 4083, 4026, 4068, 4144, 4145] avg=4096.0000 max=4145 max_rank=7 min=4026 min_rank=4 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[516, 497, 494, 535, 532, 507, 505, 474, 6, 2, 0, 0, 0, 0, 0, 0, 484, 527, 509, 522, 491, 518, 445, 517, 19, 18, 16, 4, 0, 0, 0, 0, 557, 535, 530, 520, 535, 542, 501, 504, 0, 0, 0, 0, 0, 0, 0, 0, 535, 489, 514, 508, 518, 511, 538, 515, 0, 0, 0, 0, 0, 0, 0, 0, 529, 567, 548, 483, 520, 496, 507, 495, 0, 0, 0, 0, 0, 0, 0, 0, 509, 508, 523, 508, 541, 483, 522, 491, 8, 0, 0, 0, 0, 0, 0, 0, 487, 533, 496, 517, 513, 494, 486, 491, 0, 0, 0, 0, 0, 0, 0, 0, 518, 540, 492, 519, 514, 510, 454, 476, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4070, 4224, 4128, 4145, 4093, 4017, 4023] avg=4096.0000 max=4224 max_rank=2 min=4017 min_rank=6 imbalance_ratio=1.031250 max_load_reduction=28544 max_load_reduction_pct=0.871094 moved_to_echo=16278 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[507, 501, 495, 488, 514, 519, 503, 546, 7, 2, 0, 0, 0, 0, 0, 0, 479, 514, 495, 495, 497, 516, 487, 500, 18, 17, 16, 4, 0, 0, 0, 0, 486, 503, 501, 540, 528, 535, 517, 521, 0, 0, 0, 0, 0, 0, 0, 0, 512, 498, 533, 483, 509, 515, 538, 495, 0, 0, 0, 0, 0, 0, 0, 0, 525, 525, 517, 521, 551, 520, 496, 518, 0, 0, 0, 0, 0, 0, 0, 0, 503, 503, 493, 522, 550, 518, 484, 506, 9, 0, 0, 0, 0, 0, 0, 0, 544, 458, 524, 527, 493, 520, 481, 473, 0, 0, 0, 0, 0, 0, 0, 0, 530, 513, 495, 532, 514, 520, 510, 539, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4038, 4131, 4083, 4173, 4088, 4020, 4153] avg=4096.0000 max=4173 max_rank=4 min=4020 min_rank=6 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16434 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 3968, 4096, 256, 256, 256, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33792 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[522, 502, 507, 476, 515, 487, 515, 550, 514, 511, 499, 530, 494, 536, 502, 516, 487, 491, 534, 526, 455, 527, 515, 511, 472, 503, 532, 508, 522, 548, 521, 506, 510, 531, 569, 517, 518, 500, 482, 524, 467, 553, 520, 512, 479, 557, 540, 536, 493, 481, 535, 472, 495, 468, 515, 510, 491, 502, 492, 526, 536, 529, 539, 535] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[546, 463, 525, 513, 528, 516, 542, 522, 495, 477, 510, 485, 498, 497, 511, 500, 532, 515, 518, 499, 462, 514, 550, 503, 497, 510, 529, 497, 511, 536, 537, 494, 523, 510, 502, 499, 518, 503, 518, 501, 516, 487, 550, 506, 532, 544, 550, 542, 510, 514, 560, 531, 508, 514, 505, 508, 497, 483, 519, 480, 498, 482, 528, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[456, 523, 519, 515, 520, 544, 513, 490, 504, 531, 513, 521, 518, 513, 549, 510, 533, 511, 489, 492, 512, 494, 482, 514, 488, 544, 518, 519, 529, 532, 522, 505, 520, 553, 529, 534, 515, 492, 527, 488, 494, 504, 487, 490, 536, 517, 498, 455, 545, 507, 483, 482, 539, 480, 536, 508, 497, 551, 492, 548, 540, 464, 525, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[463, 490, 504, 490, 544, 518, 552, 521, 537, 507, 487, 501, 528, 538, 482, 541, 537, 514, 496, 531, 481, 517, 485, 525, 531, 519, 506, 514, 512, 475, 520, 489, 510, 491, 523, 496, 541, 522, 501, 513, 510, 498, 538, 506, 480, 522, 486, 531, 487, 529, 532, 490, 516, 539, 488, 513, 510, 506, 530, 528, 500, 512, 519, 546] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[525, 525, 516, 505, 509, 506, 532, 577, 538, 495, 526, 473, 522, 503, 511, 520, 535, 499, 505, 538, 525, 488, 495, 515, 503, 503, 545, 495, 512, 500, 477, 504, 501, 496, 515, 500, 516, 491, 477, 497, 504, 452, 500, 526, 515, 516, 534, 500, 513, 529, 507, 556, 517, 495, 524, 556, 521, 534, 504, 503, 500, 513, 537, 497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[492, 528, 517, 462, 512, 549, 520, 488, 508, 542, 504, 508, 500, 499, 487, 531, 550, 497, 493, 516, 549, 511, 524, 528, 484, 527, 494, 486, 511, 500, 493, 536, 505, 497, 518, 503, 539, 480, 523, 522, 512, 514, 503, 493, 477, 512, 493, 489, 489, 505, 529, 530, 499, 482, 498, 535, 493, 536, 524, 556, 556, 571, 520, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[513, 532, 521, 527, 555, 513, 502, 503, 522, 490, 521, 497, 508, 517, 494, 519, 537, 533, 541, 511, 495, 513, 534, 504, 493, 504, 518, 515, 528, 475, 523, 533, 509, 422, 500, 502, 511, 491, 555, 516, 497, 562, 489, 529, 534, 499, 531, 530, 491, 469, 522, 545, 535, 499, 499, 492, 508, 497, 522, 482, 493, 526, 513, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[539, 513, 507, 509, 502, 531, 496, 506, 512, 518, 501, 492, 504, 501, 500, 494, 512, 529, 478, 521, 553, 484, 515, 502, 491, 539, 482, 494, 544, 483, 540, 531, 530, 512, 520, 498, 502, 535, 542, 534, 524, 512, 536, 484, 507, 523, 502, 463, 520, 531, 498, 518, 529, 511, 533, 526, 483, 512, 515, 505, 497, 519, 512, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[522, 502, 507, 476, 496, 487, 515, 550, 0, 0, 0, 0, 0, 0, 0, 0, 514, 511, 499, 530, 494, 536, 502, 516, 19, 0, 0, 0, 0, 0, 0, 0, 486, 491, 534, 526, 455, 527, 515, 511, 0, 0, 0, 0, 0, 0, 0, 0, 472, 503, 532, 508, 522, 548, 521, 506, 7, 0, 0, 0, 0, 0, 0, 0, 510, 531, 569, 517, 518, 500, 482, 524, 1, 1, 0, 0, 0, 0, 0, 0, 467, 553, 520, 512, 479, 557, 540, 536, 8, 0, 0, 0, 0, 0, 0, 0, 493, 481, 535, 472, 495, 468, 515, 510, 0, 0, 0, 0, 0, 0, 0, 0, 491, 502, 492, 526, 536, 529, 523, 535, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4055, 4121, 4045, 4119, 4153, 4172, 3969, 4134] avg=4096.0000 max=4172 max_rank=5 min=3969 min_rank=6 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16428 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[456, 523, 519, 515, 501, 544, 513, 490, 0, 0, 0, 0, 0, 0, 0, 0, 504, 531, 513, 521, 518, 513, 549, 510, 19, 0, 0, 0, 0, 0, 0, 0, 532, 511, 489, 492, 512, 494, 482, 514, 0, 0, 0, 0, 0, 0, 0, 0, 488, 544, 518, 519, 529, 532, 522, 505, 7, 0, 0, 0, 0, 0, 0, 0, 520, 553, 529, 534, 515, 492, 527, 488, 1, 1, 0, 0, 0, 0, 0, 0, 494, 504, 487, 490, 536, 517, 498, 455, 7, 0, 0, 0, 0, 0, 0, 0, 545, 507, 483, 482, 539, 480, 536, 508, 0, 0, 0, 0, 0, 0, 0, 0, 497, 551, 492, 548, 540, 464, 510, 509, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4178, 4026, 4164, 4160, 3988, 4080, 4111] avg=4096.0000 max=4178 max_rank=1 min=3988 min_rank=5 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16339 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[546, 463, 525, 513, 509, 516, 542, 522, 0, 0, 0, 0, 0, 0, 0, 0, 495, 477, 510, 485, 498, 497, 511, 500, 19, 0, 0, 0, 0, 0, 0, 0, 531, 515, 518, 499, 462, 514, 550, 503, 0, 0, 0, 0, 0, 0, 0, 0, 497, 510, 529, 497, 511, 536, 537, 494, 7, 0, 0, 0, 0, 0, 0, 0, 523, 510, 502, 499, 518, 503, 518, 501, 1, 1, 0, 0, 0, 0, 0, 0, 516, 487, 550, 506, 532, 544, 550, 542, 7, 0, 0, 0, 0, 0, 0, 0, 510, 514, 560, 531, 508, 514, 505, 508, 0, 0, 0, 0, 0, 0, 0, 0, 497, 483, 519, 480, 498, 482, 513, 498, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4136, 3992, 4092, 4118, 4076, 4234, 4150, 3970] avg=4096.0000 max=4234 max_rank=5 min=3970 min_rank=7 imbalance_ratio=1.033691 max_load_reduction=28534 max_load_reduction_pct=0.870789 moved_to_echo=16430 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[539, 513, 507, 509, 484, 531, 496, 506, 0, 0, 0, 0, 0, 0, 0, 0, 512, 518, 501, 492, 504, 501, 500, 494, 18, 0, 0, 0, 0, 0, 0, 0, 511, 529, 478, 521, 553, 484, 515, 502, 0, 0, 0, 0, 0, 0, 0, 0, 491, 539, 482, 494, 544, 483, 540, 531, 7, 0, 0, 0, 0, 0, 0, 0, 530, 512, 520, 498, 502, 535, 542, 534, 1, 0, 0, 0, 0, 0, 0, 0, 524, 512, 536, 484, 507, 523, 502, 463, 7, 0, 0, 0, 0, 0, 0, 0, 520, 531, 498, 518, 529, 511, 533, 526, 0, 0, 0, 0, 0, 0, 0, 0, 483, 512, 515, 505, 497, 519, 498, 512, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4040, 4093, 4111, 4174, 4058, 4166, 4041] avg=4096.0000 max=4174 max_rank=4 min=4040 min_rank=1 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16439 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[463, 490, 504, 490, 524, 518, 552, 521, 0, 0, 0, 0, 0, 0, 0, 0, 537, 507, 487, 501, 528, 538, 482, 541, 20, 0, 0, 0, 0, 0, 0, 0, 536, 514, 496, 531, 481, 517, 485, 525, 0, 0, 0, 0, 0, 0, 0, 0, 531, 519, 506, 514, 512, 475, 520, 489, 7, 0, 0, 0, 0, 0, 0, 0, 510, 491, 523, 496, 541, 522, 501, 513, 1, 0, 0, 0, 0, 0, 0, 0, 510, 498, 538, 506, 480, 522, 486, 531, 7, 0, 0, 0, 0, 0, 0, 0, 487, 529, 532, 490, 516, 539, 488, 513, 0, 0, 0, 0, 0, 0, 0, 0, 510, 506, 530, 528, 500, 512, 505, 546, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4062, 4141, 4085, 4073, 4098, 4078, 4094, 4137] avg=4096.0000 max=4141 max_rank=1 min=4062 min_rank=0 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16407 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[525, 525, 516, 505, 487, 506, 532, 577, 0, 0, 0, 0, 0, 0, 0, 0, 538, 495, 526, 473, 522, 503, 511, 520, 22, 1, 0, 0, 0, 0, 0, 0, 528, 499, 505, 538, 525, 488, 495, 515, 0, 0, 0, 0, 0, 0, 0, 0, 503, 503, 545, 495, 512, 500, 477, 504, 10, 0, 0, 0, 0, 0, 0, 0, 501, 496, 515, 500, 516, 491, 477, 497, 7, 5, 7, 0, 0, 0, 0, 0, 504, 452, 500, 526, 515, 516, 534, 500, 13, 0, 0, 0, 0, 0, 0, 0, 513, 529, 500, 556, 517, 495, 524, 556, 0, 0, 0, 0, 0, 0, 0, 0, 521, 534, 504, 503, 500, 513, 508, 497, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4173, 4111, 4093, 4049, 4012, 4060, 4190, 4080] avg=4096.0000 max=4190 max_rank=6 min=4012 min_rank=4 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16342 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[513, 532, 521, 527, 535, 513, 502, 503, 0, 0, 0, 0, 0, 0, 0, 0, 522, 490, 521, 497, 508, 517, 494, 519, 20, 0, 0, 0, 0, 0, 0, 0, 536, 533, 541, 511, 495, 513, 534, 504, 0, 0, 0, 0, 0, 0, 0, 0, 493, 504, 518, 515, 528, 475, 523, 533, 7, 0, 0, 0, 0, 0, 0, 0, 509, 422, 500, 502, 511, 491, 555, 516, 1, 0, 0, 0, 0, 0, 0, 0, 497, 562, 489, 529, 534, 499, 531, 530, 7, 0, 0, 0, 0, 0, 0, 0, 491, 469, 522, 545, 535, 499, 499, 492, 0, 0, 0, 0, 0, 0, 0, 0, 508, 497, 522, 482, 493, 526, 499, 507, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4146, 4088, 4167, 4096, 4007, 4178, 4052, 4034] avg=4096.0000 max=4178 max_rank=5 min=4007 min_rank=4 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16271 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[492, 528, 517, 462, 494, 549, 520, 488, 0, 0, 0, 0, 0, 0, 0, 0, 508, 542, 504, 508, 500, 499, 487, 531, 18, 0, 0, 0, 0, 0, 0, 0, 549, 497, 493, 516, 549, 511, 524, 528, 0, 0, 0, 0, 0, 0, 0, 0, 484, 527, 494, 486, 511, 500, 493, 536, 7, 0, 0, 0, 0, 0, 0, 0, 505, 497, 518, 503, 539, 480, 523, 522, 1, 0, 0, 0, 0, 0, 0, 0, 512, 514, 503, 493, 477, 512, 493, 489, 7, 0, 0, 0, 0, 0, 0, 0, 489, 505, 529, 530, 499, 482, 498, 535, 0, 0, 0, 0, 0, 0, 0, 0, 493, 536, 524, 556, 556, 571, 506, 519, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4050, 4097, 4167, 4038, 4088, 4000, 4067, 4261] avg=4096.0000 max=4261 max_rank=7 min=4000 min_rank=5 imbalance_ratio=1.040283 max_load_reduction=28507 max_load_reduction_pct=0.869965 moved_to_echo=16416 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3968, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[511, 525, 500, 568, 464, 507, 530, 538, 517, 522, 505, 513, 474, 504, 499, 519, 485, 505, 545, 536, 502, 478, 494, 502, 502, 514, 478, 510, 490, 560, 525, 540, 524, 533, 500, 524, 537, 477, 523, 534, 535, 512, 527, 513, 487, 493, 504, 505, 485, 506, 516, 511, 517, 472, 520, 528, 509, 465, 515, 529, 520, 524, 528, 533] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[509, 533, 488, 493, 528, 492, 510, 524, 544, 529, 515, 513, 501, 520, 487, 546, 547, 483, 537, 494, 502, 529, 510, 457, 520, 510, 467, 493, 537, 515, 489, 558, 497, 516, 513, 522, 542, 531, 505, 491, 516, 499, 528, 471, 515, 507, 488, 514, 503, 516, 548, 549, 516, 500, 497, 531, 544, 498, 519, 509, 490, 514, 488, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[481, 529, 546, 481, 521, 527, 526, 534, 533, 506, 494, 530, 548, 538, 525, 516, 493, 485, 501, 511, 471, 510, 511, 518, 487, 540, 513, 513, 509, 499, 490, 540, 523, 508, 517, 514, 516, 542, 497, 541, 529, 516, 469, 488, 531, 491, 500, 517, 539, 512, 474, 528, 481, 516, 484, 484, 520, 500, 546, 503, 551, 523, 496, 486] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[526, 529, 511, 491, 520, 557, 503, 533, 516, 513, 529, 507, 518, 493, 514, 480, 496, 541, 529, 528, 475, 484, 516, 535, 533, 486, 505, 524, 528, 557, 529, 515, 517, 498, 527, 527, 515, 519, 501, 491, 490, 513, 548, 467, 515, 523, 460, 523, 508, 533, 514, 487, 474, 496, 455, 502, 524, 498, 503, 528, 509, 527, 510, 545] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[514, 522, 529, 501, 494, 541, 488, 510, 534, 507, 494, 521, 501, 514, 493, 510, 483, 543, 530, 489, 517, 509, 529, 476, 517, 536, 504, 497, 517, 502, 560, 514, 538, 483, 488, 481, 477, 515, 549, 499, 522, 545, 528, 513, 516, 510, 508, 506, 449, 527, 503, 524, 513, 508, 507, 557, 498, 531, 519, 530, 508, 525, 484, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[512, 525, 492, 540, 520, 524, 464, 478, 519, 508, 517, 506, 536, 549, 525, 548, 502, 512, 486, 501, 496, 516, 502, 500, 483, 539, 463, 510, 538, 502, 523, 520, 503, 495, 467, 507, 532, 529, 531, 539, 503, 536, 525, 514, 538, 528, 483, 544, 485, 513, 518, 532, 509, 450, 534, 457, 520, 497, 502, 533, 498, 519, 548, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[496, 512, 516, 531, 476, 486, 523, 483, 485, 503, 478, 538, 505, 515, 475, 524, 515, 514, 513, 529, 573, 484, 529, 520, 510, 513, 504, 540, 511, 500, 488, 496, 521, 500, 506, 524, 539, 525, 509, 502, 500, 464, 503, 521, 546, 510, 543, 532, 502, 526, 529, 513, 501, 520, 555, 495, 520, 488, 520, 531, 523, 482, 520, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[511, 516, 510, 536, 530, 535, 497, 534, 504, 492, 514, 481, 472, 496, 523, 509, 521, 504, 511, 511, 532, 488, 502, 520, 539, 531, 483, 492, 513, 493, 534, 500, 463, 486, 514, 503, 533, 527, 505, 485, 521, 472, 494, 496, 515, 544, 526, 498, 489, 517, 479, 516, 530, 536, 521, 536, 534, 560, 493, 529, 548, 499, 518, 547] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[511, 511, 500, 568, 464, 507, 530, 538, 0, 0, 0, 0, 0, 0, 0, 0, 517, 522, 505, 513, 474, 504, 499, 519, 0, 0, 0, 0, 0, 0, 0, 0, 485, 505, 545, 536, 502, 478, 494, 502, 14, 6, 4, 0, 0, 0, 0, 0, 502, 514, 478, 510, 490, 560, 525, 531, 0, 0, 0, 0, 0, 0, 0, 0, 524, 533, 500, 524, 534, 477, 523, 534, 0, 0, 0, 0, 0, 0, 0, 0, 535, 512, 527, 513, 487, 493, 504, 505, 5, 3, 0, 0, 0, 0, 0, 0, 485, 506, 516, 511, 517, 472, 520, 528, 27, 0, 0, 0, 0, 0, 0, 0, 509, 465, 515, 496, 520, 524, 528, 533, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4129, 4053, 4071, 4110, 4149, 4084, 4082, 4090] avg=4096.0000 max=4149 max_rank=4 min=4053 min_rank=1 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16405 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[514, 508, 529, 501, 494, 541, 488, 510, 0, 0, 0, 0, 0, 0, 0, 0, 534, 507, 494, 521, 501, 514, 493, 510, 0, 0, 0, 0, 0, 0, 0, 0, 483, 543, 530, 489, 517, 509, 529, 476, 14, 6, 4, 0, 0, 0, 0, 0, 517, 536, 504, 497, 517, 502, 560, 505, 0, 0, 0, 0, 0, 0, 0, 0, 538, 483, 488, 481, 475, 515, 549, 499, 0, 0, 0, 0, 0, 0, 0, 0, 522, 545, 528, 513, 516, 510, 508, 506, 5, 2, 0, 0, 0, 0, 0, 0, 449, 527, 503, 524, 513, 508, 507, 557, 27, 0, 0, 0, 0, 0, 0, 0, 498, 531, 519, 497, 508, 525, 484, 511, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4074, 4100, 4138, 4028, 4155, 4115, 4073] avg=4096.0000 max=4155 max_rank=5 min=4028 min_rank=4 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16371 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[509, 519, 488, 493, 528, 492, 510, 524, 0, 0, 0, 0, 0, 0, 0, 0, 544, 529, 515, 513, 501, 520, 487, 546, 0, 0, 0, 0, 0, 0, 0, 0, 547, 483, 537, 494, 502, 529, 510, 457, 14, 6, 4, 0, 0, 0, 0, 0, 520, 510, 467, 493, 537, 515, 489, 548, 0, 0, 0, 0, 0, 0, 0, 0, 497, 516, 513, 522, 539, 531, 505, 491, 0, 0, 0, 0, 0, 0, 0, 0, 516, 499, 528, 471, 515, 507, 488, 514, 6, 3, 0, 0, 0, 0, 0, 0, 503, 516, 548, 549, 516, 500, 497, 531, 26, 0, 0, 0, 0, 0, 0, 0, 544, 498, 519, 477, 490, 514, 488, 511, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4155, 4083, 4079, 4114, 4047, 4186, 4041] avg=4096.0000 max=4186 max_rank=6 min=4041 min_rank=7 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[481, 515, 546, 481, 521, 527, 526, 534, 0, 0, 0, 0, 0, 0, 0, 0, 533, 506, 494, 530, 548, 538, 525, 516, 0, 0, 0, 0, 0, 0, 0, 0, 493, 485, 501, 511, 471, 510, 511, 518, 14, 6, 4, 0, 0, 0, 0, 0, 487, 540, 513, 513, 509, 499, 490, 531, 0, 0, 0, 0, 0, 0, 0, 0, 523, 508, 517, 514, 513, 542, 497, 541, 0, 0, 0, 0, 0, 0, 0, 0, 529, 516, 469, 488, 531, 491, 500, 517, 5, 3, 0, 0, 0, 0, 0, 0, 539, 512, 474, 528, 481, 516, 484, 484, 25, 0, 0, 0, 0, 0, 0, 0, 520, 500, 546, 472, 551, 523, 496, 486, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4131, 4190, 4024, 4082, 4155, 4049, 4043, 4094] avg=4096.0000 max=4190 max_rank=1 min=4024 min_rank=2 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16341 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[526, 515, 511, 491, 520, 557, 503, 533, 0, 0, 0, 0, 0, 0, 0, 0, 516, 513, 529, 507, 518, 493, 514, 480, 0, 0, 0, 0, 0, 0, 0, 0, 496, 541, 529, 528, 475, 484, 516, 535, 14, 6, 4, 0, 0, 0, 0, 0, 533, 486, 505, 524, 528, 557, 529, 506, 0, 0, 0, 0, 0, 0, 0, 0, 517, 498, 527, 527, 512, 519, 501, 491, 0, 0, 0, 0, 0, 0, 0, 0, 490, 513, 548, 467, 515, 523, 460, 523, 5, 3, 0, 0, 0, 0, 0, 0, 508, 533, 514, 487, 474, 496, 455, 502, 27, 0, 0, 0, 0, 0, 0, 0, 524, 498, 503, 495, 509, 527, 510, 545, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4156, 4070, 4128, 4168, 4092, 4047, 3996, 4111] avg=4096.0000 max=4168 max_rank=3 min=3996 min_rank=6 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16246 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[512, 508, 492, 540, 520, 524, 464, 478, 0, 0, 0, 0, 0, 0, 0, 0, 519, 508, 517, 506, 536, 549, 525, 548, 4, 0, 0, 0, 0, 0, 0, 0, 502, 512, 486, 501, 496, 516, 502, 500, 17, 11, 7, 0, 0, 0, 0, 0, 483, 539, 463, 510, 538, 502, 523, 504, 0, 0, 0, 0, 0, 0, 0, 0, 503, 495, 467, 507, 523, 529, 531, 539, 0, 0, 0, 0, 0, 0, 0, 0, 503, 536, 525, 514, 538, 528, 483, 544, 9, 5, 0, 0, 0, 0, 0, 0, 485, 513, 518, 532, 509, 450, 534, 457, 29, 0, 0, 0, 0, 0, 0, 0, 520, 497, 502, 493, 498, 519, 548, 523, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4038, 4212, 4050, 4062, 4094, 4185, 4027, 4100] avg=4096.0000 max=4212 max_rank=1 min=4027 min_rank=6 imbalance_ratio=1.028320 max_load_reduction=28556 max_load_reduction_pct=0.871460 moved_to_echo=16406 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[511, 503, 510, 536, 530, 535, 497, 534, 0, 0, 0, 0, 0, 0, 0, 0, 504, 492, 514, 481, 472, 496, 523, 509, 0, 0, 0, 0, 0, 0, 0, 0, 521, 504, 511, 511, 532, 488, 502, 520, 13, 6, 4, 0, 0, 0, 0, 0, 539, 531, 483, 492, 513, 493, 534, 491, 0, 0, 0, 0, 0, 0, 0, 0, 463, 486, 514, 503, 530, 527, 505, 485, 0, 0, 0, 0, 0, 0, 0, 0, 521, 472, 494, 496, 515, 544, 526, 498, 5, 3, 0, 0, 0, 0, 0, 0, 489, 517, 479, 516, 530, 536, 521, 536, 27, 0, 0, 0, 0, 0, 0, 0, 534, 560, 493, 496, 548, 499, 518, 547, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4156, 3991, 4112, 4076, 4013, 4074, 4151, 4195] avg=4096.0000 max=4195 max_rank=7 min=3991 min_rank=1 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16433 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[496, 499, 516, 531, 476, 486, 523, 483, 0, 0, 0, 0, 0, 0, 0, 0, 485, 503, 478, 538, 505, 515, 475, 524, 0, 0, 0, 0, 0, 0, 0, 0, 515, 514, 513, 529, 573, 484, 529, 520, 13, 6, 4, 0, 0, 0, 0, 0, 510, 513, 504, 540, 511, 500, 488, 487, 0, 0, 0, 0, 0, 0, 0, 0, 521, 500, 506, 524, 536, 525, 509, 502, 0, 0, 0, 0, 0, 0, 0, 0, 500, 464, 503, 521, 546, 510, 543, 532, 5, 3, 0, 0, 0, 0, 0, 0, 502, 526, 529, 513, 501, 520, 555, 495, 27, 0, 0, 0, 0, 0, 0, 0, 520, 488, 520, 498, 523, 482, 520, 513, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4010, 4023, 4200, 4053, 4123, 4127, 4168, 4064] avg=4096.0000 max=4200 max_rank=2 min=4010 min_rank=0 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16482 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3968, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 3968, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 3968, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[483, 524, 518, 487, 508, 506, 495, 507, 509, 508, 525, 540, 482, 556, 527, 498, 498, 492, 492, 503, 516, 505, 503, 455, 527, 529, 503, 527, 510, 542, 526, 527, 498, 516, 540, 517, 521, 489, 517, 507, 509, 491, 529, 503, 503, 505, 498, 522, 522, 489, 518, 505, 496, 507, 529, 522, 482, 533, 524, 528, 516, 548, 537, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[493, 499, 504, 508, 526, 508, 515, 492, 516, 509, 512, 547, 517, 519, 505, 497, 520, 523, 512, 527, 527, 513, 524, 514, 573, 548, 532, 534, 529, 513, 497, 510, 496, 515, 496, 504, 539, 486, 486, 501, 481, 531, 543, 498, 519, 483, 468, 519, 507, 492, 515, 492, 516, 497, 518, 544, 511, 517, 467, 557, 538, 501, 478, 490] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[521, 527, 485, 539, 530, 511, 525, 513, 501, 535, 450, 545, 537, 503, 523, 511, 471, 526, 515, 488, 497, 510, 499, 507, 496, 520, 513, 507, 513, 506, 504, 525, 521, 537, 517, 505, 463, 523, 541, 522, 491, 530, 516, 539, 495, 524, 501, 544, 507, 474, 502, 583, 510, 514, 478, 481, 493, 500, 526, 492, 534, 542, 487, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[498, 542, 496, 519, 514, 509, 486, 499, 518, 484, 504, 510, 483, 534, 488, 494, 497, 532, 528, 543, 528, 529, 482, 471, 515, 517, 505, 477, 492, 511, 486, 546, 516, 527, 518, 511, 541, 519, 524, 530, 505, 493, 537, 519, 495, 502, 499, 532, 513, 539, 534, 514, 473, 499, 515, 524, 539, 476, 541, 507, 534, 549, 511, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[503, 505, 540, 475, 542, 531, 494, 498, 493, 509, 517, 518, 500, 502, 479, 514, 500, 548, 531, 500, 473, 470, 532, 534, 479, 515, 492, 521, 456, 494, 529, 496, 557, 513, 497, 562, 541, 481, 468, 537, 555, 533, 567, 552, 517, 504, 497, 522, 545, 501, 500, 500, 504, 500, 495, 547, 530, 515, 484, 512, 520, 500, 478, 544] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[515, 556, 526, 512, 528, 511, 477, 487, 544, 511, 529, 555, 527, 544, 491, 523, 486, 489, 550, 514, 510, 500, 491, 493, 503, 536, 509, 516, 481, 523, 495, 501, 502, 559, 509, 516, 502, 553, 488, 474, 515, 525, 500, 508, 531, 488, 518, 468, 496, 493, 504, 490, 533, 486, 525, 530, 513, 564, 520, 499, 524, 480, 510, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[473, 501, 529, 499, 492, 510, 500, 517, 519, 481, 502, 558, 547, 543, 510, 506, 511, 485, 513, 536, 507, 541, 514, 512, 503, 527, 506, 542, 489, 503, 523, 509, 516, 487, 471, 494, 523, 530, 532, 518, 515, 559, 510, 498, 494, 539, 493, 490, 531, 534, 503, 537, 514, 487, 532, 538, 494, 509, 507, 495, 485, 538, 509, 478] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[516, 538, 503, 560, 510, 532, 496, 536, 545, 505, 500, 523, 542, 519, 505, 486, 517, 495, 515, 483, 466, 510, 506, 512, 500, 488, 519, 536, 536, 540, 494, 484, 511, 490, 551, 523, 487, 519, 542, 482, 516, 502, 536, 519, 503, 534, 489, 562, 547, 474, 521, 489, 507, 498, 517, 499, 493, 480, 521, 485, 490, 516, 525, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[483, 524, 518, 487, 508, 506, 495, 507, 8, 0, 0, 0, 0, 0, 0, 0, 509, 508, 525, 520, 482, 556, 527, 498, 0, 0, 0, 0, 0, 0, 0, 0, 498, 492, 492, 503, 516, 505, 503, 455, 20, 13, 0, 0, 0, 0, 0, 0, 527, 525, 503, 527, 510, 542, 526, 527, 0, 0, 0, 0, 0, 0, 0, 0, 498, 503, 540, 517, 521, 489, 517, 507, 0, 0, 0, 0, 0, 0, 0, 0, 509, 491, 516, 503, 503, 505, 498, 522, 0, 0, 0, 0, 0, 0, 0, 0, 522, 489, 518, 505, 496, 507, 529, 522, 5, 2, 0, 0, 0, 0, 0, 0, 482, 533, 524, 528, 516, 548, 537, 519, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4036, 4125, 3997, 4187, 4092, 4047, 4095, 4189] avg=4096.0000 max=4189 max_rank=7 min=3997 min_rank=2 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16423 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[493, 499, 504, 508, 526, 508, 515, 492, 8, 0, 0, 0, 0, 0, 0, 0, 516, 509, 512, 526, 517, 519, 505, 497, 0, 0, 0, 0, 0, 0, 0, 0, 520, 523, 512, 527, 527, 513, 524, 514, 21, 14, 0, 0, 0, 0, 0, 0, 573, 544, 532, 534, 529, 513, 497, 510, 0, 0, 0, 0, 0, 0, 0, 0, 496, 502, 496, 504, 539, 486, 486, 501, 0, 0, 0, 0, 0, 0, 0, 0, 481, 531, 529, 498, 519, 483, 468, 519, 0, 0, 0, 0, 0, 0, 0, 0, 507, 492, 515, 492, 516, 497, 518, 544, 5, 2, 0, 0, 0, 0, 0, 0, 511, 517, 467, 557, 538, 501, 478, 490, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4053, 4101, 4195, 4232, 4010, 4028, 4088, 4061] avg=4096.0000 max=4232 max_rank=3 min=4010 min_rank=4 imbalance_ratio=1.033203 max_load_reduction=28536 max_load_reduction_pct=0.870850 moved_to_echo=16187 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[515, 556, 526, 512, 528, 511, 477, 487, 8, 0, 0, 0, 0, 0, 0, 0, 544, 511, 529, 534, 527, 544, 491, 523, 0, 0, 0, 0, 0, 0, 0, 0, 486, 489, 550, 514, 510, 500, 491, 493, 21, 13, 0, 0, 0, 0, 0, 0, 503, 532, 509, 516, 481, 523, 495, 501, 0, 0, 0, 0, 0, 0, 0, 0, 502, 545, 509, 516, 502, 553, 488, 474, 0, 0, 0, 0, 0, 0, 0, 0, 515, 525, 487, 508, 531, 488, 518, 468, 0, 0, 0, 0, 0, 0, 0, 0, 496, 493, 504, 490, 533, 486, 525, 530, 6, 2, 0, 0, 0, 0, 0, 0, 513, 564, 520, 499, 524, 480, 510, 512, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4203, 4067, 4060, 4089, 4040, 4065, 4124] avg=4096.0000 max=4203 max_rank=1 min=4040 min_rank=5 imbalance_ratio=1.026123 max_load_reduction=28565 max_load_reduction_pct=0.871735 moved_to_echo=16318 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[473, 501, 529, 499, 492, 510, 500, 517, 7, 0, 0, 0, 0, 0, 0, 0, 519, 481, 502, 537, 547, 543, 510, 506, 0, 0, 0, 0, 0, 0, 0, 0, 511, 485, 513, 536, 507, 541, 514, 512, 21, 13, 0, 0, 0, 0, 0, 0, 503, 523, 506, 542, 489, 503, 523, 509, 0, 0, 0, 0, 0, 0, 0, 0, 516, 475, 471, 494, 523, 530, 532, 518, 0, 0, 0, 0, 0, 0, 0, 0, 515, 559, 497, 498, 494, 539, 493, 490, 0, 0, 0, 0, 0, 0, 0, 0, 531, 534, 503, 537, 514, 487, 532, 538, 5, 2, 0, 0, 0, 0, 0, 0, 494, 509, 507, 495, 485, 538, 509, 478, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4028, 4145, 4153, 4098, 4059, 4085, 4183, 4017] avg=4096.0000 max=4183 max_rank=6 min=4017 min_rank=7 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16344 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[498, 542, 496, 519, 514, 509, 486, 499, 8, 0, 0, 0, 0, 0, 0, 0, 518, 484, 504, 491, 483, 534, 488, 494, 0, 0, 0, 0, 0, 0, 0, 0, 497, 532, 528, 543, 528, 529, 482, 471, 19, 14, 0, 0, 0, 0, 0, 0, 515, 513, 505, 477, 492, 511, 486, 546, 0, 0, 0, 0, 0, 0, 0, 0, 516, 514, 518, 511, 541, 519, 524, 530, 0, 0, 0, 0, 0, 0, 0, 0, 505, 493, 523, 519, 495, 502, 499, 532, 0, 0, 0, 0, 0, 0, 0, 0, 513, 539, 534, 514, 473, 499, 515, 524, 5, 2, 0, 0, 0, 0, 0, 0, 539, 476, 541, 507, 534, 549, 511, 495, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4071, 3996, 4143, 4045, 4173, 4068, 4118, 4154] avg=4096.0000 max=4173 max_rank=4 min=3996 min_rank=1 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16513 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[521, 527, 485, 539, 530, 511, 525, 513, 11, 4, 0, 0, 0, 0, 0, 0, 501, 535, 450, 521, 537, 503, 523, 511, 0, 0, 0, 0, 0, 0, 0, 0, 471, 526, 515, 488, 497, 510, 499, 507, 24, 16, 0, 0, 0, 0, 0, 0, 496, 510, 513, 507, 513, 506, 504, 525, 0, 0, 0, 0, 0, 0, 0, 0, 521, 517, 517, 505, 463, 523, 541, 522, 0, 0, 0, 0, 0, 0, 0, 0, 491, 530, 496, 539, 495, 524, 501, 544, 0, 0, 0, 0, 0, 0, 0, 0, 507, 474, 502, 583, 510, 514, 478, 481, 9, 4, 0, 0, 0, 0, 0, 0, 493, 500, 526, 492, 534, 542, 487, 523, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4166, 4081, 4053, 4074, 4109, 4120, 4062, 4103] avg=4096.0000 max=4166 max_rank=0 min=4053 min_rank=2 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16394 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[503, 505, 540, 475, 542, 531, 494, 498, 8, 0, 0, 0, 0, 0, 0, 0, 493, 509, 517, 498, 500, 502, 479, 514, 0, 0, 0, 0, 0, 0, 0, 0, 500, 548, 531, 500, 473, 470, 532, 534, 20, 14, 0, 0, 0, 0, 0, 0, 479, 511, 492, 521, 456, 494, 529, 496, 0, 0, 0, 0, 0, 0, 0, 0, 557, 500, 497, 562, 541, 481, 468, 537, 0, 0, 0, 0, 0, 0, 0, 0, 555, 533, 553, 552, 517, 504, 497, 522, 0, 0, 0, 0, 0, 0, 0, 0, 545, 501, 500, 500, 504, 500, 495, 547, 5, 2, 0, 0, 0, 0, 0, 0, 530, 515, 484, 512, 520, 500, 478, 544, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4012, 4122, 3978, 4143, 4233, 4099, 4085] avg=4096.0000 max=4233 max_rank=5 min=3978 min_rank=3 imbalance_ratio=1.033447 max_load_reduction=28535 max_load_reduction_pct=0.870819 moved_to_echo=16560 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[516, 538, 503, 560, 510, 532, 496, 536, 7, 0, 0, 0, 0, 0, 0, 0, 545, 505, 500, 503, 542, 519, 505, 486, 0, 0, 0, 0, 0, 0, 0, 0, 517, 495, 515, 483, 466, 510, 506, 512, 20, 14, 0, 0, 0, 0, 0, 0, 500, 485, 519, 536, 536, 540, 494, 484, 0, 0, 0, 0, 0, 0, 0, 0, 511, 478, 551, 523, 487, 519, 542, 482, 0, 0, 0, 0, 0, 0, 0, 0, 516, 502, 522, 519, 503, 534, 489, 562, 0, 0, 0, 0, 0, 0, 0, 0, 547, 474, 521, 489, 507, 498, 517, 499, 5, 1, 0, 0, 0, 0, 0, 0, 493, 480, 521, 485, 490, 516, 525, 523, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4198, 4105, 4038, 4094, 4093, 4147, 4058, 4035] avg=4096.0000 max=4198 max_rank=0 min=4035 min_rank=7 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16333 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[520, 529, 543, 492, 520, 484, 511, 505, 485, 544, 517, 507, 512, 512, 517, 518, 498, 497, 492, 479, 542, 502, 530, 518, 562, 530, 554, 457, 518, 544, 519, 538, 477, 535, 497, 537, 525, 517, 511, 474, 516, 495, 487, 500, 502, 510, 488, 528, 512, 544, 494, 475, 511, 535, 473, 499, 488, 520, 513, 509, 553, 510, 536, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[536, 492, 522, 522, 489, 523, 509, 438, 526, 481, 506, 531, 492, 536, 533, 543, 495, 502, 479, 521, 529, 501, 498, 474, 522, 528, 439, 524, 478, 489, 549, 489, 520, 517, 522, 499, 499, 502, 531, 489, 531, 545, 525, 540, 546, 538, 462, 497, 536, 527, 522, 517, 507, 503, 494, 564, 498, 527, 539, 506, 530, 488, 526, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[532, 503, 540, 532, 444, 520, 510, 487, 513, 494, 499, 495, 491, 480, 513, 515, 521, 525, 448, 533, 499, 515, 519, 479, 531, 549, 509, 489, 520, 526, 497, 532, 527, 562, 497, 532, 500, 511, 546, 511, 509, 502, 528, 541, 485, 501, 499, 502, 534, 508, 508, 520, 543, 439, 530, 498, 535, 536, 520, 540, 501, 487, 529, 527] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[566, 488, 504, 490, 537, 502, 499, 511, 515, 483, 535, 524, 498, 561, 467, 486, 500, 481, 542, 529, 493, 462, 515, 492, 506, 517, 544, 532, 498, 482, 493, 501, 558, 566, 528, 517, 505, 514, 523, 515, 515, 531, 490, 554, 533, 502, 506, 511, 551, 497, 504, 505, 475, 540, 504, 514, 524, 519, 539, 522, 507, 483, 486, 477] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[477, 523, 487, 482, 549, 513, 508, 509, 506, 489, 513, 518, 475, 521, 552, 510, 493, 548, 520, 479, 568, 520, 505, 517, 543, 518, 479, 487, 480, 494, 502, 536, 496, 499, 496, 513, 556, 515, 553, 524, 481, 504, 520, 547, 516, 519, 502, 519, 543, 553, 504, 504, 517, 485, 534, 486, 495, 548, 530, 512, 474, 500, 502, 500] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[523, 500, 505, 498, 499, 515, 518, 510, 502, 480, 521, 483, 511, 467, 528, 500, 492, 523, 513, 504, 526, 506, 530, 491, 518, 509, 540, 504, 534, 517, 511, 523, 528, 502, 524, 519, 522, 505, 502, 498, 566, 530, 510, 512, 491, 509, 487, 530, 561, 518, 484, 509, 506, 531, 529, 547, 495, 508, 522, 496, 522, 494, 512, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[486, 514, 460, 518, 488, 532, 527, 510, 509, 488, 538, 501, 463, 486, 544, 504, 520, 492, 561, 528, 510, 531, 519, 500, 510, 529, 535, 508, 535, 511, 523, 486, 546, 484, 529, 530, 495, 504, 541, 523, 491, 501, 507, 498, 517, 489, 482, 518, 533, 504, 502, 512, 499, 522, 484, 530, 535, 531, 492, 531, 521, 522, 525, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[514, 509, 481, 502, 485, 486, 501, 528, 492, 525, 490, 516, 500, 515, 543, 500, 505, 484, 468, 516, 481, 519, 504, 519, 550, 559, 505, 472, 520, 504, 498, 502, 529, 531, 529, 540, 519, 488, 500, 566, 517, 500, 515, 534, 526, 541, 501, 513, 515, 522, 539, 521, 468, 467, 464, 521, 527, 503, 528, 553, 525, 535, 504, 534] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[536, 492, 522, 522, 489, 523, 509, 438, 39, 0, 0, 0, 0, 0, 0, 0, 526, 481, 506, 531, 492, 536, 533, 543, 17, 5, 4, 3, 0, 0, 0, 0, 495, 502, 479, 521, 529, 501, 498, 474, 22, 11, 0, 0, 0, 0, 0, 0, 505, 528, 439, 524, 478, 489, 549, 489, 0, 0, 0, 0, 0, 0, 0, 0, 520, 517, 522, 499, 499, 502, 481, 489, 0, 0, 0, 0, 0, 0, 0, 0, 531, 545, 525, 535, 546, 538, 462, 497, 0, 0, 0, 0, 0, 0, 0, 0, 533, 527, 522, 517, 507, 503, 494, 564, 0, 0, 0, 0, 0, 0, 0, 0, 498, 501, 539, 506, 530, 488, 526, 525, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4070, 4177, 4032, 4001, 4029, 4179, 4167, 4113] avg=4096.0000 max=4179 max_rank=5 min=4001 min_rank=3 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16488 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[566, 488, 504, 490, 537, 502, 499, 511, 38, 0, 0, 0, 0, 0, 0, 0, 515, 483, 535, 524, 498, 561, 467, 486, 16, 5, 4, 3, 0, 0, 0, 0, 500, 481, 542, 529, 493, 462, 515, 492, 21, 11, 0, 0, 0, 0, 0, 0, 490, 517, 544, 532, 498, 482, 493, 501, 0, 0, 0, 0, 0, 0, 0, 0, 558, 566, 528, 517, 505, 514, 474, 515, 0, 0, 0, 0, 0, 0, 0, 0, 515, 531, 490, 549, 533, 502, 506, 511, 0, 0, 0, 0, 0, 0, 0, 0, 548, 497, 504, 505, 475, 540, 504, 514, 0, 0, 0, 0, 0, 0, 0, 0, 524, 494, 539, 522, 507, 483, 486, 477, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4135, 4097, 4046, 4057, 4177, 4137, 4087, 4032] avg=4096.0000 max=4177 max_rank=4 min=4032 min_rank=7 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16433 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[477, 523, 487, 482, 549, 513, 508, 509, 40, 0, 0, 0, 0, 0, 0, 0, 506, 489, 513, 518, 475, 521, 552, 510, 17, 5, 4, 3, 0, 0, 0, 0, 493, 548, 520, 479, 568, 520, 505, 517, 22, 11, 0, 0, 0, 0, 0, 0, 526, 518, 479, 487, 480, 494, 502, 536, 0, 0, 0, 0, 0, 0, 0, 0, 496, 499, 496, 513, 556, 515, 502, 524, 0, 0, 0, 0, 0, 0, 0, 0, 481, 504, 520, 542, 516, 519, 502, 519, 0, 0, 0, 0, 0, 0, 0, 0, 540, 553, 504, 504, 517, 485, 534, 486, 0, 0, 0, 0, 0, 0, 0, 0, 495, 522, 530, 512, 474, 500, 502, 500, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4113, 4183, 4022, 4101, 4103, 4123, 4035] avg=4096.0000 max=4183 max_rank=2 min=4022 min_rank=3 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16362 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[520, 529, 543, 492, 520, 484, 511, 505, 37, 0, 0, 0, 0, 0, 0, 0, 485, 544, 517, 507, 512, 512, 517, 518, 18, 5, 4, 3, 0, 0, 0, 0, 498, 497, 492, 479, 542, 502, 530, 518, 21, 11, 0, 0, 0, 0, 0, 0, 544, 530, 554, 457, 518, 544, 519, 538, 0, 0, 0, 0, 0, 0, 0, 0, 477, 535, 497, 537, 525, 517, 463, 474, 0, 0, 0, 0, 0, 0, 0, 0, 516, 495, 487, 495, 502, 510, 488, 528, 0, 0, 0, 0, 0, 0, 0, 0, 509, 544, 494, 475, 511, 535, 473, 499, 0, 0, 0, 0, 0, 0, 0, 0, 488, 495, 513, 509, 553, 510, 536, 501, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4141, 4142, 4090, 4204, 4025, 4021, 4040, 4105] avg=4096.0000 max=4204 max_rank=3 min=4021 min_rank=5 imbalance_ratio=1.026367 max_load_reduction=28564 max_load_reduction_pct=0.871704 moved_to_echo=16191 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[514, 509, 481, 502, 485, 486, 501, 528, 36, 0, 0, 0, 0, 0, 0, 0, 492, 525, 490, 516, 500, 515, 543, 500, 18, 5, 4, 3, 0, 0, 0, 0, 505, 484, 468, 516, 481, 519, 504, 519, 20, 10, 0, 0, 0, 0, 0, 0, 532, 559, 505, 472, 520, 504, 498, 502, 0, 0, 0, 0, 0, 0, 0, 0, 529, 531, 529, 540, 519, 488, 454, 566, 0, 0, 0, 0, 0, 0, 0, 0, 517, 500, 515, 529, 526, 541, 501, 513, 0, 0, 0, 0, 0, 0, 0, 0, 512, 522, 539, 521, 468, 467, 464, 521, 0, 0, 0, 0, 0, 0, 0, 0, 527, 479, 528, 553, 525, 535, 504, 534, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4042, 4111, 4026, 4092, 4156, 4142, 4014, 4185] avg=4096.0000 max=4185 max_rank=7 min=4014 min_rank=6 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16497 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[486, 514, 460, 518, 488, 532, 527, 510, 39, 0, 0, 0, 0, 0, 0, 0, 509, 488, 538, 501, 463, 486, 544, 504, 16, 5, 4, 3, 0, 0, 0, 0, 520, 492, 561, 528, 510, 531, 519, 500, 22, 11, 0, 0, 0, 0, 0, 0, 494, 529, 535, 508, 535, 511, 523, 486, 0, 0, 0, 0, 0, 0, 0, 0, 546, 484, 529, 530, 495, 504, 491, 523, 0, 0, 0, 0, 0, 0, 0, 0, 491, 501, 507, 493, 517, 489, 482, 518, 0, 0, 0, 0, 0, 0, 0, 0, 530, 504, 502, 512, 499, 522, 484, 530, 0, 0, 0, 0, 0, 0, 0, 0, 535, 505, 492, 531, 521, 522, 525, 504, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4061, 4194, 4121, 4102, 3998, 4083, 4135] avg=4096.0000 max=4194 max_rank=2 min=3998 min_rank=5 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16318 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[523, 500, 505, 498, 499, 515, 518, 510, 42, 0, 0, 0, 0, 0, 0, 0, 502, 480, 521, 483, 511, 467, 528, 500, 21, 9, 8, 8, 0, 0, 0, 0, 492, 523, 513, 504, 526, 506, 530, 491, 25, 15, 0, 0, 0, 0, 0, 0, 497, 509, 540, 504, 534, 517, 511, 523, 0, 0, 0, 0, 0, 0, 0, 0, 528, 502, 524, 519, 522, 505, 445, 498, 0, 0, 0, 0, 0, 0, 0, 0, 566, 530, 510, 503, 491, 509, 487, 530, 0, 0, 0, 0, 0, 0, 0, 0, 553, 518, 484, 509, 506, 531, 529, 547, 0, 0, 0, 0, 0, 0, 0, 0, 495, 475, 522, 496, 522, 494, 512, 498, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4038, 4125, 4135, 4043, 4126, 4177, 4014] avg=4096.0000 max=4177 max_rank=6 min=4014 min_rank=7 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16360 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[532, 503, 540, 532, 444, 520, 510, 487, 40, 0, 0, 0, 0, 0, 0, 0, 513, 494, 499, 495, 491, 480, 513, 515, 17, 5, 4, 3, 0, 0, 0, 0, 521, 525, 448, 533, 499, 515, 519, 479, 22, 11, 0, 0, 0, 0, 0, 0, 514, 549, 509, 489, 520, 526, 497, 532, 0, 0, 0, 0, 0, 0, 0, 0, 527, 562, 497, 532, 500, 511, 495, 511, 0, 0, 0, 0, 0, 0, 0, 0, 509, 502, 528, 536, 485, 501, 499, 502, 0, 0, 0, 0, 0, 0, 0, 0, 531, 508, 508, 520, 543, 439, 530, 498, 0, 0, 0, 0, 0, 0, 0, 0, 535, 510, 520, 540, 501, 487, 529, 527, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4029, 4072, 4136, 4135, 4062, 4077, 4149] avg=4096.0000 max=4149 max_rank=7 min=4029 min_rank=1 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16423 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 3840, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4352, 4224, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4096, 3968, 4096, 4224, 4096, 256, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4352, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4224, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[536, 540, 469, 476, 509, 556, 540, 464, 525, 481, 531, 517, 489, 522, 509, 534, 511, 478, 484, 500, 481, 533, 498, 502, 494, 512, 518, 499, 469, 525, 576, 536, 511, 490, 479, 502, 514, 519, 502, 482, 518, 497, 503, 514, 528, 552, 516, 524, 536, 509, 527, 515, 477, 530, 545, 496, 508, 555, 520, 532, 505, 497, 532, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[550, 519, 515, 536, 530, 520, 513, 528, 495, 524, 520, 494, 515, 510, 474, 492, 503, 493, 535, 526, 533, 510, 536, 489, 495, 531, 546, 531, 494, 519, 461, 504, 536, 503, 514, 497, 483, 484, 520, 463, 487, 515, 521, 505, 532, 510, 488, 506, 515, 491, 520, 515, 536, 492, 500, 503, 560, 511, 537, 524, 511, 479, 547, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[518, 479, 507, 474, 545, 537, 483, 518, 515, 528, 505, 507, 494, 498, 522, 496, 487, 538, 523, 498, 506, 502, 495, 530, 519, 530, 514, 514, 488, 472, 508, 532, 539, 506, 499, 515, 499, 536, 526, 552, 530, 511, 547, 506, 515, 482, 543, 526, 499, 531, 472, 501, 484, 527, 508, 532, 518, 489, 495, 481, 532, 522, 525, 538] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[503, 494, 515, 514, 506, 509, 499, 499, 480, 504, 492, 522, 504, 528, 527, 538, 522, 499, 528, 501, 524, 491, 511, 505, 533, 527, 497, 558, 484, 516, 467, 491, 495, 549, 516, 526, 487, 528, 513, 502, 470, 505, 533, 501, 499, 487, 498, 512, 543, 528, 517, 548, 509, 504, 468, 544, 523, 506, 549, 531, 502, 532, 520, 535] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[518, 505, 551, 491, 555, 549, 460, 500, 546, 490, 503, 548, 482, 503, 482, 531, 518, 493, 512, 510, 540, 508, 511, 530, 530, 491, 519, 501, 509, 482, 530, 528, 494, 518, 468, 568, 484, 537, 527, 499, 500, 519, 544, 521, 496, 502, 504, 536, 517, 510, 513, 498, 482, 527, 488, 526, 521, 496, 485, 500, 545, 503, 491, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[472, 505, 498, 490, 508, 506, 553, 507, 465, 488, 527, 508, 518, 488, 499, 508, 494, 508, 524, 524, 518, 544, 510, 476, 533, 488, 491, 512, 530, 519, 516, 505, 535, 512, 517, 539, 530, 514, 471, 522, 500, 481, 505, 519, 517, 513, 545, 548, 525, 513, 544, 539, 492, 493, 537, 498, 522, 530, 510, 551, 520, 513, 485, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[485, 517, 482, 501, 511, 493, 542, 510, 509, 522, 496, 503, 556, 512, 513, 527, 525, 492, 549, 520, 487, 463, 495, 507, 496, 485, 524, 525, 520, 511, 542, 499, 546, 519, 519, 513, 507, 506, 492, 519, 469, 504, 549, 506, 517, 526, 513, 482, 523, 526, 525, 525, 500, 496, 536, 530, 490, 522, 513, 490, 531, 512, 515, 528] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[532, 503, 471, 515, 519, 490, 476, 499, 531, 495, 492, 526, 506, 515, 503, 524, 517, 525, 485, 540, 520, 507, 480, 523, 509, 513, 522, 534, 477, 542, 476, 538, 520, 503, 481, 513, 534, 522, 526, 575, 517, 513, 534, 515, 541, 513, 501, 511, 526, 490, 500, 526, 498, 519, 519, 522, 508, 507, 489, 480, 511, 526, 509, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[536, 540, 469, 476, 509, 556, 540, 464, 11, 8, 0, 0, 0, 0, 0, 0, 525, 481, 531, 517, 489, 522, 509, 534, 19, 0, 0, 0, 0, 0, 0, 0, 511, 478, 484, 500, 481, 533, 498, 502, 8, 4, 3, 0, 0, 0, 0, 0, 494, 512, 518, 499, 469, 525, 576, 536, 1, 0, 0, 0, 0, 0, 0, 0, 506, 490, 479, 502, 514, 519, 502, 482, 0, 0, 0, 0, 0, 0, 0, 0, 518, 497, 495, 514, 528, 552, 516, 524, 0, 0, 0, 0, 0, 0, 0, 0, 522, 509, 527, 515, 477, 530, 545, 496, 0, 0, 0, 0, 0, 0, 0, 0, 508, 555, 520, 532, 505, 497, 532, 492, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4127, 4002, 4130, 3994, 4144, 4121, 4141] avg=4096.0000 max=4144 max_rank=5 min=3994 min_rank=4 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16400 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[550, 519, 515, 536, 530, 520, 513, 528, 10, 8, 0, 0, 0, 0, 0, 0, 495, 524, 520, 494, 515, 510, 474, 492, 20, 0, 0, 0, 0, 0, 0, 0, 503, 493, 535, 526, 533, 510, 536, 489, 9, 4, 3, 0, 0, 0, 0, 0, 495, 531, 546, 531, 494, 519, 461, 504, 1, 0, 0, 0, 0, 0, 0, 0, 531, 503, 514, 497, 483, 484, 520, 463, 0, 0, 0, 0, 0, 0, 0, 0, 487, 515, 512, 505, 532, 510, 488, 506, 0, 0, 0, 0, 0, 0, 0, 0, 502, 491, 520, 515, 536, 492, 500, 503, 0, 0, 0, 0, 0, 0, 0, 0, 560, 511, 537, 524, 511, 479, 547, 494, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4229, 4044, 4141, 4082, 3995, 4055, 4059, 4163] avg=4096.0000 max=4229 max_rank=0 min=3995 min_rank=4 imbalance_ratio=1.032471 max_load_reduction=28539 max_load_reduction_pct=0.870941 moved_to_echo=16272 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[503, 494, 515, 514, 506, 509, 499, 499, 11, 8, 0, 0, 0, 0, 0, 0, 480, 504, 492, 522, 504, 528, 527, 538, 20, 0, 0, 0, 0, 0, 0, 0, 522, 499, 528, 501, 524, 491, 511, 505, 9, 4, 3, 0, 0, 0, 0, 0, 533, 527, 497, 558, 484, 516, 467, 491, 1, 0, 0, 0, 0, 0, 0, 0, 490, 549, 516, 526, 487, 528, 513, 502, 0, 0, 0, 0, 0, 0, 0, 0, 470, 505, 524, 501, 499, 487, 498, 512, 0, 0, 0, 0, 0, 0, 0, 0, 529, 528, 517, 548, 509, 504, 468, 544, 0, 0, 0, 0, 0, 0, 0, 0, 523, 506, 549, 531, 502, 532, 520, 507, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4058, 4115, 4097, 4074, 4111, 3996, 4147, 4170] avg=4096.0000 max=4170 max_rank=7 min=3996 min_rank=5 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16424 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[518, 479, 507, 474, 545, 537, 483, 518, 10, 8, 0, 0, 0, 0, 0, 0, 515, 528, 505, 507, 494, 498, 522, 496, 20, 0, 0, 0, 0, 0, 0, 0, 487, 538, 523, 498, 506, 502, 495, 530, 9, 4, 3, 0, 0, 0, 0, 0, 519, 530, 514, 514, 488, 472, 508, 532, 1, 0, 0, 0, 0, 0, 0, 0, 534, 506, 499, 515, 499, 536, 526, 552, 0, 0, 0, 0, 0, 0, 0, 0, 530, 511, 538, 506, 515, 482, 543, 526, 0, 0, 0, 0, 0, 0, 0, 0, 486, 531, 472, 501, 484, 527, 508, 532, 0, 0, 0, 0, 0, 0, 0, 0, 518, 489, 495, 481, 532, 522, 525, 510, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4085, 4095, 4078, 4167, 4151, 4041, 4072] avg=4096.0000 max=4167 max_rank=4 min=4041 min_rank=6 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[472, 505, 498, 490, 508, 506, 553, 507, 14, 9, 0, 0, 0, 0, 0, 0, 465, 488, 527, 508, 518, 488, 499, 508, 22, 0, 0, 0, 0, 0, 0, 0, 494, 508, 524, 524, 518, 544, 510, 476, 12, 10, 8, 0, 0, 0, 0, 0, 533, 488, 491, 512, 530, 519, 516, 505, 4, 0, 0, 0, 0, 0, 0, 0, 521, 512, 517, 539, 530, 514, 471, 522, 0, 0, 0, 0, 0, 0, 0, 0, 500, 481, 493, 519, 517, 513, 545, 548, 0, 0, 0, 0, 0, 0, 0, 0, 503, 513, 544, 539, 492, 493, 537, 498, 0, 0, 0, 0, 0, 0, 0, 0, 522, 530, 510, 551, 520, 513, 485, 465, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4062, 4023, 4128, 4098, 4126, 4116, 4119, 4096] avg=4096.0000 max=4128 max_rank=2 min=4023 min_rank=1 imbalance_ratio=1.007812 max_load_reduction=28640 max_load_reduction_pct=0.874023 moved_to_echo=16457 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[518, 505, 551, 491, 555, 549, 460, 500, 10, 8, 0, 0, 0, 0, 0, 0, 546, 490, 503, 548, 482, 503, 482, 531, 20, 0, 0, 0, 0, 0, 0, 0, 518, 493, 512, 510, 540, 508, 511, 530, 9, 4, 3, 0, 0, 0, 0, 0, 530, 491, 519, 501, 509, 482, 530, 528, 1, 0, 0, 0, 0, 0, 0, 0, 489, 518, 468, 568, 484, 537, 527, 499, 0, 0, 0, 0, 0, 0, 0, 0, 500, 519, 535, 521, 496, 502, 504, 536, 0, 0, 0, 0, 0, 0, 0, 0, 504, 510, 513, 498, 482, 527, 488, 526, 0, 0, 0, 0, 0, 0, 0, 0, 521, 496, 485, 500, 545, 503, 491, 495, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4147, 4105, 4138, 4091, 4090, 4113, 4048, 4036] avg=4096.0000 max=4147 max_rank=0 min=4036 min_rank=7 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16287 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[532, 503, 471, 515, 519, 490, 476, 499, 11, 8, 0, 0, 0, 0, 0, 0, 531, 495, 492, 526, 506, 515, 503, 524, 19, 0, 0, 0, 0, 0, 0, 0, 517, 525, 485, 540, 520, 507, 480, 523, 9, 4, 3, 0, 0, 0, 0, 0, 509, 513, 522, 534, 477, 542, 476, 538, 1, 0, 0, 0, 0, 0, 0, 0, 515, 503, 481, 513, 534, 522, 526, 575, 0, 0, 0, 0, 0, 0, 0, 0, 517, 513, 525, 515, 541, 513, 501, 511, 0, 0, 0, 0, 0, 0, 0, 0, 512, 490, 500, 526, 498, 519, 519, 522, 0, 0, 0, 0, 0, 0, 0, 0, 508, 507, 489, 480, 511, 526, 509, 487, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4024, 4111, 4113, 4112, 4169, 4136, 4086, 4017] avg=4096.0000 max=4169 max_rank=4 min=4017 min_rank=7 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16408 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[485, 517, 482, 501, 511, 493, 542, 510, 11, 8, 0, 0, 0, 0, 0, 0, 509, 522, 496, 503, 556, 512, 513, 527, 20, 0, 0, 0, 0, 0, 0, 0, 525, 492, 549, 520, 487, 463, 495, 507, 9, 4, 3, 0, 0, 0, 0, 0, 496, 485, 524, 525, 520, 511, 542, 499, 1, 0, 0, 0, 0, 0, 0, 0, 541, 519, 519, 513, 507, 506, 492, 519, 0, 0, 0, 0, 0, 0, 0, 0, 469, 504, 540, 506, 517, 526, 513, 482, 0, 0, 0, 0, 0, 0, 0, 0, 509, 526, 525, 525, 500, 496, 536, 530, 0, 0, 0, 0, 0, 0, 0, 0, 490, 522, 513, 490, 531, 512, 515, 500, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4158, 4054, 4103, 4116, 4057, 4147, 4073] avg=4096.0000 max=4158 max_rank=1 min=4054 min_rank=2 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4224, 3968, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[573, 511, 503, 499, 508, 515, 513, 508, 524, 496, 497, 531, 453, 485, 505, 535, 494, 522, 513, 539, 496, 505, 512, 558, 509, 489, 471, 495, 479, 500, 558, 534, 521, 501, 504, 502, 536, 529, 512, 517, 503, 507, 533, 514, 512, 509, 525, 500, 530, 552, 491, 525, 565, 480, 490, 522, 522, 489, 502, 511, 475, 535, 529, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[526, 522, 534, 495, 499, 513, 517, 517, 505, 530, 516, 506, 549, 509, 522, 522, 483, 483, 544, 521, 535, 514, 485, 452, 551, 540, 511, 539, 509, 488, 493, 516, 515, 499, 531, 474, 515, 530, 515, 492, 504, 510, 507, 539, 479, 497, 525, 489, 522, 523, 525, 503, 498, 495, 520, 486, 502, 527, 514, 493, 518, 520, 544, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[475, 541, 493, 504, 534, 458, 518, 490, 489, 497, 529, 483, 545, 512, 529, 527, 495, 486, 503, 506, 526, 515, 524, 481, 525, 537, 520, 542, 527, 510, 518, 526, 531, 545, 484, 497, 511, 493, 551, 516, 531, 540, 500, 453, 564, 530, 539, 525, 494, 518, 515, 492, 502, 529, 544, 464, 477, 516, 524, 516, 496, 520, 510, 476] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[509, 486, 497, 485, 489, 515, 474, 531, 526, 514, 540, 538, 523, 484, 565, 508, 499, 545, 510, 495, 508, 464, 519, 536, 528, 531, 526, 536, 507, 497, 490, 505, 494, 514, 502, 518, 519, 494, 508, 550, 503, 559, 509, 493, 503, 466, 518, 537, 484, 480, 507, 558, 513, 541, 520, 558, 491, 511, 504, 501, 466, 493, 529, 545] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[496, 543, 543, 485, 516, 506, 528, 521, 477, 463, 486, 511, 506, 497, 531, 511, 561, 528, 533, 512, 509, 532, 486, 477, 505, 494, 445, 525, 541, 545, 540, 526, 512, 520, 518, 502, 542, 554, 565, 514, 485, 486, 488, 491, 543, 523, 486, 545, 511, 522, 519, 491, 502, 518, 513, 513, 506, 449, 517, 488, 496, 518, 514, 538] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[494, 534, 523, 505, 494, 491, 487, 501, 506, 522, 483, 466, 511, 504, 514, 511, 521, 504, 499, 502, 506, 519, 494, 524, 512, 516, 509, 503, 507, 556, 516, 554, 519, 493, 531, 540, 484, 518, 489, 532, 509, 551, 491, 542, 548, 486, 504, 550, 527, 536, 509, 485, 506, 504, 549, 514, 482, 488, 493, 507, 539, 514, 509, 531] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[521, 510, 480, 520, 528, 513, 510, 506, 481, 493, 540, 556, 504, 516, 512, 510, 516, 500, 551, 566, 502, 522, 495, 480, 547, 507, 523, 496, 530, 481, 538, 490, 511, 508, 498, 506, 528, 518, 514, 551, 517, 546, 489, 548, 529, 550, 505, 494, 467, 484, 500, 530, 490, 495, 485, 497, 506, 504, 524, 493, 535, 461, 547, 494] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[518, 507, 504, 511, 454, 514, 511, 512, 504, 514, 508, 483, 536, 569, 499, 472, 487, 468, 545, 519, 516, 495, 508, 498, 527, 525, 541, 531, 514, 522, 504, 547, 500, 480, 508, 522, 504, 561, 510, 494, 526, 529, 535, 532, 495, 497, 544, 493, 484, 505, 522, 515, 509, 496, 511, 521, 519, 497, 488, 497, 531, 546, 534, 500] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[573, 511, 503, 499, 508, 515, 513, 508, 13, 13, 0, 0, 0, 0, 0, 0, 524, 496, 497, 531, 453, 485, 505, 535, 9, 0, 0, 0, 0, 0, 0, 0, 494, 522, 513, 539, 496, 505, 512, 558, 11, 3, 0, 0, 0, 0, 0, 0, 467, 489, 471, 495, 479, 500, 558, 534, 0, 0, 0, 0, 0, 0, 0, 0, 521, 501, 504, 502, 536, 506, 512, 517, 0, 0, 0, 0, 0, 0, 0, 0, 503, 483, 533, 514, 512, 509, 525, 500, 0, 0, 0, 0, 0, 0, 0, 0, 530, 552, 491, 525, 565, 480, 490, 522, 11, 0, 0, 0, 0, 0, 0, 0, 522, 489, 502, 511, 475, 535, 529, 495, 29, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4156, 4035, 4153, 3993, 4099, 4079, 4166, 4087] avg=4096.0000 max=4166 max_rank=6 min=3993 min_rank=3 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[526, 522, 534, 495, 499, 513, 517, 517, 13, 15, 0, 0, 0, 0, 0, 0, 505, 530, 516, 506, 549, 509, 522, 522, 9, 0, 0, 0, 0, 0, 0, 0, 483, 483, 544, 521, 535, 514, 485, 452, 11, 3, 0, 0, 0, 0, 0, 0, 505, 540, 511, 539, 509, 488, 493, 516, 0, 0, 0, 0, 0, 0, 0, 0, 515, 499, 531, 474, 515, 507, 515, 492, 0, 0, 0, 0, 0, 0, 0, 0, 504, 486, 507, 539, 479, 497, 525, 489, 0, 0, 0, 0, 0, 0, 0, 0, 522, 523, 525, 503, 498, 495, 520, 486, 11, 0, 0, 0, 0, 0, 0, 0, 502, 527, 514, 493, 518, 520, 544, 511, 31, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4151, 4168, 4031, 4101, 4048, 4026, 4083, 4160] avg=4096.0000 max=4168 max_rank=1 min=4026 min_rank=5 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16317 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[521, 510, 480, 520, 528, 513, 510, 506, 14, 14, 0, 0, 0, 0, 0, 0, 481, 493, 540, 556, 504, 516, 512, 510, 9, 0, 0, 0, 0, 0, 0, 0, 516, 500, 551, 566, 502, 522, 495, 480, 12, 3, 0, 0, 0, 0, 0, 0, 502, 507, 523, 496, 530, 481, 538, 490, 0, 0, 0, 0, 0, 0, 0, 0, 511, 508, 498, 506, 528, 495, 514, 551, 0, 0, 0, 0, 0, 0, 0, 0, 517, 520, 489, 548, 529, 550, 505, 494, 0, 0, 0, 0, 0, 0, 0, 0, 467, 484, 500, 530, 490, 495, 485, 497, 11, 0, 0, 0, 0, 0, 0, 0, 506, 504, 524, 493, 535, 461, 547, 494, 31, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4121, 4147, 4067, 4111, 4152, 3959, 4095] avg=4096.0000 max=4152 max_rank=5 min=3959 min_rank=6 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16317 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[475, 541, 493, 504, 534, 458, 518, 490, 14, 14, 0, 0, 0, 0, 0, 0, 489, 497, 529, 483, 545, 512, 529, 527, 9, 0, 0, 0, 0, 0, 0, 0, 495, 486, 503, 506, 526, 515, 524, 481, 12, 3, 0, 0, 0, 0, 0, 0, 481, 537, 520, 542, 527, 510, 518, 526, 0, 0, 0, 0, 0, 0, 0, 0, 531, 545, 484, 497, 511, 471, 551, 516, 0, 0, 0, 0, 0, 0, 0, 0, 531, 514, 500, 453, 564, 530, 539, 525, 0, 0, 0, 0, 0, 0, 0, 0, 494, 518, 515, 492, 502, 529, 544, 464, 10, 0, 0, 0, 0, 0, 0, 0, 477, 516, 524, 516, 496, 520, 510, 476, 30, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4041, 4120, 4051, 4161, 4106, 4156, 4068, 4065] avg=4096.0000 max=4161 max_rank=3 min=4041 min_rank=0 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16395 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[509, 486, 497, 485, 489, 515, 474, 531, 15, 14, 0, 0, 0, 0, 0, 0, 526, 514, 540, 538, 523, 484, 565, 508, 9, 0, 0, 0, 0, 0, 0, 0, 499, 545, 510, 495, 508, 464, 519, 536, 12, 3, 0, 0, 0, 0, 0, 0, 484, 531, 526, 536, 507, 497, 490, 505, 0, 0, 0, 0, 0, 0, 0, 0, 494, 514, 502, 518, 519, 472, 508, 550, 0, 0, 0, 0, 0, 0, 0, 0, 503, 532, 509, 493, 503, 466, 518, 537, 0, 0, 0, 0, 0, 0, 0, 0, 484, 480, 507, 558, 513, 541, 520, 558, 10, 0, 0, 0, 0, 0, 0, 0, 491, 511, 504, 501, 466, 493, 529, 545, 30, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4015, 4207, 4091, 4076, 4077, 4061, 4171, 4070] avg=4096.0000 max=4207 max_rank=1 min=4015 min_rank=0 imbalance_ratio=1.027100 max_load_reduction=28561 max_load_reduction_pct=0.871613 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[496, 543, 543, 485, 516, 506, 528, 521, 18, 17, 0, 0, 0, 0, 0, 0, 477, 463, 486, 511, 506, 497, 531, 511, 14, 0, 0, 0, 0, 0, 0, 0, 561, 528, 533, 512, 509, 532, 486, 477, 15, 7, 0, 0, 0, 0, 0, 0, 457, 494, 445, 525, 541, 545, 540, 526, 0, 0, 0, 0, 0, 0, 0, 0, 512, 520, 518, 502, 542, 517, 565, 514, 0, 0, 0, 0, 0, 0, 0, 0, 485, 453, 488, 491, 543, 523, 486, 545, 0, 0, 0, 0, 0, 0, 0, 0, 511, 522, 519, 491, 502, 518, 513, 513, 16, 0, 0, 0, 0, 0, 0, 0, 506, 449, 517, 488, 496, 518, 514, 538, 31, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4173, 3996, 4160, 4073, 4190, 4014, 4105, 4057] avg=4096.0000 max=4190 max_rank=4 min=3996 min_rank=1 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[494, 534, 523, 505, 494, 491, 487, 501, 14, 14, 0, 0, 0, 0, 0, 0, 506, 522, 483, 466, 511, 504, 514, 511, 9, 0, 0, 0, 0, 0, 0, 0, 521, 504, 499, 502, 506, 519, 494, 524, 12, 3, 0, 0, 0, 0, 0, 0, 469, 516, 509, 503, 507, 556, 516, 554, 0, 0, 0, 0, 0, 0, 0, 0, 519, 493, 531, 540, 484, 495, 489, 532, 0, 0, 0, 0, 0, 0, 0, 0, 509, 525, 491, 542, 548, 486, 504, 550, 0, 0, 0, 0, 0, 0, 0, 0, 527, 536, 509, 485, 506, 504, 549, 514, 11, 0, 0, 0, 0, 0, 0, 0, 482, 488, 493, 507, 539, 514, 509, 531, 29, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4057, 4026, 4084, 4130, 4083, 4155, 4141, 4092] avg=4096.0000 max=4155 max_rank=5 min=4026 min_rank=1 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16471 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[518, 507, 504, 511, 454, 514, 511, 512, 14, 14, 0, 0, 0, 0, 0, 0, 504, 514, 508, 483, 536, 569, 499, 472, 10, 0, 0, 0, 0, 0, 0, 0, 487, 468, 545, 519, 516, 495, 508, 498, 12, 3, 0, 0, 0, 0, 0, 0, 483, 525, 541, 531, 514, 522, 504, 547, 0, 0, 0, 0, 0, 0, 0, 0, 500, 480, 508, 522, 504, 536, 510, 494, 0, 0, 0, 0, 0, 0, 0, 0, 526, 503, 535, 532, 495, 497, 544, 493, 0, 0, 0, 0, 0, 0, 0, 0, 484, 505, 522, 515, 509, 496, 511, 521, 12, 0, 0, 0, 0, 0, 0, 0, 519, 497, 488, 497, 531, 546, 534, 500, 30, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4059, 4095, 4051, 4167, 4054, 4125, 4075, 4142] avg=4096.0000 max=4167 max_rank=3 min=4051 min_rank=2 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16396 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3968, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[546, 490, 516, 489, 503, 524, 501, 506, 499, 557, 531, 536, 484, 473, 527, 526, 527, 490, 505, 504, 513, 539, 502, 526, 466, 498, 549, 492, 508, 494, 492, 500, 510, 489, 514, 547, 536, 488, 522, 506, 536, 478, 477, 491, 500, 508, 543, 523, 479, 533, 492, 536, 518, 487, 529, 532, 511, 489, 534, 500, 541, 537, 526, 543] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[524, 532, 507, 546, 511, 526, 502, 504, 472, 487, 524, 520, 500, 518, 510, 513, 528, 500, 482, 513, 510, 542, 530, 511, 518, 540, 491, 536, 483, 467, 522, 514, 491, 538, 491, 526, 478, 516, 499, 482, 515, 521, 507, 495, 523, 491, 527, 500, 521, 499, 541, 505, 500, 519, 555, 540, 500, 504, 496, 517, 572, 527, 500, 489] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[511, 498, 521, 529, 505, 538, 478, 558, 479, 508, 535, 503, 496, 531, 512, 486, 483, 510, 494, 545, 520, 523, 493, 499, 536, 541, 476, 509, 509, 509, 546, 521, 504, 533, 531, 513, 492, 540, 515, 498, 530, 539, 555, 485, 489, 525, 503, 480, 520, 512, 530, 508, 506, 501, 510, 487, 515, 547, 510, 526, 481, 479, 477, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[523, 545, 518, 505, 517, 490, 555, 493, 506, 495, 498, 509, 505, 472, 522, 517, 517, 518, 510, 497, 504, 527, 504, 532, 525, 523, 534, 518, 481, 542, 507, 495, 543, 522, 510, 551, 499, 477, 505, 474, 523, 533, 526, 539, 470, 529, 509, 510, 522, 484, 518, 500, 531, 487, 533, 539, 493, 485, 534, 535, 497, 459, 540, 487] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[546, 501, 553, 497, 490, 536, 527, 501, 509, 533, 510, 487, 490, 498, 526, 507, 482, 546, 546, 496, 543, 483, 546, 515, 515, 485, 486, 543, 512, 507, 495, 530, 459, 481, 525, 523, 506, 525, 515, 523, 547, 488, 538, 530, 517, 506, 510, 508, 547, 520, 505, 491, 503, 537, 510, 531, 487, 474, 481, 517, 533, 477, 497, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[538, 503, 484, 494, 496, 491, 479, 514, 507, 519, 488, 526, 475, 487, 496, 535, 516, 528, 526, 522, 527, 513, 502, 509, 532, 513, 511, 533, 521, 533, 523, 534, 508, 507, 520, 496, 526, 511, 533, 510, 507, 551, 502, 469, 533, 482, 522, 529, 496, 507, 513, 521, 524, 521, 497, 492, 529, 507, 530, 529, 509, 487, 494, 531] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[513, 525, 536, 528, 511, 535, 534, 498, 522, 516, 469, 532, 509, 529, 500, 500, 508, 487, 531, 506, 527, 499, 499, 539, 488, 528, 526, 535, 527, 519, 498, 497, 528, 488, 538, 513, 483, 485, 538, 513, 510, 513, 507, 529, 470, 443, 505, 538, 511, 510, 503, 520, 514, 494, 520, 506, 515, 520, 477, 542, 506, 515, 526, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[506, 526, 491, 514, 517, 469, 521, 510, 512, 557, 519, 526, 503, 507, 477, 544, 527, 511, 500, 520, 530, 543, 551, 489, 480, 484, 509, 510, 490, 522, 479, 478, 495, 506, 496, 537, 510, 502, 520, 524, 513, 548, 470, 555, 530, 531, 522, 521, 534, 552, 451, 514, 512, 505, 527, 486, 470, 510, 511, 537, 503, 484, 528, 542] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[531, 490, 516, 489, 503, 524, 501, 506, 0, 0, 0, 0, 0, 0, 0, 0, 499, 557, 531, 536, 484, 473, 527, 526, 24, 0, 0, 0, 0, 0, 0, 0, 527, 490, 505, 504, 489, 539, 502, 526, 0, 0, 0, 0, 0, 0, 0, 0, 466, 498, 549, 492, 508, 494, 492, 500, 7, 3, 0, 0, 0, 0, 0, 0, 510, 489, 514, 547, 536, 488, 522, 506, 6, 4, 0, 0, 0, 0, 0, 0, 529, 478, 477, 491, 500, 508, 543, 523, 0, 0, 0, 0, 0, 0, 0, 0, 479, 533, 492, 536, 518, 487, 520, 532, 0, 0, 0, 0, 0, 0, 0, 0, 511, 489, 534, 500, 541, 537, 526, 543, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4157, 4082, 4009, 4122, 4049, 4097, 4192] avg=4096.0000 max=4192 max_rank=7 min=4009 min_rank=3 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16460 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[497, 498, 521, 529, 505, 538, 478, 558, 0, 0, 0, 0, 0, 0, 0, 0, 479, 508, 535, 503, 496, 531, 512, 486, 24, 0, 0, 0, 0, 0, 0, 0, 483, 510, 494, 545, 496, 523, 493, 499, 0, 0, 0, 0, 0, 0, 0, 0, 536, 541, 476, 509, 509, 509, 546, 521, 7, 3, 0, 0, 0, 0, 0, 0, 504, 533, 531, 513, 492, 540, 515, 498, 6, 3, 0, 0, 0, 0, 0, 0, 523, 539, 555, 485, 489, 525, 503, 480, 0, 0, 0, 0, 0, 0, 0, 0, 520, 512, 530, 508, 506, 501, 501, 487, 0, 0, 0, 0, 0, 0, 0, 0, 515, 547, 510, 526, 481, 479, 477, 525, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4124, 4074, 4043, 4157, 4135, 4099, 4065, 4071] avg=4096.0000 max=4157 max_rank=3 min=4043 min_rank=2 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16370 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[531, 501, 553, 497, 490, 536, 527, 501, 0, 0, 0, 0, 0, 0, 0, 0, 509, 533, 510, 487, 490, 498, 526, 507, 25, 0, 0, 0, 0, 0, 0, 0, 482, 546, 546, 496, 518, 483, 546, 515, 0, 0, 0, 0, 0, 0, 0, 0, 515, 485, 486, 543, 512, 507, 495, 530, 7, 3, 0, 0, 0, 0, 0, 0, 459, 481, 525, 523, 506, 525, 515, 523, 6, 4, 0, 0, 0, 0, 0, 0, 540, 488, 538, 530, 517, 506, 510, 508, 0, 0, 0, 0, 0, 0, 0, 0, 547, 520, 505, 491, 503, 537, 501, 531, 0, 0, 0, 0, 0, 0, 0, 0, 487, 474, 481, 517, 533, 477, 497, 516, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4136, 4085, 4132, 4083, 4067, 4137, 4135, 3993] avg=4096.0000 max=4137 max_rank=5 min=3993 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16332 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[510, 532, 507, 546, 511, 526, 502, 504, 0, 0, 0, 0, 0, 0, 0, 0, 472, 487, 524, 520, 500, 518, 510, 513, 24, 0, 0, 0, 0, 0, 0, 0, 528, 500, 482, 513, 486, 542, 530, 511, 0, 0, 0, 0, 0, 0, 0, 0, 518, 540, 491, 536, 483, 467, 522, 514, 6, 3, 0, 0, 0, 0, 0, 0, 491, 538, 491, 526, 478, 516, 499, 482, 7, 3, 0, 0, 0, 0, 0, 0, 509, 521, 507, 495, 523, 491, 527, 500, 0, 0, 0, 0, 0, 0, 0, 0, 521, 499, 541, 505, 500, 519, 545, 540, 0, 0, 0, 0, 0, 0, 0, 0, 500, 504, 496, 517, 572, 527, 500, 489, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4138, 4068, 4092, 4080, 4031, 4073, 4170, 4116] avg=4096.0000 max=4170 max_rank=6 min=4031 min_rank=4 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[495, 545, 518, 505, 517, 490, 555, 493, 0, 0, 0, 0, 0, 0, 0, 0, 506, 495, 498, 509, 505, 472, 522, 517, 27, 5, 0, 0, 0, 0, 0, 0, 517, 518, 510, 497, 477, 527, 504, 532, 0, 0, 0, 0, 0, 0, 0, 0, 525, 523, 534, 518, 481, 542, 507, 495, 11, 6, 0, 0, 0, 0, 0, 0, 543, 522, 510, 551, 499, 477, 505, 474, 10, 8, 0, 0, 0, 0, 0, 0, 512, 533, 526, 539, 470, 529, 509, 510, 0, 0, 0, 0, 0, 0, 0, 0, 522, 484, 518, 500, 531, 487, 517, 539, 0, 0, 0, 0, 0, 0, 0, 0, 493, 485, 534, 535, 497, 459, 540, 487, 15, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4056, 4082, 4142, 4099, 4128, 4098, 4045] avg=4096.0000 max=4142 max_rank=3 min=4045 min_rank=7 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16370 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[524, 503, 484, 494, 496, 491, 479, 514, 0, 0, 0, 0, 0, 0, 0, 0, 507, 519, 488, 526, 475, 487, 496, 535, 24, 0, 0, 0, 0, 0, 0, 0, 516, 528, 526, 522, 503, 513, 502, 509, 0, 0, 0, 0, 0, 0, 0, 0, 532, 513, 511, 533, 521, 533, 523, 534, 6, 3, 0, 0, 0, 0, 0, 0, 508, 507, 520, 496, 526, 511, 533, 510, 6, 3, 0, 0, 0, 0, 0, 0, 501, 551, 502, 469, 533, 482, 522, 529, 0, 0, 0, 0, 0, 0, 0, 0, 496, 507, 513, 521, 524, 521, 488, 492, 0, 0, 0, 0, 0, 0, 0, 0, 529, 507, 530, 529, 509, 487, 494, 531, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3985, 4057, 4119, 4209, 4120, 4089, 4062, 4127] avg=4096.0000 max=4209 max_rank=3 min=3985 min_rank=0 imbalance_ratio=1.027588 max_load_reduction=28559 max_load_reduction_pct=0.871552 moved_to_echo=16398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[499, 525, 536, 528, 511, 535, 534, 498, 0, 0, 0, 0, 0, 0, 0, 0, 522, 516, 469, 532, 509, 529, 500, 500, 24, 0, 0, 0, 0, 0, 0, 0, 508, 487, 531, 506, 503, 499, 499, 539, 0, 0, 0, 0, 0, 0, 0, 0, 488, 528, 526, 535, 527, 519, 498, 497, 6, 3, 0, 0, 0, 0, 0, 0, 528, 488, 538, 513, 483, 485, 538, 513, 6, 3, 0, 0, 0, 0, 0, 0, 504, 513, 507, 529, 470, 443, 505, 538, 0, 0, 0, 0, 0, 0, 0, 0, 511, 510, 503, 520, 514, 494, 511, 506, 0, 0, 0, 0, 0, 0, 0, 0, 515, 520, 477, 542, 506, 515, 526, 517, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4166, 4101, 4072, 4127, 4095, 4009, 4069, 4129] avg=4096.0000 max=4166 max_rank=0 min=4009 min_rank=5 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16302 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[493, 526, 491, 514, 517, 469, 521, 510, 0, 0, 0, 0, 0, 0, 0, 0, 512, 557, 519, 526, 503, 507, 477, 544, 25, 0, 0, 0, 0, 0, 0, 0, 527, 511, 500, 520, 505, 543, 551, 489, 0, 0, 0, 0, 0, 0, 0, 0, 480, 484, 509, 510, 490, 522, 479, 478, 6, 3, 0, 0, 0, 0, 0, 0, 495, 506, 496, 537, 510, 502, 520, 524, 6, 3, 0, 0, 0, 0, 0, 0, 507, 548, 470, 555, 530, 531, 522, 521, 0, 0, 0, 0, 0, 0, 0, 0, 534, 552, 451, 514, 512, 505, 518, 486, 0, 0, 0, 0, 0, 0, 0, 0, 470, 510, 511, 537, 503, 484, 528, 542, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4041, 4170, 4146, 3961, 4099, 4184, 4072, 4095] avg=4096.0000 max=4184 max_rank=5 min=3961 min_rank=3 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16450 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 3968, 4096, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 3968, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True + [2026-09-21 08:26:47] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 407.8 | throughput per GPU (TFLOP/s/GPU): 79.0 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.121118E+01 | loss scale: 1.0 | grad norm: 35.604 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[494, 504, 535, 521, 540, 482, 486, 489, 521, 511, 527, 483, 496, 463, 520, 523, 526, 498, 534, 479, 515, 510, 488, 489, 545, 506, 501, 490, 515, 556, 507, 496, 527, 537, 500, 495, 500, 537, 478, 571, 498, 498, 532, 542, 485, 481, 500, 497, 532, 550, 506, 509, 531, 501, 532, 523, 543, 530, 522, 514, 478, 533, 526, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[518, 541, 520, 558, 512, 532, 536, 531, 441, 525, 529, 543, 528, 495, 539, 543, 515, 539, 543, 487, 519, 504, 517, 507, 522, 528, 503, 504, 511, 494, 512, 509, 500, 512, 537, 494, 517, 536, 531, 517, 495, 503, 473, 523, 508, 521, 502, 431, 515, 488, 485, 506, 495, 522, 538, 525, 517, 515, 481, 519, 541, 488, 477, 451] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[522, 528, 529, 533, 525, 515, 490, 528, 544, 551, 529, 525, 536, 516, 499, 519, 502, 512, 510, 550, 519, 513, 532, 553, 532, 500, 522, 477, 478, 493, 507, 542, 531, 496, 511, 505, 513, 486, 488, 536, 490, 488, 515, 515, 545, 478, 495, 532, 536, 525, 496, 488, 479, 507, 505, 530, 476, 479, 475, 493, 468, 505, 527, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[534, 513, 505, 523, 489, 506, 507, 516, 504, 504, 526, 500, 523, 516, 506, 531, 492, 504, 480, 541, 505, 506, 541, 494, 509, 495, 515, 527, 523, 507, 492, 542, 508, 502, 469, 492, 478, 512, 509, 488, 506, 542, 533, 501, 493, 520, 530, 535, 509, 501, 520, 506, 543, 499, 516, 480, 531, 505, 541, 550, 513, 522, 532, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[506, 510, 520, 493, 496, 531, 502, 532, 493, 490, 541, 536, 525, 486, 476, 514, 526, 507, 509, 525, 497, 543, 501, 529, 515, 546, 498, 515, 516, 513, 509, 532, 518, 503, 493, 536, 515, 498, 507, 514, 535, 527, 526, 534, 514, 538, 506, 530, 504, 486, 523, 497, 482, 497, 501, 486, 538, 502, 530, 536, 482, 508, 481, 489] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[538, 520, 480, 521, 536, 519, 473, 501, 512, 504, 505, 520, 576, 521, 501, 502, 480, 523, 533, 492, 517, 499, 540, 499, 522, 531, 474, 538, 487, 494, 499, 487, 536, 501, 544, 467, 527, 523, 555, 502, 484, 483, 515, 525, 559, 514, 492, 494, 512, 468, 510, 512, 511, 468, 515, 518, 558, 504, 524, 522, 529, 509, 548, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[533, 527, 524, 532, 487, 532, 477, 534, 543, 489, 504, 510, 483, 493, 495, 502, 516, 478, 524, 481, 521, 533, 531, 495, 504, 516, 517, 493, 495, 523, 498, 557, 499, 499, 506, 524, 495, 502, 528, 509, 492, 549, 496, 502, 545, 508, 462, 544, 495, 525, 485, 529, 520, 517, 527, 512, 506, 541, 518, 515, 510, 541, 511, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[505, 492, 488, 494, 482, 509, 531, 486, 517, 507, 537, 501, 492, 480, 514, 538, 526, 566, 530, 527, 491, 548, 470, 488, 499, 517, 481, 518, 563, 531, 487, 507, 552, 552, 477, 490, 524, 511, 486, 484, 503, 491, 511, 515, 531, 502, 523, 537, 516, 509, 545, 522, 499, 527, 522, 537, 503, 525, 517, 529, 492, 519, 474, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[522, 528, 529, 521, 525, 515, 490, 528, 0, 0, 0, 0, 0, 0, 0, 0, 544, 551, 523, 525, 536, 516, 499, 519, 0, 0, 0, 0, 0, 0, 0, 0, 502, 512, 498, 550, 519, 513, 532, 553, 0, 0, 0, 0, 0, 0, 0, 0, 532, 500, 522, 477, 478, 493, 507, 542, 2, 1, 0, 0, 0, 0, 0, 0, 531, 496, 511, 505, 513, 486, 488, 536, 8, 1, 0, 0, 0, 0, 0, 0, 490, 488, 515, 515, 545, 478, 495, 532, 4, 4, 0, 0, 0, 0, 0, 0, 536, 525, 496, 488, 479, 507, 505, 530, 11, 0, 0, 0, 0, 0, 0, 0, 476, 479, 475, 492, 468, 505, 527, 524, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4158, 4213, 4179, 4054, 4075, 4066, 4077, 3946] avg=4096.0000 max=4213 max_rank=1 min=3946 min_rank=7 imbalance_ratio=1.028564 max_load_reduction=28555 max_load_reduction_pct=0.871429 moved_to_echo=16164 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[518, 541, 520, 545, 512, 532, 536, 531, 0, 0, 0, 0, 0, 0, 0, 0, 441, 525, 523, 543, 528, 495, 539, 543, 0, 0, 0, 0, 0, 0, 0, 0, 515, 539, 531, 487, 519, 504, 517, 507, 0, 0, 0, 0, 0, 0, 0, 0, 522, 528, 503, 504, 511, 494, 512, 509, 2, 1, 0, 0, 0, 0, 0, 0, 500, 512, 537, 494, 517, 536, 531, 517, 8, 1, 0, 0, 0, 0, 0, 0, 495, 503, 473, 523, 508, 521, 502, 431, 4, 4, 0, 0, 0, 0, 0, 0, 515, 488, 485, 506, 495, 522, 538, 525, 12, 0, 0, 0, 0, 0, 0, 0, 517, 515, 481, 518, 541, 488, 477, 451, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4235, 4137, 4119, 4086, 4153, 3964, 4086, 3988] avg=4096.0000 max=4235 max_rank=0 min=3964 min_rank=5 imbalance_ratio=1.033936 max_load_reduction=28533 max_load_reduction_pct=0.870758 moved_to_echo=16191 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[494, 504, 535, 509, 540, 482, 486, 489, 0, 0, 0, 0, 0, 0, 0, 0, 521, 511, 521, 483, 496, 463, 520, 523, 0, 0, 0, 0, 0, 0, 0, 0, 526, 498, 522, 479, 515, 510, 488, 489, 0, 0, 0, 0, 0, 0, 0, 0, 545, 506, 501, 490, 515, 556, 507, 496, 2, 1, 0, 0, 0, 0, 0, 0, 527, 537, 500, 495, 500, 537, 478, 571, 8, 1, 0, 0, 0, 0, 0, 0, 498, 498, 532, 542, 485, 481, 500, 497, 4, 4, 0, 0, 0, 0, 0, 0, 532, 550, 506, 509, 531, 501, 532, 523, 11, 0, 0, 0, 0, 0, 0, 0, 543, 530, 522, 513, 478, 533, 526, 510, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4039, 4038, 4027, 4119, 4154, 4041, 4195, 4155] avg=4096.0000 max=4195 max_rank=6 min=4027 min_rank=2 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16545 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[506, 510, 520, 482, 496, 531, 502, 532, 0, 0, 0, 0, 0, 0, 0, 0, 493, 490, 534, 536, 525, 486, 476, 514, 0, 0, 0, 0, 0, 0, 0, 0, 526, 507, 497, 525, 497, 543, 501, 529, 0, 0, 0, 0, 0, 0, 0, 0, 515, 546, 498, 515, 516, 513, 509, 532, 2, 1, 0, 0, 0, 0, 0, 0, 518, 503, 493, 536, 515, 498, 507, 514, 8, 1, 0, 0, 0, 0, 0, 0, 535, 527, 526, 534, 514, 538, 506, 530, 5, 4, 0, 0, 0, 0, 0, 0, 504, 486, 523, 497, 482, 497, 501, 486, 10, 0, 0, 0, 0, 0, 0, 0, 538, 502, 530, 535, 482, 508, 481, 489, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4054, 4125, 4147, 4093, 4219, 3986, 4065] avg=4096.0000 max=4219 max_rank=5 min=3986 min_rank=6 imbalance_ratio=1.030029 max_load_reduction=28549 max_load_reduction_pct=0.871246 moved_to_echo=16363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[534, 513, 505, 502, 489, 506, 507, 516, 0, 0, 0, 0, 0, 0, 0, 0, 504, 504, 512, 500, 523, 516, 506, 531, 0, 0, 0, 0, 0, 0, 0, 0, 492, 504, 463, 541, 505, 506, 541, 494, 0, 0, 0, 0, 0, 0, 0, 0, 509, 495, 515, 527, 523, 507, 492, 542, 4, 3, 0, 0, 0, 0, 0, 0, 508, 502, 469, 492, 478, 512, 509, 488, 10, 5, 0, 0, 0, 0, 0, 0, 506, 542, 533, 501, 493, 520, 530, 535, 10, 7, 0, 0, 0, 0, 0, 0, 509, 501, 520, 506, 543, 499, 516, 480, 16, 0, 0, 0, 0, 0, 0, 0, 531, 505, 541, 547, 513, 522, 532, 506, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4096, 4046, 4117, 3973, 4177, 4090, 4197] avg=4096.0000 max=4197 max_rank=7 min=3973 min_rank=4 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16437 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[538, 520, 480, 509, 536, 519, 473, 501, 0, 0, 0, 0, 0, 0, 0, 0, 512, 504, 500, 520, 576, 521, 501, 502, 0, 0, 0, 0, 0, 0, 0, 0, 480, 523, 521, 492, 517, 499, 540, 499, 0, 0, 0, 0, 0, 0, 0, 0, 522, 531, 474, 538, 487, 494, 499, 487, 1, 1, 0, 0, 0, 0, 0, 0, 536, 501, 544, 467, 527, 523, 555, 502, 8, 1, 0, 0, 0, 0, 0, 0, 484, 483, 515, 525, 559, 514, 492, 494, 4, 4, 0, 0, 0, 0, 0, 0, 512, 468, 510, 512, 511, 468, 515, 518, 11, 0, 0, 0, 0, 0, 0, 0, 558, 504, 524, 521, 529, 509, 548, 495, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4136, 4071, 4034, 4164, 4074, 4025, 4188] avg=4096.0000 max=4188 max_rank=7 min=4025 min_rank=6 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16451 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[533, 527, 524, 520, 487, 532, 477, 534, 0, 0, 0, 0, 0, 0, 0, 0, 543, 489, 499, 510, 483, 493, 495, 502, 0, 0, 0, 0, 0, 0, 0, 0, 516, 478, 512, 481, 521, 533, 531, 495, 0, 0, 0, 0, 0, 0, 0, 0, 504, 516, 517, 493, 495, 523, 498, 557, 1, 1, 0, 0, 0, 0, 0, 0, 499, 499, 506, 524, 495, 502, 528, 509, 8, 1, 0, 0, 0, 0, 0, 0, 492, 549, 496, 502, 545, 508, 462, 544, 4, 4, 0, 0, 0, 0, 0, 0, 495, 525, 485, 529, 520, 517, 527, 512, 11, 0, 0, 0, 0, 0, 0, 0, 506, 541, 518, 514, 510, 541, 511, 509, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4134, 4014, 4067, 4105, 4071, 4106, 4121, 4150] avg=4096.0000 max=4150 max_rank=7 min=4014 min_rank=1 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16448 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[505, 492, 488, 482, 482, 509, 531, 486, 0, 0, 0, 0, 0, 0, 0, 0, 517, 507, 531, 501, 492, 480, 514, 538, 0, 0, 0, 0, 0, 0, 0, 0, 526, 566, 518, 527, 491, 548, 470, 488, 0, 0, 0, 0, 0, 0, 0, 0, 499, 517, 481, 518, 563, 531, 487, 507, 2, 1, 0, 0, 0, 0, 0, 0, 552, 552, 477, 490, 524, 511, 486, 484, 8, 1, 0, 0, 0, 0, 0, 0, 503, 491, 511, 515, 531, 502, 523, 537, 4, 4, 0, 0, 0, 0, 0, 0, 516, 509, 545, 522, 499, 527, 522, 537, 11, 0, 0, 0, 0, 0, 0, 0, 503, 525, 517, 528, 492, 519, 474, 521, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3975, 4080, 4134, 4106, 4085, 4121, 4188, 4079] avg=4096.0000 max=4188 max_rank=6 min=3975 min_rank=0 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16473 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[554, 501, 495, 517, 516, 511, 547, 489, 509, 480, 536, 554, 541, 484, 509, 507, 501, 507, 498, 542, 509, 523, 487, 489, 509, 536, 501, 525, 485, 487, 483, 490, 504, 524, 516, 553, 462, 491, 492, 488, 514, 512, 509, 513, 472, 493, 521, 557, 522, 542, 515, 512, 509, 529, 548, 497, 551, 511, 496, 493, 529, 513, 543, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[504, 551, 559, 513, 515, 504, 495, 488, 547, 524, 514, 510, 471, 494, 517, 551, 538, 472, 468, 526, 556, 476, 506, 489, 530, 479, 511, 550, 489, 470, 500, 527, 475, 491, 519, 504, 532, 506, 503, 537, 508, 534, 524, 520, 522, 532, 460, 503, 534, 522, 512, 517, 534, 490, 532, 492, 532, 494, 512, 492, 512, 518, 550, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[486, 498, 502, 517, 511, 518, 500, 512, 520, 494, 501, 502, 510, 509, 487, 486, 543, 471, 460, 516, 514, 523, 498, 509, 511, 499, 516, 517, 502, 543, 524, 498, 526, 519, 504, 543, 503, 516, 526, 521, 527, 516, 551, 567, 529, 495, 467, 549, 515, 503, 529, 496, 534, 541, 488, 535, 522, 499, 526, 476, 515, 522, 515, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[534, 529, 526, 535, 538, 522, 519, 498, 499, 492, 525, 505, 499, 527, 511, 521, 504, 544, 528, 532, 496, 487, 533, 500, 483, 499, 522, 485, 469, 531, 526, 554, 532, 505, 552, 513, 545, 521, 482, 533, 516, 507, 508, 497, 473, 513, 504, 517, 522, 490, 483, 522, 486, 464, 506, 563, 519, 521, 504, 478, 498, 475, 541, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[496, 525, 536, 498, 495, 523, 513, 530, 502, 519, 517, 510, 537, 510, 499, 514, 457, 482, 550, 505, 560, 523, 507, 517, 501, 522, 559, 486, 505, 512, 548, 486, 503, 518, 521, 498, 519, 516, 519, 548, 517, 461, 544, 541, 532, 493, 528, 500, 498, 503, 536, 512, 500, 512, 490, 508, 492, 505, 503, 493, 497, 508, 519, 490] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[520, 522, 478, 496, 499, 480, 517, 488, 539, 485, 495, 512, 544, 508, 498, 541, 526, 505, 486, 502, 497, 501, 499, 551, 499, 522, 543, 518, 520, 494, 517, 471, 517, 500, 545, 534, 513, 551, 521, 520, 530, 476, 531, 474, 486, 508, 511, 518, 505, 500, 531, 494, 513, 495, 539, 508, 528, 506, 488, 548, 527, 537, 523, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[509, 530, 482, 538, 499, 474, 533, 518, 496, 513, 523, 505, 510, 503, 509, 501, 494, 491, 520, 501, 529, 544, 526, 522, 480, 524, 486, 502, 459, 517, 481, 484, 527, 511, 519, 523, 504, 546, 501, 510, 556, 483, 532, 508, 556, 475, 533, 466, 519, 543, 527, 532, 470, 538, 531, 485, 514, 530, 521, 536, 513, 512, 506, 538] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[519, 531, 538, 512, 524, 535, 503, 535, 486, 530, 506, 485, 549, 566, 538, 511, 493, 476, 502, 484, 471, 533, 468, 534, 497, 517, 496, 493, 549, 522, 509, 506, 504, 477, 497, 499, 524, 506, 523, 530, 544, 483, 508, 538, 522, 498, 484, 572, 476, 531, 532, 505, 487, 460, 521, 493, 526, 500, 477, 533, 480, 553, 525, 542] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[554, 486, 495, 517, 516, 511, 547, 489, 0, 0, 0, 0, 0, 0, 0, 0, 509, 480, 536, 554, 538, 484, 509, 507, 0, 0, 0, 0, 0, 0, 0, 0, 501, 507, 498, 542, 509, 523, 487, 489, 13, 8, 6, 3, 1, 0, 0, 0, 509, 536, 501, 525, 485, 487, 483, 490, 24, 9, 0, 0, 0, 0, 0, 0, 504, 524, 516, 553, 462, 491, 492, 464, 0, 0, 0, 0, 0, 0, 0, 0, 506, 512, 509, 513, 472, 493, 521, 557, 0, 0, 0, 0, 0, 0, 0, 0, 522, 542, 514, 512, 509, 529, 548, 497, 0, 0, 0, 0, 0, 0, 0, 0, 551, 511, 496, 493, 529, 513, 530, 515, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4117, 4087, 4049, 4006, 4083, 4173, 4138] avg=4096.0000 max=4173 max_rank=6 min=4006 min_rank=4 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16400 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[504, 534, 559, 513, 515, 504, 495, 488, 0, 0, 0, 0, 0, 0, 0, 0, 547, 524, 514, 510, 468, 494, 517, 551, 0, 0, 0, 0, 0, 0, 0, 0, 538, 472, 468, 526, 556, 476, 506, 489, 13, 8, 7, 3, 1, 0, 0, 0, 530, 479, 511, 550, 489, 470, 500, 527, 27, 10, 0, 0, 0, 0, 0, 0, 475, 491, 519, 504, 532, 506, 503, 510, 0, 0, 0, 0, 0, 0, 0, 0, 500, 534, 524, 520, 522, 532, 460, 503, 0, 0, 0, 0, 0, 0, 0, 0, 534, 522, 511, 517, 534, 490, 532, 492, 0, 0, 0, 0, 0, 0, 0, 0, 532, 494, 512, 492, 512, 518, 537, 511, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4125, 4063, 4093, 4040, 4095, 4132, 4108] avg=4096.0000 max=4132 max_rank=6 min=4040 min_rank=4 imbalance_ratio=1.008789 max_load_reduction=28636 max_load_reduction_pct=0.873901 moved_to_echo=16375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[486, 483, 502, 517, 511, 518, 500, 512, 0, 0, 0, 0, 0, 0, 0, 0, 520, 494, 501, 502, 507, 509, 487, 486, 0, 0, 0, 0, 0, 0, 0, 0, 543, 471, 460, 516, 514, 523, 498, 509, 12, 8, 6, 3, 1, 0, 0, 0, 511, 499, 516, 517, 502, 543, 524, 498, 26, 9, 0, 0, 0, 0, 0, 0, 526, 519, 504, 543, 503, 516, 526, 495, 0, 0, 0, 0, 0, 0, 0, 0, 519, 516, 551, 567, 529, 495, 467, 549, 0, 0, 0, 0, 0, 0, 0, 0, 515, 503, 528, 496, 534, 541, 488, 535, 0, 0, 0, 0, 0, 0, 0, 0, 522, 499, 526, 476, 515, 522, 503, 496, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4029, 4006, 4064, 4145, 4132, 4193, 4140, 4059] avg=4096.0000 max=4193 max_rank=5 min=4006 min_rank=1 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16524 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[520, 507, 478, 496, 499, 480, 517, 488, 0, 0, 0, 0, 0, 0, 0, 0, 539, 485, 495, 512, 541, 508, 498, 541, 0, 0, 0, 0, 0, 0, 0, 0, 526, 505, 486, 502, 497, 501, 499, 551, 12, 8, 6, 3, 1, 0, 0, 0, 499, 522, 543, 518, 520, 494, 517, 471, 26, 9, 0, 0, 0, 0, 0, 0, 517, 500, 545, 534, 513, 551, 521, 494, 0, 0, 0, 0, 0, 0, 0, 0, 522, 476, 531, 474, 486, 508, 511, 518, 0, 0, 0, 0, 0, 0, 0, 0, 505, 500, 530, 494, 513, 495, 539, 508, 0, 0, 0, 0, 0, 0, 0, 0, 528, 506, 488, 548, 527, 537, 511, 518, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3985, 4119, 4097, 4119, 4175, 4026, 4084, 4163] avg=4096.0000 max=4175 max_rank=4 min=3985 min_rank=0 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16448 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[534, 514, 526, 535, 538, 522, 519, 498, 0, 0, 0, 0, 0, 0, 0, 0, 499, 492, 525, 505, 496, 527, 511, 521, 0, 0, 0, 0, 0, 0, 0, 0, 504, 544, 528, 532, 496, 487, 533, 500, 13, 8, 6, 3, 1, 0, 0, 0, 483, 499, 522, 485, 469, 531, 526, 554, 27, 9, 0, 0, 0, 0, 0, 0, 532, 505, 552, 513, 545, 521, 482, 506, 0, 0, 0, 0, 0, 0, 0, 0, 508, 507, 508, 497, 473, 513, 504, 517, 0, 0, 0, 0, 0, 0, 0, 0, 522, 490, 482, 522, 486, 464, 506, 563, 0, 0, 0, 0, 0, 0, 0, 0, 519, 521, 504, 478, 498, 475, 528, 505, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4186, 4076, 4155, 4105, 4156, 4027, 4035, 4028] avg=4096.0000 max=4186 max_rank=0 min=4027 min_rank=5 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16246 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[496, 500, 536, 498, 495, 523, 513, 530, 0, 0, 0, 0, 0, 0, 0, 0, 502, 519, 517, 510, 529, 510, 499, 514, 0, 0, 0, 0, 0, 0, 0, 0, 457, 482, 550, 505, 560, 523, 507, 517, 17, 12, 11, 8, 3, 0, 0, 0, 501, 522, 559, 486, 505, 512, 548, 486, 31, 14, 0, 0, 0, 0, 0, 0, 503, 518, 521, 498, 519, 516, 519, 517, 0, 0, 0, 0, 0, 0, 0, 0, 505, 461, 544, 541, 532, 493, 528, 500, 0, 0, 0, 0, 0, 0, 0, 0, 498, 503, 533, 512, 500, 512, 490, 508, 0, 0, 0, 0, 0, 0, 0, 0, 492, 505, 503, 493, 497, 508, 502, 490, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4100, 4152, 4164, 4111, 4104, 4056, 3990] avg=4096.0000 max=4164 max_rank=3 min=3990 min_rank=7 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16261 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[519, 516, 538, 512, 524, 535, 503, 535, 0, 0, 0, 0, 0, 0, 0, 0, 486, 530, 506, 485, 546, 566, 538, 511, 0, 0, 0, 0, 0, 0, 0, 0, 493, 476, 502, 484, 471, 533, 468, 534, 12, 9, 6, 3, 1, 0, 0, 0, 497, 517, 496, 493, 549, 522, 509, 506, 27, 9, 0, 0, 0, 0, 0, 0, 504, 477, 497, 499, 524, 506, 523, 503, 0, 0, 0, 0, 0, 0, 0, 0, 535, 483, 508, 538, 522, 498, 484, 572, 0, 0, 0, 0, 0, 0, 0, 0, 476, 531, 531, 505, 487, 460, 521, 493, 0, 0, 0, 0, 0, 0, 0, 0, 526, 500, 477, 533, 480, 553, 513, 542, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4182, 4168, 3992, 4125, 4033, 4140, 4004, 4124] avg=4096.0000 max=4182 max_rank=0 min=3992 min_rank=2 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16301 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[509, 515, 482, 538, 499, 474, 533, 518, 0, 0, 0, 0, 0, 0, 0, 0, 496, 513, 523, 505, 507, 503, 509, 501, 0, 0, 0, 0, 0, 0, 0, 0, 494, 491, 520, 501, 529, 544, 526, 522, 12, 9, 6, 3, 1, 0, 0, 0, 480, 524, 486, 502, 459, 517, 481, 484, 26, 9, 0, 0, 0, 0, 0, 0, 527, 511, 519, 523, 504, 546, 501, 484, 0, 0, 0, 0, 0, 0, 0, 0, 547, 483, 532, 508, 556, 475, 533, 466, 0, 0, 0, 0, 0, 0, 0, 0, 519, 543, 526, 532, 470, 538, 531, 485, 0, 0, 0, 0, 0, 0, 0, 0, 514, 530, 521, 536, 513, 512, 494, 538, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4057, 4158, 3968, 4115, 4100, 4144, 4158] avg=4096.0000 max=4158 max_rank=2 min=3968 min_rank=3 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16517 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[4096, 3968, 4096, 4224, 4224, 4224, 4096, 4224, 128, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33792 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[476, 503, 562, 488, 519, 493, 519, 549, 516, 557, 515, 556, 503, 498, 517, 540, 503, 517, 505, 503, 536, 447, 483, 506, 546, 485, 500, 508, 509, 514, 484, 468, 501, 527, 518, 520, 551, 495, 544, 559, 508, 525, 504, 529, 531, 474, 488, 510, 479, 504, 527, 553, 518, 497, 506, 478, 510, 508, 490, 517, 516, 508, 506, 542] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[536, 524, 488, 530, 505, 531, 495, 473, 513, 523, 527, 506, 471, 557, 550, 529, 498, 490, 509, 513, 525, 540, 488, 520, 499, 498, 529, 514, 494, 501, 525, 532, 505, 535, 518, 479, 532, 495, 530, 471, 525, 504, 503, 524, 527, 489, 510, 511, 500, 485, 503, 494, 509, 471, 521, 503, 509, 551, 535, 483, 511, 522, 556, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[512, 509, 539, 542, 519, 512, 543, 519, 510, 481, 527, 529, 479, 510, 541, 505, 477, 508, 510, 509, 534, 488, 513, 479, 497, 501, 516, 489, 537, 500, 493, 491, 524, 514, 521, 504, 531, 534, 472, 503, 511, 507, 506, 504, 507, 533, 550, 550, 516, 511, 503, 503, 493, 544, 514, 475, 498, 502, 544, 498, 532, 523, 531, 491] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[483, 485, 530, 535, 513, 477, 495, 486, 501, 544, 516, 513, 518, 493, 502, 511, 534, 514, 563, 496, 494, 499, 509, 503, 521, 490, 523, 492, 523, 540, 492, 519, 507, 545, 524, 506, 489, 538, 532, 516, 517, 528, 521, 511, 542, 515, 555, 518, 498, 516, 491, 509, 494, 496, 486, 538, 515, 499, 515, 491, 524, 509, 521, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[545, 529, 549, 552, 496, 527, 505, 541, 494, 557, 510, 514, 514, 526, 486, 511, 504, 535, 527, 539, 496, 484, 506, 508, 541, 519, 508, 475, 494, 491, 507, 510, 555, 543, 514, 527, 478, 502, 468, 509, 508, 482, 524, 481, 480, 489, 523, 473, 523, 543, 524, 516, 522, 510, 506, 522, 482, 486, 508, 502, 494, 537, 534, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[526, 509, 516, 567, 521, 521, 493, 507, 506, 535, 465, 515, 532, 517, 520, 525, 474, 532, 520, 457, 501, 533, 514, 448, 500, 515, 532, 474, 529, 536, 497, 504, 533, 463, 552, 523, 505, 494, 492, 508, 491, 541, 503, 507, 501, 522, 495, 496, 498, 538, 558, 519, 520, 494, 521, 490, 495, 487, 545, 522, 524, 521, 514, 555] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[492, 477, 518, 494, 505, 535, 497, 499, 495, 553, 459, 469, 511, 536, 532, 490, 530, 544, 498, 522, 520, 524, 501, 488, 521, 531, 521, 524, 545, 504, 508, 559, 516, 518, 510, 516, 512, 518, 510, 524, 509, 519, 476, 529, 497, 497, 570, 533, 495, 515, 461, 515, 514, 500, 477, 512, 505, 567, 540, 518, 505, 480, 471, 537] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[474, 507, 536, 554, 520, 513, 477, 538, 517, 495, 535, 500, 511, 484, 516, 503, 488, 478, 505, 551, 507, 525, 510, 517, 480, 548, 515, 512, 544, 488, 562, 510, 555, 512, 523, 492, 524, 504, 542, 524, 495, 530, 521, 513, 491, 520, 513, 495, 508, 527, 529, 508, 520, 517, 491, 518, 524, 505, 458, 492, 498, 481, 526, 492] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[476, 503, 562, 471, 519, 493, 519, 549, 0, 0, 0, 0, 0, 0, 0, 0, 516, 542, 515, 556, 503, 498, 517, 540, 0, 0, 0, 0, 0, 0, 0, 0, 503, 517, 505, 503, 536, 447, 483, 506, 28, 3, 0, 0, 0, 0, 0, 0, 546, 485, 500, 508, 509, 514, 484, 468, 4, 0, 0, 0, 0, 0, 0, 0, 473, 527, 518, 520, 551, 495, 544, 559, 0, 0, 0, 0, 0, 0, 0, 0, 508, 525, 504, 529, 531, 474, 488, 510, 0, 0, 0, 0, 0, 0, 0, 0, 479, 504, 527, 553, 518, 497, 506, 478, 14, 11, 0, 0, 0, 0, 0, 0, 510, 508, 490, 517, 516, 508, 506, 542, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4187, 4031, 4018, 4187, 4069, 4087, 4097] avg=4096.0000 max=4187 max_rank=1 min=4018 min_rank=3 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16440 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[512, 509, 539, 523, 519, 512, 543, 519, 0, 0, 0, 0, 0, 0, 0, 0, 510, 468, 527, 529, 479, 510, 541, 505, 0, 0, 0, 0, 0, 0, 0, 0, 477, 508, 510, 509, 534, 488, 513, 479, 29, 3, 0, 0, 0, 0, 0, 0, 497, 501, 516, 489, 537, 500, 493, 491, 3, 0, 0, 0, 0, 0, 0, 0, 495, 514, 521, 504, 531, 534, 472, 503, 0, 0, 0, 0, 0, 0, 0, 0, 511, 507, 506, 504, 507, 533, 550, 550, 0, 0, 0, 0, 0, 0, 0, 0, 516, 511, 503, 503, 493, 544, 514, 475, 16, 10, 0, 0, 0, 0, 0, 0, 498, 502, 544, 498, 532, 523, 531, 491, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4176, 4069, 4050, 4027, 4074, 4168, 4085, 4119] avg=4096.0000 max=4176 max_rank=0 min=4027 min_rank=3 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16446 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[474, 507, 536, 533, 520, 513, 477, 538, 0, 0, 0, 0, 0, 0, 0, 0, 517, 482, 535, 500, 511, 484, 516, 503, 0, 0, 0, 0, 0, 0, 0, 0, 488, 478, 505, 551, 507, 525, 510, 517, 31, 4, 0, 0, 0, 0, 0, 0, 480, 548, 515, 512, 544, 488, 562, 510, 3, 0, 0, 0, 0, 0, 0, 0, 524, 512, 523, 492, 524, 504, 542, 524, 0, 0, 0, 0, 0, 0, 0, 0, 495, 530, 521, 513, 491, 520, 513, 495, 0, 0, 0, 0, 0, 0, 0, 0, 508, 527, 529, 508, 520, 517, 491, 518, 17, 10, 0, 0, 0, 0, 0, 0, 524, 505, 458, 492, 498, 481, 526, 492, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4048, 4116, 4162, 4145, 4078, 4145, 3976] avg=4096.0000 max=4162 max_rank=3 min=3976 min_rank=7 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16344 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[536, 524, 488, 503, 505, 531, 495, 473, 0, 0, 0, 0, 0, 0, 0, 0, 513, 501, 527, 506, 471, 557, 550, 529, 0, 0, 0, 0, 0, 0, 0, 0, 498, 490, 509, 513, 525, 540, 488, 520, 32, 7, 0, 0, 0, 0, 0, 0, 499, 498, 529, 514, 494, 501, 525, 532, 6, 2, 0, 0, 0, 0, 0, 0, 473, 535, 518, 479, 532, 495, 530, 471, 0, 0, 0, 0, 0, 0, 0, 0, 525, 504, 503, 524, 527, 489, 510, 511, 7, 0, 0, 0, 0, 0, 0, 0, 500, 485, 503, 494, 509, 471, 521, 503, 20, 16, 0, 0, 0, 0, 0, 0, 509, 551, 535, 483, 511, 522, 547, 524, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4055, 4154, 4122, 4100, 4033, 4100, 4022, 4182] avg=4096.0000 max=4182 max_rank=7 min=4022 min_rank=6 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16337 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[483, 485, 530, 516, 513, 477, 495, 486, 0, 0, 0, 0, 0, 0, 0, 0, 501, 529, 516, 513, 518, 493, 502, 511, 0, 0, 0, 0, 0, 0, 0, 0, 534, 514, 563, 496, 494, 499, 509, 503, 28, 3, 0, 0, 0, 0, 0, 0, 521, 490, 523, 492, 523, 540, 492, 519, 4, 0, 0, 0, 0, 0, 0, 0, 479, 545, 524, 506, 489, 538, 532, 516, 0, 0, 0, 0, 0, 0, 0, 0, 517, 528, 521, 511, 542, 515, 555, 518, 0, 0, 0, 0, 0, 0, 0, 0, 498, 516, 491, 509, 494, 496, 486, 538, 16, 11, 0, 0, 0, 0, 0, 0, 515, 499, 515, 491, 524, 509, 521, 488, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3985, 4083, 4143, 4104, 4129, 4207, 4055, 4062] avg=4096.0000 max=4207 max_rank=5 min=3985 min_rank=0 imbalance_ratio=1.027100 max_load_reduction=28561 max_load_reduction_pct=0.871613 moved_to_echo=16453 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[526, 509, 516, 546, 521, 521, 493, 507, 0, 0, 0, 0, 0, 0, 0, 0, 506, 520, 465, 515, 532, 517, 520, 525, 0, 0, 0, 0, 0, 0, 0, 0, 474, 532, 520, 457, 501, 533, 514, 448, 30, 4, 0, 0, 0, 0, 0, 0, 500, 515, 532, 474, 529, 536, 497, 504, 4, 0, 0, 0, 0, 0, 0, 0, 503, 463, 552, 523, 505, 494, 492, 508, 0, 0, 0, 0, 0, 0, 0, 0, 491, 541, 503, 507, 501, 522, 495, 496, 0, 0, 0, 0, 0, 0, 0, 0, 498, 538, 558, 519, 520, 494, 521, 490, 17, 11, 0, 0, 0, 0, 0, 0, 495, 487, 545, 522, 524, 521, 514, 555, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4139, 4100, 4013, 4091, 4040, 4056, 4166, 4163] avg=4096.0000 max=4166 max_rank=6 min=4013 min_rank=2 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16425 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[545, 529, 549, 532, 496, 527, 505, 541, 0, 0, 0, 0, 0, 0, 0, 0, 494, 542, 510, 514, 514, 526, 486, 511, 0, 0, 0, 0, 0, 0, 0, 0, 504, 535, 527, 539, 496, 484, 506, 508, 31, 4, 0, 0, 0, 0, 0, 0, 541, 519, 508, 475, 494, 491, 507, 510, 4, 0, 0, 0, 0, 0, 0, 0, 524, 543, 514, 527, 478, 502, 468, 509, 0, 0, 0, 0, 0, 0, 0, 0, 508, 482, 524, 481, 480, 489, 523, 473, 0, 0, 0, 0, 0, 0, 0, 0, 523, 543, 524, 516, 522, 510, 506, 522, 16, 11, 0, 0, 0, 0, 0, 0, 482, 486, 508, 502, 494, 537, 534, 503, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4224, 4097, 4134, 4049, 4065, 3960, 4193, 4046] avg=4096.0000 max=4224 max_rank=0 min=3960 min_rank=5 imbalance_ratio=1.031250 max_load_reduction=28544 max_load_reduction_pct=0.871094 moved_to_echo=16264 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[492, 477, 518, 476, 505, 535, 497, 499, 0, 0, 0, 0, 0, 0, 0, 0, 495, 538, 459, 469, 511, 536, 532, 490, 0, 0, 0, 0, 0, 0, 0, 0, 530, 544, 498, 522, 520, 524, 501, 488, 29, 3, 0, 0, 0, 0, 0, 0, 521, 531, 521, 524, 545, 504, 508, 559, 4, 0, 0, 0, 0, 0, 0, 0, 487, 518, 510, 516, 512, 518, 510, 524, 0, 0, 0, 0, 0, 0, 0, 0, 509, 519, 476, 529, 497, 497, 570, 533, 0, 0, 0, 0, 0, 0, 0, 0, 495, 515, 461, 515, 514, 500, 477, 512, 15, 11, 0, 0, 0, 0, 0, 0, 505, 567, 540, 518, 505, 480, 471, 537, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3999, 4030, 4159, 4217, 4095, 4130, 4015, 4123] avg=4096.0000 max=4217 max_rank=3 min=3999 min_rank=0 imbalance_ratio=1.029541 max_load_reduction=28551 max_load_reduction_pct=0.871307 moved_to_echo=16363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3968, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4352, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[488, 512, 518, 562, 540, 518, 504, 523, 539, 515, 488, 490, 492, 512, 503, 527, 546, 491, 503, 500, 526, 509, 533, 546, 535, 526, 552, 531, 494, 565, 527, 475, 529, 499, 528, 466, 527, 511, 493, 508, 539, 485, 504, 484, 478, 499, 512, 532, 476, 488, 493, 499, 527, 527, 519, 519, 479, 503, 492, 488, 513, 502, 509, 550] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[539, 536, 490, 496, 531, 509, 513, 531, 482, 474, 515, 509, 520, 549, 516, 523, 507, 470, 504, 517, 504, 479, 522, 542, 505, 543, 485, 533, 478, 565, 527, 493, 500, 503, 495, 472, 513, 502, 539, 495, 492, 471, 531, 492, 548, 550, 517, 533, 487, 557, 519, 523, 503, 525, 519, 513, 490, 500, 484, 532, 526, 541, 491, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[554, 512, 505, 517, 507, 504, 553, 493, 557, 513, 512, 500, 509, 512, 509, 500, 488, 475, 450, 507, 511, 514, 492, 564, 491, 513, 514, 507, 495, 565, 478, 507, 481, 500, 516, 475, 535, 525, 514, 544, 527, 496, 511, 513, 516, 526, 534, 496, 486, 493, 510, 540, 514, 536, 501, 531, 505, 516, 541, 526, 545, 472, 504, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[529, 518, 483, 470, 528, 505, 523, 500, 546, 514, 536, 495, 525, 523, 513, 519, 485, 507, 466, 516, 515, 545, 498, 557, 523, 492, 488, 526, 512, 513, 523, 509, 543, 497, 490, 553, 471, 518, 516, 513, 517, 483, 508, 514, 522, 506, 494, 522, 510, 501, 488, 489, 514, 547, 475, 525, 503, 547, 538, 480, 499, 533, 532, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[504, 520, 504, 512, 485, 493, 514, 542, 497, 536, 509, 511, 561, 489, 503, 493, 526, 473, 534, 506, 536, 489, 522, 487, 486, 514, 514, 545, 522, 508, 527, 538, 485, 486, 534, 483, 513, 486, 531, 519, 494, 488, 502, 476, 544, 544, 530, 522, 524, 499, 522, 497, 490, 526, 527, 509, 528, 534, 517, 542, 523, 482, 492, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[506, 482, 531, 498, 545, 519, 487, 504, 489, 500, 545, 491, 500, 533, 513, 510, 521, 521, 509, 505, 531, 491, 543, 531, 511, 497, 511, 518, 512, 493, 481, 505, 531, 507, 547, 537, 523, 539, 520, 493, 502, 485, 511, 532, 486, 518, 520, 489, 504, 536, 516, 524, 533, 548, 510, 514, 491, 557, 495, 516, 474, 464, 527, 486] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[557, 489, 506, 475, 525, 502, 482, 531, 484, 506, 493, 518, 507, 493, 513, 491, 471, 521, 542, 520, 543, 512, 492, 509, 533, 548, 512, 530, 515, 487, 471, 507, 534, 512, 533, 489, 482, 523, 534, 496, 510, 486, 503, 543, 501, 574, 538, 526, 519, 490, 492, 518, 546, 517, 511, 555, 537, 475, 473, 489, 520, 524, 498, 535] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[532, 498, 535, 473, 511, 535, 492, 525, 532, 526, 518, 545, 476, 476, 493, 527, 501, 459, 491, 509, 555, 507, 473, 509, 496, 509, 496, 513, 508, 508, 514, 507, 510, 536, 556, 513, 480, 509, 514, 527, 513, 535, 487, 542, 539, 500, 492, 496, 487, 522, 495, 539, 515, 546, 527, 508, 523, 526, 468, 548, 500, 521, 515, 530] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[482, 512, 518, 562, 540, 518, 504, 523, 0, 0, 0, 0, 0, 0, 0, 0, 539, 515, 488, 490, 492, 512, 503, 527, 6, 0, 0, 0, 0, 0, 0, 0, 546, 491, 503, 500, 526, 509, 533, 546, 17, 0, 0, 0, 0, 0, 0, 0, 535, 526, 552, 531, 494, 558, 527, 475, 0, 0, 0, 0, 0, 0, 0, 0, 529, 499, 528, 466, 527, 511, 493, 508, 1, 0, 0, 0, 0, 0, 0, 0, 539, 485, 504, 484, 478, 498, 512, 532, 0, 0, 0, 0, 0, 0, 0, 0, 476, 488, 493, 499, 527, 509, 519, 519, 0, 0, 0, 0, 0, 0, 0, 0, 479, 503, 492, 488, 513, 502, 509, 550, 7, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4159, 4072, 4171, 4198, 4062, 4032, 4030, 4044] avg=4096.0000 max=4198 max_rank=3 min=4030 min_rank=6 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16168 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[498, 520, 504, 512, 485, 493, 514, 542, 0, 0, 0, 0, 0, 0, 0, 0, 497, 536, 509, 511, 561, 489, 503, 493, 6, 0, 0, 0, 0, 0, 0, 0, 526, 473, 534, 506, 536, 489, 522, 487, 17, 0, 0, 0, 0, 0, 0, 0, 486, 514, 514, 545, 522, 501, 527, 538, 0, 0, 0, 0, 0, 0, 0, 0, 485, 486, 534, 483, 513, 486, 531, 519, 1, 0, 0, 0, 0, 0, 0, 0, 494, 488, 502, 476, 544, 543, 530, 522, 0, 0, 0, 0, 0, 0, 0, 0, 524, 499, 522, 497, 490, 508, 527, 509, 0, 0, 0, 0, 0, 0, 0, 0, 528, 534, 517, 542, 523, 482, 492, 519, 7, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4105, 4090, 4147, 4038, 4099, 4076, 4145] avg=4096.0000 max=4147 max_rank=3 min=4038 min_rank=4 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16358 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[500, 482, 531, 498, 545, 519, 487, 504, 0, 0, 0, 0, 0, 0, 0, 0, 489, 500, 545, 491, 500, 533, 513, 510, 6, 0, 0, 0, 0, 0, 0, 0, 521, 521, 509, 505, 531, 491, 543, 531, 17, 0, 0, 0, 0, 0, 0, 0, 511, 497, 511, 518, 512, 487, 481, 505, 0, 0, 0, 0, 0, 0, 0, 0, 531, 507, 547, 537, 523, 539, 520, 493, 1, 0, 0, 0, 0, 0, 0, 0, 502, 485, 511, 532, 486, 517, 520, 489, 0, 0, 0, 0, 0, 0, 0, 0, 504, 536, 516, 524, 533, 530, 510, 514, 0, 0, 0, 0, 0, 0, 0, 0, 491, 557, 495, 516, 474, 464, 527, 486, 6, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 4087, 4169, 4022, 4198, 4042, 4167, 4017] avg=4096.0000 max=4198 max_rank=4 min=4017 min_rank=7 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16424 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[533, 536, 490, 496, 531, 509, 513, 531, 0, 0, 0, 0, 0, 0, 0, 0, 482, 474, 515, 509, 520, 549, 516, 523, 6, 0, 0, 0, 0, 0, 0, 0, 507, 470, 504, 517, 504, 479, 522, 542, 17, 0, 0, 0, 0, 0, 0, 0, 505, 543, 485, 533, 478, 558, 527, 493, 0, 0, 0, 0, 0, 0, 0, 0, 500, 503, 495, 472, 513, 502, 539, 495, 1, 0, 0, 0, 0, 0, 0, 0, 492, 471, 531, 492, 548, 549, 517, 533, 0, 0, 0, 0, 0, 0, 0, 0, 487, 557, 519, 523, 503, 507, 519, 513, 0, 0, 0, 0, 0, 0, 0, 0, 490, 500, 484, 532, 526, 541, 491, 498, 7, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4139, 4094, 4062, 4122, 4020, 4133, 4128, 4070] avg=4096.0000 max=4139 max_rank=0 min=4020 min_rank=4 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16351 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[515, 518, 483, 470, 528, 505, 523, 500, 0, 0, 0, 0, 0, 0, 0, 0, 546, 514, 536, 495, 525, 523, 513, 519, 10, 0, 0, 0, 0, 0, 0, 0, 485, 507, 466, 516, 515, 545, 498, 557, 22, 0, 0, 0, 0, 0, 0, 0, 523, 492, 488, 526, 512, 502, 523, 509, 0, 0, 0, 0, 0, 0, 0, 0, 543, 497, 490, 553, 471, 518, 516, 513, 5, 3, 0, 0, 0, 0, 0, 0, 517, 483, 508, 514, 522, 501, 494, 522, 0, 0, 0, 0, 0, 0, 0, 0, 510, 501, 488, 489, 514, 520, 475, 525, 0, 0, 0, 0, 0, 0, 0, 0, 503, 547, 538, 480, 499, 533, 532, 518, 11, 5, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4042, 4181, 4111, 4075, 4109, 4061, 4022, 4167] avg=4096.0000 max=4181 max_rank=1 min=4022 min_rank=6 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16359 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[548, 512, 505, 517, 507, 504, 553, 493, 0, 0, 0, 0, 0, 0, 0, 0, 557, 513, 512, 500, 509, 512, 509, 500, 6, 0, 0, 0, 0, 0, 0, 0, 488, 475, 450, 507, 511, 514, 492, 564, 17, 0, 0, 0, 0, 0, 0, 0, 491, 513, 514, 507, 495, 558, 478, 507, 0, 0, 0, 0, 0, 0, 0, 0, 481, 500, 516, 475, 535, 525, 514, 544, 1, 0, 0, 0, 0, 0, 0, 0, 527, 496, 511, 513, 516, 525, 534, 496, 0, 0, 0, 0, 0, 0, 0, 0, 486, 493, 510, 540, 514, 518, 501, 531, 0, 0, 0, 0, 0, 0, 0, 0, 505, 516, 541, 526, 545, 472, 504, 511, 7, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4139, 4118, 4018, 4063, 4091, 4118, 4093, 4128] avg=4096.0000 max=4139 max_rank=0 min=4018 min_rank=2 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16430 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[550, 489, 506, 475, 525, 502, 482, 531, 0, 0, 0, 0, 0, 0, 0, 0, 484, 506, 493, 518, 507, 493, 513, 491, 7, 0, 0, 0, 0, 0, 0, 0, 471, 521, 542, 520, 543, 512, 492, 509, 16, 0, 0, 0, 0, 0, 0, 0, 533, 548, 512, 530, 515, 481, 471, 507, 0, 0, 0, 0, 0, 0, 0, 0, 534, 512, 533, 489, 482, 523, 534, 496, 1, 0, 0, 0, 0, 0, 0, 0, 510, 486, 503, 543, 501, 573, 538, 526, 0, 0, 0, 0, 0, 0, 0, 0, 519, 490, 492, 518, 546, 500, 511, 555, 0, 0, 0, 0, 0, 0, 0, 0, 537, 475, 473, 489, 520, 524, 498, 535, 6, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4012, 4126, 4097, 4104, 4180, 4131, 4058] avg=4096.0000 max=4180 max_rank=5 min=4012 min_rank=1 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16473 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[526, 498, 535, 473, 511, 535, 492, 525, 0, 0, 0, 0, 0, 0, 0, 0, 532, 526, 518, 545, 476, 476, 493, 527, 6, 0, 0, 0, 0, 0, 0, 0, 501, 459, 491, 509, 555, 507, 473, 509, 17, 0, 0, 0, 0, 0, 0, 0, 496, 509, 496, 513, 508, 501, 514, 507, 0, 0, 0, 0, 0, 0, 0, 0, 510, 536, 556, 513, 480, 509, 514, 527, 1, 0, 0, 0, 0, 0, 0, 0, 513, 535, 487, 542, 539, 499, 492, 496, 0, 0, 0, 0, 0, 0, 0, 0, 487, 522, 495, 539, 515, 528, 527, 508, 0, 0, 0, 0, 0, 0, 0, 0, 523, 526, 468, 548, 500, 521, 515, 530, 7, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4099, 4021, 4044, 4146, 4103, 4121, 4139] avg=4096.0000 max=4146 max_rank=4 min=4021 min_rank=2 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16509 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 3968, 4096, 4096, 4224, 4096, 4096, 4352, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 3968, 4096, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[514, 535, 522, 515, 497, 506, 508, 500, 503, 497, 483, 500, 518, 495, 551, 477, 487, 501, 507, 543, 504, 497, 512, 475, 459, 519, 507, 493, 490, 528, 551, 484, 529, 509, 510, 522, 487, 490, 536, 515, 544, 535, 509, 526, 515, 508, 561, 506, 539, 511, 500, 507, 532, 533, 515, 538, 491, 550, 514, 503, 524, 499, 541, 491] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[509, 499, 509, 496, 520, 507, 519, 489, 483, 503, 502, 477, 521, 550, 523, 543, 535, 503, 535, 465, 490, 528, 483, 505, 518, 521, 514, 513, 536, 487, 518, 515, 523, 526, 532, 482, 506, 492, 499, 524, 516, 537, 527, 551, 513, 506, 514, 522, 464, 550, 491, 510, 470, 533, 497, 504, 524, 513, 533, 497, 549, 506, 529, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[518, 501, 492, 530, 546, 489, 512, 511, 517, 548, 508, 525, 526, 518, 535, 534, 488, 517, 558, 478, 521, 506, 497, 524, 492, 477, 491, 516, 561, 470, 504, 505, 495, 545, 510, 499, 512, 484, 476, 486, 514, 534, 506, 509, 534, 502, 520, 532, 526, 513, 510, 469, 507, 493, 515, 521, 492, 536, 491, 531, 518, 521, 519, 533] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[536, 522, 527, 495, 522, 543, 516, 529, 484, 531, 499, 508, 529, 519, 522, 498, 503, 505, 530, 527, 555, 527, 482, 507, 535, 486, 530, 525, 489, 462, 480, 523, 535, 530, 476, 497, 511, 474, 534, 508, 531, 474, 488, 521, 511, 469, 513, 528, 548, 486, 484, 521, 501, 518, 519, 494, 509, 527, 541, 528, 497, 517, 513, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[520, 524, 522, 512, 458, 556, 522, 517, 502, 498, 518, 503, 537, 508, 500, 513, 512, 518, 512, 537, 511, 491, 551, 512, 538, 526, 536, 471, 520, 485, 522, 484, 503, 514, 525, 466, 519, 515, 494, 487, 536, 469, 531, 518, 493, 511, 496, 561, 500, 545, 493, 534, 508, 533, 507, 532, 473, 505, 509, 495, 534, 496, 515, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[476, 504, 500, 500, 512, 518, 490, 501, 509, 512, 531, 486, 553, 476, 509, 497, 519, 512, 535, 546, 505, 518, 500, 512, 513, 518, 528, 573, 545, 463, 508, 547, 531, 481, 528, 506, 496, 508, 514, 545, 498, 503, 514, 515, 485, 494, 514, 520, 525, 513, 487, 490, 533, 466, 533, 496, 555, 500, 497, 550, 510, 510, 500, 535] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[520, 504, 485, 528, 485, 515, 533, 509, 481, 523, 478, 518, 535, 482, 521, 478, 480, 547, 509, 505, 500, 508, 540, 510, 489, 471, 511, 549, 513, 517, 513, 549, 503, 521, 499, 483, 509, 516, 537, 491, 508, 512, 532, 478, 537, 525, 511, 520, 531, 523, 521, 535, 532, 501, 540, 497, 518, 490, 512, 527, 535, 494, 523, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[476, 541, 490, 530, 523, 529, 506, 554, 552, 503, 524, 496, 497, 487, 497, 530, 496, 515, 536, 496, 556, 527, 467, 518, 500, 483, 519, 519, 514, 517, 511, 501, 505, 524, 540, 500, 504, 551, 528, 526, 523, 485, 529, 544, 532, 492, 466, 497, 479, 536, 513, 499, 500, 528, 542, 549, 492, 494, 479, 507, 459, 510, 492, 533] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[514, 535, 522, 515, 497, 503, 508, 500, 0, 0, 0, 0, 0, 0, 0, 0, 503, 497, 483, 500, 518, 495, 551, 477, 3, 3, 3, 0, 0, 0, 0, 0, 487, 501, 504, 543, 504, 497, 512, 475, 0, 0, 0, 0, 0, 0, 0, 0, 459, 519, 507, 493, 490, 528, 551, 484, 8, 3, 1, 0, 0, 0, 0, 0, 529, 509, 510, 522, 487, 490, 536, 515, 17, 0, 0, 0, 0, 0, 0, 0, 544, 535, 509, 526, 515, 508, 561, 488, 0, 0, 0, 0, 0, 0, 0, 0, 539, 503, 500, 507, 532, 533, 515, 538, 0, 0, 0, 0, 0, 0, 0, 0, 491, 550, 514, 503, 524, 499, 541, 485, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4033, 4023, 4043, 4115, 4186, 4167, 4107] avg=4096.0000 max=4186 max_rank=5 min=4023 min_rank=2 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16575 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[509, 499, 509, 496, 520, 504, 519, 489, 0, 0, 0, 0, 0, 0, 0, 0, 483, 503, 502, 477, 521, 550, 523, 543, 4, 3, 3, 0, 0, 0, 0, 0, 535, 503, 532, 465, 490, 528, 483, 505, 0, 0, 0, 0, 0, 0, 0, 0, 518, 521, 514, 513, 536, 487, 518, 515, 9, 3, 1, 0, 0, 0, 0, 0, 523, 526, 532, 482, 506, 492, 499, 524, 18, 0, 0, 0, 0, 0, 0, 0, 516, 537, 527, 551, 513, 506, 514, 503, 0, 0, 0, 0, 0, 0, 0, 0, 464, 541, 491, 510, 470, 533, 497, 504, 0, 0, 0, 0, 0, 0, 0, 0, 524, 513, 533, 497, 549, 506, 529, 505, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4045, 4112, 4041, 4135, 4102, 4167, 4010, 4156] avg=4096.0000 max=4167 max_rank=5 min=4010 min_rank=6 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16435 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[518, 501, 492, 530, 546, 486, 512, 511, 0, 0, 0, 0, 0, 0, 0, 0, 517, 548, 508, 525, 526, 518, 535, 534, 4, 3, 3, 0, 0, 0, 0, 0, 488, 517, 555, 478, 521, 506, 497, 524, 0, 0, 0, 0, 0, 0, 0, 0, 492, 477, 491, 516, 561, 470, 504, 505, 8, 4, 1, 0, 0, 0, 0, 0, 495, 545, 510, 499, 512, 484, 476, 486, 18, 0, 0, 0, 0, 0, 0, 0, 514, 534, 506, 509, 534, 502, 520, 513, 0, 0, 0, 0, 0, 0, 0, 0, 526, 505, 510, 469, 507, 493, 515, 521, 0, 0, 0, 0, 0, 0, 0, 0, 492, 536, 491, 531, 518, 521, 519, 525, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4221, 4086, 4029, 4025, 4132, 4046, 4133] avg=4096.0000 max=4221 max_rank=1 min=4025 min_rank=4 imbalance_ratio=1.030518 max_load_reduction=28547 max_load_reduction_pct=0.871185 moved_to_echo=16336 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[536, 522, 527, 495, 522, 540, 516, 529, 0, 0, 0, 0, 0, 0, 0, 0, 484, 531, 499, 508, 529, 519, 522, 498, 4, 3, 3, 0, 0, 0, 0, 0, 503, 505, 527, 527, 555, 527, 482, 507, 0, 0, 0, 0, 0, 0, 0, 0, 535, 486, 530, 525, 489, 462, 480, 523, 8, 4, 1, 0, 0, 0, 0, 0, 535, 530, 476, 497, 511, 474, 534, 508, 18, 0, 0, 0, 0, 0, 0, 0, 531, 474, 488, 521, 511, 469, 513, 509, 0, 0, 0, 0, 0, 0, 0, 0, 548, 478, 484, 521, 501, 518, 519, 494, 0, 0, 0, 0, 0, 0, 0, 0, 509, 527, 541, 528, 497, 517, 513, 511, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4187, 4100, 4133, 4043, 4083, 4016, 4063, 4143] avg=4096.0000 max=4187 max_rank=0 min=4016 min_rank=5 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16305 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[520, 524, 522, 512, 458, 551, 522, 517, 0, 0, 0, 0, 0, 0, 0, 0, 502, 498, 518, 503, 537, 508, 500, 513, 7, 7, 5, 0, 0, 0, 0, 0, 512, 518, 505, 537, 511, 491, 551, 512, 0, 0, 0, 0, 0, 0, 0, 0, 538, 526, 536, 471, 520, 485, 522, 484, 13, 7, 5, 0, 0, 0, 0, 0, 503, 514, 525, 466, 519, 515, 494, 487, 21, 0, 0, 0, 0, 0, 0, 0, 536, 469, 531, 518, 493, 511, 496, 535, 0, 0, 0, 0, 0, 0, 0, 0, 500, 532, 493, 534, 508, 533, 507, 532, 0, 0, 0, 0, 0, 0, 0, 0, 473, 505, 509, 495, 534, 496, 515, 501, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4098, 4137, 4107, 4044, 4089, 4139, 4028] avg=4096.0000 max=4139 max_rank=6 min=4028 min_rank=7 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16300 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[520, 504, 485, 528, 485, 512, 533, 509, 0, 0, 0, 0, 0, 0, 0, 0, 481, 523, 478, 518, 535, 482, 521, 478, 3, 3, 3, 0, 0, 0, 0, 0, 480, 547, 506, 505, 500, 508, 540, 510, 0, 0, 0, 0, 0, 0, 0, 0, 489, 471, 511, 549, 513, 517, 513, 549, 9, 3, 1, 0, 0, 0, 0, 0, 503, 521, 499, 483, 509, 516, 537, 491, 18, 0, 0, 0, 0, 0, 0, 0, 508, 512, 532, 478, 537, 525, 511, 501, 0, 0, 0, 0, 0, 0, 0, 0, 531, 514, 521, 535, 532, 501, 540, 497, 0, 0, 0, 0, 0, 0, 0, 0, 518, 490, 512, 527, 535, 494, 523, 495, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4025, 4096, 4125, 4077, 4104, 4171, 4094] avg=4096.0000 max=4171 max_rank=6 min=4025 min_rank=1 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16446 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[476, 504, 500, 500, 512, 515, 490, 501, 0, 0, 0, 0, 0, 0, 0, 0, 509, 512, 531, 486, 553, 476, 509, 497, 4, 3, 3, 0, 0, 0, 0, 0, 519, 512, 532, 546, 505, 518, 500, 512, 0, 0, 0, 0, 0, 0, 0, 0, 513, 518, 528, 573, 545, 463, 508, 547, 8, 4, 1, 0, 0, 0, 0, 0, 531, 481, 528, 506, 496, 508, 514, 545, 18, 0, 0, 0, 0, 0, 0, 0, 498, 503, 514, 515, 485, 494, 514, 501, 0, 0, 0, 0, 0, 0, 0, 0, 525, 505, 487, 490, 533, 466, 533, 496, 0, 0, 0, 0, 0, 0, 0, 0, 555, 500, 497, 550, 510, 510, 500, 527, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3998, 4083, 4144, 4208, 4127, 4024, 4035, 4149] avg=4096.0000 max=4208 max_rank=3 min=3998 min_rank=0 imbalance_ratio=1.027344 max_load_reduction=28560 max_load_reduction_pct=0.871582 moved_to_echo=16335 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[476, 541, 490, 530, 523, 526, 506, 554, 0, 0, 0, 0, 0, 0, 0, 0, 552, 503, 524, 496, 497, 487, 497, 530, 4, 3, 3, 0, 0, 0, 0, 0, 496, 515, 533, 496, 556, 527, 467, 518, 0, 0, 0, 0, 0, 0, 0, 0, 500, 483, 519, 519, 514, 517, 511, 501, 9, 4, 1, 0, 0, 0, 0, 0, 505, 524, 540, 500, 504, 551, 528, 526, 17, 0, 0, 0, 0, 0, 0, 0, 523, 485, 529, 544, 532, 492, 466, 479, 0, 0, 0, 0, 0, 0, 0, 0, 479, 527, 513, 499, 500, 528, 542, 549, 0, 0, 0, 0, 0, 0, 0, 0, 492, 494, 479, 507, 459, 510, 492, 525, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4146, 4096, 4108, 4078, 4195, 4050, 4137, 3958] avg=4096.0000 max=4195 max_rank=4 min=3958 min_rank=7 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16340 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 3968, 4096, 4096, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 3968, 4224, 4224, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[526, 513, 516, 526, 503, 485, 501, 501, 531, 512, 503, 577, 502, 527, 502, 514, 524, 514, 523, 491, 521, 502, 527, 493, 470, 497, 510, 543, 529, 528, 519, 548, 499, 517, 544, 514, 525, 467, 494, 505, 526, 491, 478, 534, 471, 513, 473, 502, 516, 512, 502, 494, 525, 514, 517, 525, 555, 505, 505, 507, 516, 533, 510, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[514, 529, 516, 513, 538, 498, 514, 532, 518, 539, 505, 511, 527, 485, 515, 522, 541, 456, 489, 474, 499, 545, 487, 529, 489, 533, 523, 498, 517, 530, 512, 486, 531, 484, 517, 522, 528, 489, 526, 490, 545, 552, 521, 492, 516, 483, 519, 504, 503, 534, 475, 545, 521, 482, 514, 503, 511, 511, 512, 487, 494, 535, 528, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[468, 476, 509, 504, 490, 480, 555, 493, 502, 501, 522, 518, 478, 534, 499, 523, 502, 520, 480, 508, 501, 489, 497, 549, 536, 513, 494, 479, 513, 511, 532, 504, 512, 528, 545, 449, 533, 502, 483, 530, 501, 536, 519, 481, 507, 505, 537, 486, 543, 500, 546, 540, 530, 554, 507, 528, 529, 537, 537, 505, 537, 511, 536, 494] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[491, 518, 540, 526, 530, 524, 521, 500, 507, 500, 528, 536, 494, 558, 528, 489, 531, 556, 535, 472, 530, 492, 529, 538, 504, 539, 524, 515, 530, 538, 529, 488, 503, 444, 493, 527, 525, 489, 547, 505, 535, 470, 503, 485, 540, 476, 466, 515, 474, 535, 507, 472, 547, 481, 496, 471, 505, 537, 477, 490, 517, 525, 538, 533] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[539, 507, 518, 511, 490, 491, 506, 493, 511, 507, 490, 490, 543, 504, 514, 545, 494, 518, 493, 503, 449, 515, 514, 527, 554, 554, 478, 486, 509, 486, 486, 518, 491, 482, 532, 544, 552, 531, 509, 549, 511, 575, 525, 511, 493, 534, 521, 513, 503, 506, 475, 468, 538, 503, 537, 488, 522, 523, 520, 502, 562, 514, 491, 500] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[497, 506, 504, 504, 490, 510, 512, 475, 476, 552, 507, 572, 525, 529, 511, 510, 526, 521, 532, 510, 500, 478, 526, 502, 516, 530, 507, 477, 529, 515, 509, 477, 546, 520, 476, 505, 525, 497, 523, 524, 511, 522, 547, 508, 513, 506, 522, 511, 532, 507, 526, 481, 511, 579, 498, 463, 531, 474, 475, 487, 530, 544, 537, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[531, 484, 506, 514, 499, 514, 481, 517, 512, 514, 512, 522, 500, 492, 538, 511, 500, 522, 469, 468, 509, 504, 536, 526, 525, 532, 520, 513, 498, 540, 546, 483, 484, 509, 518, 468, 540, 510, 538, 480, 513, 500, 490, 511, 507, 498, 511, 548, 511, 557, 474, 534, 532, 498, 549, 512, 526, 502, 486, 523, 513, 526, 535, 527] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[509, 479, 500, 513, 487, 526, 558, 507, 480, 496, 469, 525, 520, 514, 501, 505, 503, 545, 513, 491, 526, 517, 527, 498, 491, 522, 528, 509, 514, 540, 540, 520, 519, 495, 516, 522, 490, 518, 575, 562, 471, 531, 506, 500, 510, 521, 523, 495, 505, 539, 500, 506, 511, 504, 514, 495, 533, 509, 492, 499, 495, 514, 491, 534] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[514, 529, 516, 513, 538, 498, 514, 532, 21, 7, 0, 0, 0, 0, 0, 0, 518, 539, 505, 496, 527, 485, 515, 522, 0, 0, 0, 0, 0, 0, 0, 0, 541, 456, 489, 474, 499, 545, 487, 529, 13, 6, 0, 0, 0, 0, 0, 0, 489, 513, 523, 498, 517, 530, 512, 486, 0, 0, 0, 0, 0, 0, 0, 0, 531, 484, 517, 522, 522, 489, 526, 490, 0, 0, 0, 0, 0, 0, 0, 0, 545, 552, 521, 492, 516, 483, 519, 504, 9, 6, 0, 0, 0, 0, 0, 0, 503, 534, 475, 545, 521, 482, 514, 503, 0, 0, 0, 0, 0, 0, 0, 0, 490, 511, 512, 487, 494, 535, 528, 510, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4182, 4107, 4039, 4068, 4081, 4147, 4077, 4067] avg=4096.0000 max=4182 max_rank=0 min=4039 min_rank=2 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[468, 476, 509, 504, 490, 480, 555, 493, 22, 7, 0, 0, 0, 0, 0, 0, 502, 501, 522, 502, 478, 534, 499, 523, 0, 0, 0, 0, 0, 0, 0, 0, 502, 520, 480, 508, 501, 489, 497, 549, 12, 7, 0, 0, 0, 0, 0, 0, 536, 494, 494, 479, 513, 511, 532, 504, 0, 0, 0, 0, 0, 0, 0, 0, 512, 528, 545, 449, 527, 502, 483, 530, 0, 0, 0, 0, 0, 0, 0, 0, 501, 536, 519, 481, 507, 505, 537, 486, 9, 6, 0, 0, 0, 0, 0, 0, 543, 500, 546, 540, 530, 554, 507, 528, 0, 0, 0, 0, 0, 0, 0, 0, 507, 537, 537, 505, 537, 511, 536, 494, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4004, 4061, 4065, 4063, 4076, 4087, 4248, 4164] avg=4096.0000 max=4248 max_rank=6 min=4004 min_rank=0 imbalance_ratio=1.037109 max_load_reduction=28520 max_load_reduction_pct=0.870361 moved_to_echo=16575 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[539, 507, 518, 511, 490, 491, 506, 493, 22, 8, 0, 0, 0, 0, 0, 0, 511, 507, 490, 476, 543, 504, 514, 545, 0, 0, 0, 0, 0, 0, 0, 0, 494, 518, 493, 503, 449, 515, 514, 527, 13, 6, 0, 0, 0, 0, 0, 0, 554, 533, 478, 486, 509, 486, 486, 518, 0, 0, 0, 0, 0, 0, 0, 0, 491, 482, 532, 544, 546, 531, 509, 549, 0, 0, 0, 0, 0, 0, 0, 0, 511, 575, 525, 511, 493, 534, 521, 513, 8, 6, 0, 0, 0, 0, 0, 0, 503, 506, 475, 468, 538, 503, 537, 488, 0, 0, 0, 0, 0, 0, 0, 0, 500, 523, 520, 502, 562, 514, 491, 500, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4090, 4032, 4050, 4184, 4197, 4018, 4112] avg=4096.0000 max=4197 max_rank=5 min=4018 min_rank=6 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16511 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[526, 513, 516, 526, 503, 485, 501, 501, 23, 7, 0, 0, 0, 0, 0, 0, 531, 512, 503, 560, 502, 527, 502, 514, 0, 0, 0, 0, 0, 0, 0, 0, 524, 514, 523, 491, 521, 502, 527, 493, 12, 7, 0, 0, 0, 0, 0, 0, 470, 478, 510, 543, 529, 528, 519, 548, 0, 0, 0, 0, 0, 0, 0, 0, 499, 517, 544, 514, 519, 467, 494, 505, 0, 0, 0, 0, 0, 0, 0, 0, 526, 491, 478, 534, 471, 513, 473, 502, 10, 6, 0, 0, 0, 0, 0, 0, 516, 512, 502, 494, 525, 514, 517, 525, 0, 0, 0, 0, 0, 0, 0, 0, 532, 505, 505, 507, 516, 533, 510, 501, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4151, 4114, 4125, 4059, 4004, 4105, 4109] avg=4096.0000 max=4151 max_rank=1 min=4004 min_rank=5 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16277 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[491, 518, 540, 526, 530, 524, 521, 500, 25, 11, 0, 0, 0, 0, 0, 0, 507, 500, 528, 511, 494, 558, 528, 489, 0, 0, 0, 0, 0, 0, 0, 0, 531, 556, 535, 472, 530, 492, 529, 538, 16, 11, 0, 0, 0, 0, 0, 0, 504, 512, 524, 515, 530, 538, 529, 488, 0, 0, 0, 0, 0, 0, 0, 0, 503, 444, 493, 527, 517, 489, 547, 505, 0, 0, 0, 0, 0, 0, 0, 0, 535, 470, 503, 485, 540, 476, 466, 515, 14, 8, 1, 0, 0, 0, 0, 0, 474, 535, 507, 472, 546, 481, 496, 471, 0, 0, 0, 0, 0, 0, 0, 0, 480, 537, 477, 490, 517, 525, 538, 533, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4186, 4115, 4210, 4140, 4025, 4013, 3982, 4097] avg=4096.0000 max=4210 max_rank=2 min=3982 min_rank=6 imbalance_ratio=1.027832 max_load_reduction=28558 max_load_reduction_pct=0.871521 moved_to_echo=16117 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[497, 506, 504, 504, 490, 510, 512, 475, 22, 7, 0, 0, 0, 0, 0, 0, 476, 552, 507, 555, 525, 529, 511, 510, 0, 0, 0, 0, 0, 0, 0, 0, 526, 521, 532, 510, 500, 478, 526, 502, 13, 7, 0, 0, 0, 0, 0, 0, 516, 510, 507, 477, 529, 515, 509, 477, 0, 0, 0, 0, 0, 0, 0, 0, 546, 520, 476, 505, 519, 497, 523, 524, 0, 0, 0, 0, 0, 0, 0, 0, 511, 522, 547, 508, 513, 506, 522, 511, 10, 6, 0, 0, 0, 0, 0, 0, 532, 507, 526, 481, 511, 579, 498, 463, 0, 0, 0, 0, 0, 0, 0, 0, 509, 474, 475, 487, 530, 544, 537, 502, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4027, 4165, 4115, 4040, 4110, 4156, 4097, 4058] avg=4096.0000 max=4165 max_rank=1 min=4027 min_rank=0 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16421 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[531, 484, 506, 514, 499, 514, 481, 517, 22, 7, 0, 0, 0, 0, 0, 0, 512, 514, 512, 506, 500, 492, 538, 511, 0, 0, 0, 0, 0, 0, 0, 0, 500, 522, 469, 468, 509, 504, 536, 526, 13, 7, 0, 0, 0, 0, 0, 0, 525, 512, 520, 513, 498, 540, 546, 483, 0, 0, 0, 0, 0, 0, 0, 0, 484, 509, 518, 468, 534, 510, 538, 480, 0, 0, 0, 0, 0, 0, 0, 0, 513, 500, 490, 511, 507, 498, 511, 548, 9, 6, 0, 0, 0, 0, 0, 0, 511, 557, 474, 534, 532, 498, 549, 512, 0, 0, 0, 0, 0, 0, 0, 0, 504, 502, 486, 523, 513, 526, 535, 527, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4085, 4054, 4137, 4041, 4093, 4167, 4116] avg=4096.0000 max=4167 max_rank=6 min=4041 min_rank=4 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16417 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[509, 479, 500, 513, 487, 526, 558, 507, 22, 7, 0, 0, 0, 0, 0, 0, 480, 496, 469, 509, 520, 514, 501, 505, 0, 0, 0, 0, 0, 0, 0, 0, 503, 545, 513, 491, 526, 517, 527, 498, 12, 7, 0, 0, 0, 0, 0, 0, 491, 503, 528, 509, 514, 540, 540, 520, 0, 0, 0, 0, 0, 0, 0, 0, 519, 495, 516, 522, 485, 518, 575, 562, 0, 0, 0, 0, 0, 0, 0, 0, 471, 531, 506, 500, 510, 521, 523, 495, 9, 5, 0, 0, 0, 0, 0, 0, 505, 539, 500, 506, 511, 504, 514, 495, 0, 0, 0, 0, 0, 0, 0, 0, 511, 509, 492, 499, 495, 514, 491, 534, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 3994, 4139, 4145, 4192, 4071, 4074, 4045] avg=4096.0000 max=4192 max_rank=4 min=3994 min_rank=1 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 3968, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[495, 528, 498, 527, 510, 525, 529, 508, 476, 487, 525, 500, 551, 528, 511, 523, 521, 509, 512, 542, 492, 511, 561, 511, 509, 525, 532, 510, 517, 514, 535, 494, 495, 482, 517, 509, 541, 498, 524, 523, 498, 475, 524, 503, 478, 488, 519, 506, 546, 533, 474, 512, 550, 495, 481, 531, 478, 523, 497, 512, 516, 488, 540, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[454, 495, 550, 509, 531, 530, 488, 523, 502, 480, 511, 498, 508, 548, 515, 505, 490, 511, 514, 527, 524, 498, 544, 558, 500, 515, 523, 496, 527, 515, 503, 497, 503, 497, 517, 520, 505, 494, 489, 494, 505, 492, 514, 510, 518, 499, 541, 484, 490, 534, 510, 531, 521, 558, 519, 538, 528, 511, 494, 526, 530, 502, 490, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[494, 537, 510, 491, 465, 490, 487, 510, 531, 503, 550, 530, 527, 514, 475, 526, 507, 517, 563, 521, 559, 438, 528, 513, 523, 518, 486, 494, 494, 530, 517, 522, 515, 511, 507, 485, 517, 540, 499, 504, 535, 497, 502, 505, 486, 489, 479, 512, 515, 498, 560, 535, 526, 571, 527, 528, 498, 517, 490, 503, 519, 504, 521, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[498, 539, 504, 492, 516, 479, 530, 526, 508, 500, 512, 499, 512, 486, 506, 494, 546, 511, 518, 542, 516, 525, 504, 480, 510, 513, 515, 502, 507, 519, 519, 500, 513, 526, 494, 461, 482, 514, 525, 456, 531, 521, 512, 558, 564, 477, 524, 532, 498, 527, 517, 534, 533, 530, 462, 498, 481, 511, 536, 523, 532, 547, 505, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[527, 515, 512, 518, 474, 506, 518, 502, 562, 516, 483, 544, 518, 514, 493, 515, 511, 476, 505, 502, 534, 499, 522, 549, 530, 530, 524, 513, 498, 498, 559, 564, 480, 526, 515, 522, 525, 527, 521, 484, 465, 484, 505, 506, 510, 507, 500, 517, 503, 507, 524, 482, 539, 542, 498, 470, 511, 514, 523, 508, 522, 488, 514, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[512, 519, 511, 501, 546, 490, 537, 494, 514, 520, 536, 552, 503, 517, 508, 474, 511, 478, 504, 501, 524, 528, 500, 512, 554, 528, 506, 494, 473, 470, 518, 480, 514, 531, 513, 511, 525, 488, 527, 509, 516, 552, 483, 522, 512, 519, 520, 517, 511, 500, 512, 516, 510, 518, 513, 490, 507, 462, 559, 518, 516, 522, 519, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[524, 485, 533, 466, 540, 499, 493, 519, 504, 515, 520, 512, 513, 518, 491, 478, 502, 519, 516, 510, 535, 541, 519, 528, 507, 487, 496, 517, 559, 540, 530, 497, 499, 525, 496, 517, 489, 515, 502, 509, 544, 486, 521, 528, 520, 504, 447, 537, 527, 538, 504, 530, 526, 545, 479, 528, 513, 496, 511, 512, 516, 474, 511, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[485, 492, 555, 509, 518, 505, 465, 520, 519, 557, 537, 492, 493, 510, 481, 550, 533, 527, 523, 503, 510, 522, 519, 496, 498, 483, 537, 534, 497, 514, 501, 534, 509, 477, 507, 516, 509, 511, 522, 519, 546, 475, 524, 512, 495, 519, 520, 501, 480, 517, 516, 489, 508, 563, 498, 493, 480, 547, 481, 517, 517, 522, 542, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[454, 495, 550, 509, 531, 530, 488, 523, 22, 4, 0, 0, 0, 0, 0, 0, 502, 480, 508, 498, 508, 548, 515, 505, 0, 0, 0, 0, 0, 0, 0, 0, 490, 511, 514, 527, 524, 498, 505, 558, 0, 0, 0, 0, 0, 0, 0, 0, 500, 515, 523, 496, 527, 515, 494, 497, 0, 0, 0, 0, 0, 0, 0, 0, 503, 497, 517, 520, 505, 494, 489, 494, 35, 0, 0, 0, 0, 0, 0, 0, 505, 492, 514, 510, 518, 499, 541, 484, 14, 7, 0, 0, 0, 0, 0, 0, 490, 534, 510, 531, 521, 522, 519, 538, 0, 0, 0, 0, 0, 0, 0, 0, 528, 511, 494, 526, 530, 502, 490, 515, 3, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4064, 4127, 4067, 4054, 4084, 4165, 4101] avg=4096.0000 max=4165 max_rank=6 min=4054 min_rank=4 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16404 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[495, 528, 498, 527, 510, 525, 529, 508, 20, 4, 0, 0, 0, 0, 0, 0, 476, 487, 521, 500, 551, 528, 511, 523, 0, 0, 0, 0, 0, 0, 0, 0, 521, 509, 512, 542, 492, 511, 521, 511, 0, 0, 0, 0, 0, 0, 0, 0, 509, 525, 532, 510, 517, 514, 525, 494, 0, 0, 0, 0, 0, 0, 0, 0, 495, 482, 517, 509, 541, 498, 524, 523, 36, 0, 0, 0, 0, 0, 0, 0, 498, 475, 524, 503, 478, 488, 519, 506, 12, 8, 0, 0, 0, 0, 0, 0, 546, 533, 474, 512, 550, 463, 481, 531, 0, 0, 0, 0, 0, 0, 0, 0, 478, 523, 497, 512, 516, 488, 540, 496, 4, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4144, 4097, 4119, 4126, 4125, 4011, 4090, 4056] avg=4096.0000 max=4144 max_rank=0 min=4011 min_rank=5 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16282 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[498, 539, 504, 492, 516, 479, 530, 526, 21, 3, 0, 0, 0, 0, 0, 0, 508, 500, 509, 499, 512, 486, 506, 494, 0, 0, 0, 0, 0, 0, 0, 0, 546, 511, 518, 542, 516, 525, 469, 480, 0, 0, 0, 0, 0, 0, 0, 0, 510, 513, 515, 502, 507, 519, 510, 500, 0, 0, 0, 0, 0, 0, 0, 0, 513, 526, 494, 461, 482, 514, 525, 456, 32, 0, 0, 0, 0, 0, 0, 0, 531, 521, 512, 558, 564, 477, 524, 532, 13, 7, 0, 0, 0, 0, 0, 0, 498, 527, 517, 534, 533, 496, 462, 498, 0, 0, 0, 0, 0, 0, 0, 0, 481, 511, 536, 523, 532, 547, 505, 516, 3, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4014, 4107, 4076, 4003, 4239, 4065, 4156] avg=4096.0000 max=4239 max_rank=5 min=4003 min_rank=4 imbalance_ratio=1.034912 max_load_reduction=28529 max_load_reduction_pct=0.870636 moved_to_echo=16463 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[512, 519, 511, 501, 546, 490, 537, 494, 21, 3, 0, 0, 0, 0, 0, 0, 514, 520, 532, 552, 503, 517, 508, 474, 0, 0, 0, 0, 0, 0, 0, 0, 511, 478, 504, 501, 524, 528, 465, 512, 0, 0, 0, 0, 0, 0, 0, 0, 554, 528, 506, 494, 473, 470, 509, 480, 0, 0, 0, 0, 0, 0, 0, 0, 514, 531, 513, 511, 525, 488, 527, 509, 32, 0, 0, 0, 0, 0, 0, 0, 516, 552, 483, 522, 512, 519, 520, 517, 13, 7, 0, 0, 0, 0, 0, 0, 511, 500, 512, 516, 510, 484, 513, 490, 0, 0, 0, 0, 0, 0, 0, 0, 507, 462, 559, 518, 516, 522, 519, 521, 4, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4134, 4120, 4023, 4014, 4150, 4161, 4036, 4130] avg=4096.0000 max=4161 max_rank=5 min=4014 min_rank=3 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16477 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[494, 537, 510, 491, 465, 490, 487, 510, 23, 4, 0, 0, 0, 0, 0, 0, 531, 503, 546, 530, 527, 514, 475, 526, 0, 0, 0, 0, 0, 0, 0, 0, 507, 517, 563, 521, 559, 438, 490, 513, 0, 0, 0, 0, 0, 0, 0, 0, 523, 518, 486, 494, 494, 530, 508, 522, 0, 0, 0, 0, 0, 0, 0, 0, 515, 511, 507, 485, 517, 540, 499, 504, 34, 0, 0, 0, 0, 0, 0, 0, 535, 497, 502, 505, 486, 489, 479, 512, 14, 7, 0, 0, 0, 0, 0, 0, 515, 498, 560, 535, 526, 534, 527, 528, 0, 0, 0, 0, 0, 0, 0, 0, 498, 517, 490, 503, 519, 504, 521, 503, 4, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4011, 4152, 4108, 4075, 4112, 4026, 4223, 4061] avg=4096.0000 max=4223 max_rank=6 min=4011 min_rank=0 imbalance_ratio=1.031006 max_load_reduction=28545 max_load_reduction_pct=0.871124 moved_to_echo=16422 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[527, 515, 512, 518, 474, 506, 518, 502, 25, 7, 0, 0, 0, 0, 0, 0, 562, 516, 476, 544, 518, 514, 493, 515, 0, 0, 0, 0, 0, 0, 0, 0, 511, 476, 505, 502, 534, 499, 479, 549, 0, 0, 0, 0, 0, 0, 0, 0, 530, 530, 524, 513, 498, 498, 540, 564, 0, 0, 0, 0, 0, 0, 0, 0, 480, 526, 515, 522, 525, 527, 521, 484, 36, 0, 0, 0, 0, 0, 0, 0, 465, 484, 505, 506, 510, 507, 500, 517, 19, 13, 0, 0, 0, 0, 0, 0, 503, 507, 524, 482, 539, 498, 498, 470, 0, 0, 0, 0, 0, 0, 0, 0, 511, 514, 523, 508, 522, 488, 514, 498, 7, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4138, 4055, 4197, 4136, 4026, 4021, 4091] avg=4096.0000 max=4197 max_rank=3 min=4021 min_rank=6 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16274 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[524, 485, 533, 466, 540, 499, 493, 519, 22, 4, 0, 0, 0, 0, 0, 0, 504, 515, 517, 512, 513, 518, 491, 478, 0, 0, 0, 0, 0, 0, 0, 0, 502, 519, 516, 510, 535, 541, 482, 528, 0, 0, 0, 0, 0, 0, 0, 0, 507, 487, 496, 517, 559, 540, 521, 497, 0, 0, 0, 0, 0, 0, 0, 0, 499, 525, 496, 517, 489, 515, 502, 509, 33, 0, 0, 0, 0, 0, 0, 0, 544, 486, 521, 528, 520, 504, 447, 537, 14, 7, 0, 0, 0, 0, 0, 0, 527, 538, 504, 530, 526, 509, 479, 528, 0, 0, 0, 0, 0, 0, 0, 0, 513, 496, 511, 512, 516, 474, 511, 506, 3, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4048, 4133, 4124, 4085, 4108, 4141, 4044] avg=4096.0000 max=4141 max_rank=6 min=4044 min_rank=7 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[485, 492, 555, 509, 518, 505, 465, 520, 23, 4, 0, 0, 0, 0, 0, 0, 519, 557, 533, 492, 493, 510, 481, 550, 0, 0, 0, 0, 0, 0, 0, 0, 533, 527, 523, 503, 510, 522, 482, 496, 0, 0, 0, 0, 0, 0, 0, 0, 498, 483, 537, 534, 497, 514, 492, 534, 0, 0, 0, 0, 0, 0, 0, 0, 509, 477, 507, 516, 509, 511, 522, 519, 33, 0, 0, 0, 0, 0, 0, 0, 546, 475, 524, 512, 495, 519, 520, 501, 14, 7, 0, 0, 0, 0, 0, 0, 480, 517, 516, 489, 508, 526, 498, 493, 0, 0, 0, 0, 0, 0, 0, 0, 480, 547, 481, 517, 517, 522, 542, 517, 4, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4135, 4096, 4089, 4103, 4113, 4027, 4129] avg=4096.0000 max=4135 max_rank=1 min=4027 min_rank=6 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[511, 527, 530, 517, 493, 457, 517, 526, 475, 488, 521, 561, 523, 495, 515, 542, 521, 504, 484, 524, 506, 512, 524, 488, 525, 525, 572, 501, 520, 513, 542, 455, 511, 487, 473, 509, 510, 529, 502, 512, 505, 523, 523, 503, 519, 496, 530, 499, 504, 484, 530, 516, 517, 533, 524, 535, 505, 486, 500, 492, 511, 556, 515, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[511, 530, 507, 495, 497, 500, 522, 477, 500, 515, 496, 501, 506, 493, 511, 509, 533, 525, 506, 499, 553, 502, 480, 490, 509, 538, 548, 543, 517, 503, 534, 514, 536, 506, 487, 533, 548, 499, 542, 525, 496, 501, 508, 532, 489, 544, 538, 508, 529, 508, 511, 508, 504, 491, 496, 495, 496, 496, 504, 534, 513, 508, 518, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[512, 515, 526, 477, 521, 521, 492, 513, 550, 494, 528, 506, 553, 517, 474, 473, 532, 497, 537, 496, 493, 515, 493, 481, 491, 491, 548, 488, 540, 531, 521, 514, 511, 519, 477, 503, 531, 478, 575, 558, 522, 499, 514, 495, 546, 525, 538, 474, 496, 539, 487, 493, 494, 499, 518, 474, 539, 498, 525, 534, 533, 499, 517, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[548, 511, 536, 497, 509, 521, 542, 490, 495, 546, 510, 530, 502, 545, 495, 516, 498, 494, 502, 500, 513, 489, 521, 498, 533, 524, 508, 506, 486, 521, 500, 529, 508, 497, 527, 555, 517, 477, 470, 500, 493, 506, 513, 521, 546, 501, 482, 538, 529, 487, 535, 507, 524, 515, 522, 475, 546, 516, 478, 528, 504, 521, 496, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[525, 546, 510, 504, 487, 498, 494, 504, 500, 535, 537, 502, 526, 509, 523, 477, 525, 500, 504, 509, 502, 527, 495, 533, 523, 485, 483, 539, 522, 556, 488, 492, 513, 529, 490, 497, 524, 521, 457, 509, 527, 475, 513, 503, 488, 496, 556, 521, 534, 504, 530, 507, 490, 545, 538, 520, 488, 503, 558, 512, 531, 494, 513, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[511, 578, 501, 517, 516, 491, 510, 527, 477, 513, 540, 504, 547, 475, 549, 524, 519, 527, 477, 526, 494, 512, 480, 508, 516, 488, 522, 549, 494, 521, 513, 497, 485, 490, 527, 505, 538, 476, 526, 501, 500, 525, 489, 537, 516, 509, 504, 493, 517, 472, 539, 509, 528, 538, 516, 536, 510, 510, 519, 512, 472, 491, 510, 545] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[509, 552, 541, 534, 525, 545, 492, 517, 511, 499, 516, 525, 512, 480, 537, 515, 496, 492, 504, 527, 492, 495, 529, 498, 508, 520, 513, 499, 507, 520, 545, 514, 516, 475, 524, 523, 464, 515, 531, 512, 477, 505, 502, 513, 543, 537, 525, 519, 508, 500, 512, 473, 501, 471, 522, 509, 459, 493, 532, 539, 511, 513, 573, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[520, 494, 495, 514, 501, 505, 522, 537, 498, 495, 506, 528, 525, 501, 485, 512, 508, 528, 503, 507, 564, 520, 523, 498, 485, 504, 495, 533, 501, 493, 524, 527, 525, 527, 500, 517, 503, 519, 521, 514, 512, 527, 550, 545, 492, 506, 489, 522, 488, 517, 504, 528, 496, 490, 541, 488, 524, 520, 480, 541, 524, 520, 480, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[511, 516, 530, 517, 493, 457, 517, 526, 0, 0, 0, 0, 0, 0, 0, 0, 475, 488, 521, 561, 523, 495, 515, 542, 0, 0, 0, 0, 0, 0, 0, 0, 521, 504, 484, 524, 506, 512, 524, 488, 27, 4, 0, 0, 0, 0, 0, 0, 525, 525, 545, 501, 520, 513, 542, 455, 0, 0, 0, 0, 0, 0, 0, 0, 511, 487, 473, 509, 510, 529, 502, 512, 7, 2, 0, 0, 0, 0, 0, 0, 505, 523, 523, 503, 519, 496, 522, 499, 0, 0, 0, 0, 0, 0, 0, 0, 504, 484, 530, 516, 517, 533, 524, 535, 7, 6, 0, 0, 0, 0, 0, 0, 505, 486, 500, 485, 511, 556, 515, 515, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4120, 4094, 4126, 4042, 4090, 4156, 4073] avg=4096.0000 max=4156 max_rank=6 min=4042 min_rank=4 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16361 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[511, 519, 507, 495, 497, 500, 522, 477, 0, 0, 0, 0, 0, 0, 0, 0, 500, 515, 496, 501, 506, 493, 511, 509, 0, 0, 0, 0, 0, 0, 0, 0, 533, 525, 506, 499, 553, 502, 480, 490, 25, 4, 0, 0, 0, 0, 0, 0, 509, 538, 523, 543, 517, 503, 534, 514, 0, 0, 0, 0, 0, 0, 0, 0, 536, 506, 487, 533, 548, 499, 542, 525, 7, 2, 0, 0, 0, 0, 0, 0, 496, 501, 508, 532, 489, 544, 530, 508, 0, 0, 0, 0, 0, 0, 0, 0, 529, 508, 511, 508, 504, 491, 496, 495, 7, 6, 0, 0, 0, 0, 0, 0, 496, 496, 504, 527, 513, 508, 518, 501, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4028, 4031, 4117, 4181, 4185, 4108, 4055, 4063] avg=4096.0000 max=4185 max_rank=4 min=4028 min_rank=0 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16411 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[548, 490, 536, 497, 509, 521, 542, 490, 0, 0, 0, 0, 0, 0, 0, 0, 495, 546, 510, 530, 502, 545, 495, 516, 0, 0, 0, 0, 0, 0, 0, 0, 498, 494, 502, 500, 513, 489, 521, 498, 28, 9, 0, 0, 0, 0, 0, 0, 533, 524, 480, 506, 486, 521, 500, 529, 0, 0, 0, 0, 0, 0, 0, 0, 508, 497, 527, 555, 517, 477, 470, 500, 12, 7, 0, 0, 0, 0, 0, 0, 493, 506, 513, 521, 546, 501, 464, 538, 0, 0, 0, 0, 0, 0, 0, 0, 529, 487, 535, 507, 524, 515, 522, 475, 12, 11, 0, 0, 0, 0, 0, 0, 546, 516, 478, 516, 504, 521, 496, 519, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4133, 4139, 4052, 4079, 4070, 4082, 4117, 4096] avg=4096.0000 max=4139 max_rank=1 min=4052 min_rank=2 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16365 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[525, 534, 510, 504, 487, 498, 494, 504, 0, 0, 0, 0, 0, 0, 0, 0, 500, 535, 537, 502, 526, 509, 523, 477, 0, 0, 0, 0, 0, 0, 0, 0, 525, 500, 504, 509, 502, 527, 495, 533, 22, 4, 0, 0, 0, 0, 0, 0, 523, 485, 461, 539, 522, 556, 488, 492, 0, 0, 0, 0, 0, 0, 0, 0, 513, 529, 490, 497, 524, 521, 457, 509, 7, 2, 0, 0, 0, 0, 0, 0, 527, 475, 513, 503, 488, 496, 547, 521, 0, 0, 0, 0, 0, 0, 0, 0, 534, 504, 530, 507, 490, 545, 538, 520, 8, 7, 0, 0, 0, 0, 0, 0, 488, 503, 558, 505, 531, 494, 513, 522, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4056, 4109, 4121, 4066, 4049, 4070, 4183, 4114] avg=4096.0000 max=4183 max_rank=6 min=4049 min_rank=4 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16416 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[511, 565, 501, 517, 516, 491, 510, 527, 0, 0, 0, 0, 0, 0, 0, 0, 477, 513, 540, 504, 547, 475, 549, 524, 0, 0, 0, 0, 0, 0, 0, 0, 519, 527, 477, 526, 494, 512, 480, 508, 24, 5, 0, 0, 0, 0, 0, 0, 516, 488, 498, 549, 494, 521, 513, 497, 0, 0, 0, 0, 0, 0, 0, 0, 485, 490, 527, 505, 538, 476, 526, 501, 7, 2, 0, 0, 0, 0, 0, 0, 500, 525, 489, 537, 516, 509, 496, 493, 0, 0, 0, 0, 0, 0, 0, 0, 517, 472, 539, 509, 528, 538, 516, 536, 8, 6, 0, 0, 0, 0, 0, 0, 510, 510, 519, 505, 472, 491, 510, 545, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4138, 4129, 4072, 4076, 4057, 4065, 4169, 4062] avg=4096.0000 max=4169 max_rank=6 min=4057 min_rank=4 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16353 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[512, 504, 526, 477, 521, 521, 492, 513, 0, 0, 0, 0, 0, 0, 0, 0, 550, 494, 528, 506, 553, 517, 474, 473, 0, 0, 0, 0, 0, 0, 0, 0, 532, 497, 537, 496, 493, 515, 493, 481, 25, 4, 0, 0, 0, 0, 0, 0, 491, 491, 523, 488, 540, 531, 521, 514, 0, 0, 0, 0, 0, 0, 0, 0, 511, 519, 477, 503, 531, 478, 575, 558, 7, 2, 0, 0, 0, 0, 0, 0, 522, 499, 514, 495, 546, 525, 530, 474, 0, 0, 0, 0, 0, 0, 0, 0, 496, 539, 487, 493, 494, 499, 518, 474, 7, 6, 0, 0, 0, 0, 0, 0, 539, 498, 525, 527, 533, 499, 517, 518, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 4095, 4073, 4099, 4161, 4105, 4013, 4156] avg=4096.0000 max=4161 max_rank=4 min=4013 min_rank=6 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16435 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[509, 540, 541, 534, 525, 545, 492, 517, 0, 0, 0, 0, 0, 0, 0, 0, 511, 499, 516, 525, 512, 480, 537, 515, 0, 0, 0, 0, 0, 0, 0, 0, 496, 492, 504, 527, 492, 495, 529, 498, 24, 4, 0, 0, 0, 0, 0, 0, 508, 520, 489, 499, 507, 520, 545, 514, 0, 0, 0, 0, 0, 0, 0, 0, 516, 475, 524, 523, 464, 515, 531, 512, 7, 2, 0, 0, 0, 0, 0, 0, 477, 505, 502, 513, 543, 537, 517, 519, 0, 0, 0, 0, 0, 0, 0, 0, 508, 500, 512, 473, 501, 471, 522, 509, 8, 6, 0, 0, 0, 0, 0, 0, 459, 493, 532, 532, 511, 513, 573, 502, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4203, 4095, 4061, 4102, 4069, 4113, 4010, 4115] avg=4096.0000 max=4203 max_rank=0 min=4010 min_rank=6 imbalance_ratio=1.026123 max_load_reduction=28565 max_load_reduction_pct=0.871735 moved_to_echo=16307 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[520, 483, 495, 514, 501, 505, 522, 537, 0, 0, 0, 0, 0, 0, 0, 0, 498, 495, 506, 528, 525, 501, 485, 512, 0, 0, 0, 0, 0, 0, 0, 0, 508, 528, 503, 507, 564, 520, 523, 498, 23, 4, 0, 0, 0, 0, 0, 0, 485, 504, 472, 533, 501, 493, 524, 527, 0, 0, 0, 0, 0, 0, 0, 0, 525, 527, 500, 517, 503, 519, 521, 514, 7, 2, 0, 0, 0, 0, 0, 0, 512, 527, 550, 545, 492, 506, 481, 522, 0, 0, 0, 0, 0, 0, 0, 0, 488, 517, 504, 528, 496, 490, 541, 488, 7, 6, 0, 0, 0, 0, 0, 0, 524, 520, 480, 534, 524, 520, 480, 507, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4050, 4178, 4039, 4135, 4135, 4065, 4089] avg=4096.0000 max=4178 max_rank=2 min=4039 min_rank=3 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16424 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[531, 461, 498, 498, 518, 496, 507, 534, 477, 494, 540, 548, 512, 516, 485, 516, 524, 510, 537, 504, 513, 516, 474, 518, 495, 530, 496, 524, 518, 509, 520, 547, 520, 544, 514, 519, 506, 509, 526, 499, 511, 509, 489, 509, 524, 508, 486, 507, 538, 495, 537, 527, 501, 505, 484, 507, 512, 510, 535, 510, 514, 537, 502, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[523, 526, 517, 493, 551, 525, 505, 523, 509, 508, 508, 505, 536, 521, 482, 536, 525, 513, 525, 499, 543, 520, 548, 472, 471, 527, 496, 501, 524, 512, 506, 546, 513, 530, 492, 438, 538, 523, 526, 545, 523, 523, 466, 525, 499, 499, 539, 493, 504, 512, 478, 510, 499, 511, 457, 530, 510, 499, 498, 515, 504, 541, 527, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[510, 527, 526, 501, 504, 493, 505, 550, 542, 519, 505, 475, 512, 506, 520, 539, 555, 509, 514, 531, 496, 508, 512, 510, 540, 514, 513, 466, 491, 505, 509, 503, 515, 505, 520, 454, 486, 547, 553, 506, 524, 489, 523, 499, 498, 498, 548, 510, 527, 547, 520, 510, 529, 493, 483, 503, 506, 498, 504, 488, 539, 525, 492, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[544, 477, 537, 485, 526, 521, 483, 524, 508, 554, 494, 517, 514, 491, 488, 513, 515, 530, 511, 491, 493, 490, 532, 509, 488, 479, 502, 505, 513, 526, 471, 547, 513, 529, 468, 493, 497, 523, 557, 515, 505, 514, 510, 536, 526, 524, 538, 515, 519, 532, 512, 537, 499, 495, 500, 497, 542, 521, 509, 536, 543, 498, 489, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[518, 490, 482, 520, 542, 511, 469, 491, 505, 538, 510, 512, 499, 555, 557, 527, 492, 510, 534, 546, 495, 524, 497, 525, 482, 538, 471, 549, 510, 535, 507, 538, 519, 518, 529, 518, 504, 478, 529, 498, 510, 522, 525, 507, 506, 476, 528, 496, 491, 500, 481, 539, 512, 514, 463, 524, 546, 494, 530, 527, 538, 521, 477, 469] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[489, 528, 464, 486, 523, 521, 521, 484, 506, 508, 548, 469, 537, 526, 525, 535, 497, 500, 489, 534, 541, 492, 507, 511, 505, 506, 503, 511, 515, 531, 538, 495, 544, 510, 530, 503, 511, 529, 504, 540, 553, 549, 497, 491, 532, 515, 557, 512, 476, 528, 473, 527, 522, 512, 496, 504, 472, 523, 482, 498, 481, 502, 533, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[515, 570, 520, 531, 522, 503, 514, 488, 526, 520, 493, 496, 492, 542, 474, 500, 493, 498, 550, 530, 502, 530, 500, 530, 507, 517, 503, 483, 515, 512, 520, 498, 500, 528, 514, 499, 541, 508, 489, 494, 531, 487, 525, 518, 520, 529, 521, 499, 560, 491, 509, 513, 521, 496, 547, 490, 516, 478, 496, 511, 519, 500, 522, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[509, 503, 521, 494, 512, 507, 518, 503, 502, 499, 517, 494, 528, 519, 505, 505, 497, 544, 509, 483, 499, 504, 524, 530, 529, 534, 521, 509, 487, 499, 548, 510, 526, 493, 528, 511, 516, 479, 543, 504, 545, 559, 526, 463, 497, 523, 519, 465, 508, 500, 547, 531, 537, 532, 494, 479, 508, 554, 469, 516, 491, 473, 544, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[531, 461, 498, 498, 518, 496, 507, 534, 5, 4, 0, 0, 0, 0, 0, 0, 477, 494, 540, 548, 512, 506, 485, 516, 0, 0, 0, 0, 0, 0, 0, 0, 524, 510, 526, 504, 513, 516, 474, 518, 0, 0, 0, 0, 0, 0, 0, 0, 495, 530, 496, 524, 518, 509, 520, 547, 5, 0, 0, 0, 0, 0, 0, 0, 520, 544, 514, 519, 506, 509, 516, 499, 0, 0, 0, 0, 0, 0, 0, 0, 511, 509, 489, 509, 524, 508, 475, 507, 0, 0, 0, 0, 0, 0, 0, 0, 538, 495, 537, 527, 501, 505, 484, 507, 11, 6, 0, 0, 0, 0, 0, 0, 512, 510, 535, 510, 514, 537, 502, 508, 6, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4052, 4078, 4085, 4144, 4127, 4032, 4111, 4139] avg=4096.0000 max=4144 max_rank=3 min=4032 min_rank=5 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16409 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[523, 526, 517, 493, 551, 525, 505, 523, 5, 4, 0, 0, 0, 0, 0, 0, 509, 508, 508, 505, 536, 511, 482, 536, 0, 0, 0, 0, 0, 0, 0, 0, 525, 513, 514, 499, 543, 520, 548, 472, 0, 0, 0, 0, 0, 0, 0, 0, 471, 527, 496, 501, 524, 512, 506, 546, 5, 0, 0, 0, 0, 0, 0, 0, 513, 530, 492, 438, 538, 523, 516, 545, 0, 0, 0, 0, 0, 0, 0, 0, 523, 523, 466, 525, 499, 499, 527, 493, 0, 0, 0, 0, 0, 0, 0, 0, 504, 512, 478, 510, 499, 511, 457, 530, 12, 6, 0, 0, 0, 0, 0, 0, 510, 499, 498, 515, 504, 541, 527, 505, 6, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4172, 4095, 4134, 4088, 4095, 4055, 4019, 4110] avg=4096.0000 max=4172 max_rank=0 min=4019 min_rank=6 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16279 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[510, 527, 526, 501, 504, 493, 505, 550, 5, 4, 0, 0, 0, 0, 0, 0, 542, 519, 505, 475, 512, 496, 520, 539, 0, 0, 0, 0, 0, 0, 0, 0, 555, 509, 503, 531, 496, 508, 512, 510, 0, 0, 0, 0, 0, 0, 0, 0, 540, 514, 513, 466, 491, 505, 509, 503, 5, 0, 0, 0, 0, 0, 0, 0, 515, 505, 520, 454, 486, 547, 542, 506, 0, 0, 0, 0, 0, 0, 0, 0, 524, 489, 523, 499, 498, 498, 535, 510, 0, 0, 0, 0, 0, 0, 0, 0, 527, 547, 520, 510, 529, 493, 483, 503, 13, 6, 0, 0, 0, 0, 0, 0, 506, 498, 504, 488, 539, 525, 492, 519, 7, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4108, 4124, 4046, 4075, 4076, 4131, 4083] avg=4096.0000 max=4131 max_rank=6 min=4046 min_rank=3 imbalance_ratio=1.008545 max_load_reduction=28637 max_load_reduction_pct=0.873932 moved_to_echo=16365 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[518, 490, 482, 520, 542, 511, 469, 491, 5, 4, 0, 0, 0, 0, 0, 0, 505, 538, 510, 512, 499, 544, 557, 527, 0, 0, 0, 0, 0, 0, 0, 0, 492, 510, 523, 546, 495, 524, 497, 525, 0, 0, 0, 0, 0, 0, 0, 0, 482, 538, 471, 549, 510, 535, 507, 538, 6, 0, 0, 0, 0, 0, 0, 0, 519, 518, 529, 518, 504, 478, 519, 498, 0, 0, 0, 0, 0, 0, 0, 0, 510, 522, 525, 507, 506, 476, 516, 496, 0, 0, 0, 0, 0, 0, 0, 0, 491, 500, 481, 539, 512, 514, 463, 524, 12, 6, 0, 0, 0, 0, 0, 0, 546, 494, 530, 527, 538, 521, 477, 469, 6, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4032, 4192, 4112, 4136, 4083, 4058, 4042, 4113] avg=4096.0000 max=4192 max_rank=1 min=4032 min_rank=0 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16296 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[544, 477, 537, 485, 526, 521, 483, 524, 8, 9, 0, 0, 0, 0, 0, 0, 508, 554, 494, 517, 514, 473, 488, 513, 0, 0, 0, 0, 0, 0, 0, 0, 515, 530, 491, 491, 493, 490, 532, 509, 0, 0, 0, 0, 0, 0, 0, 0, 488, 479, 502, 505, 513, 526, 471, 547, 10, 0, 0, 0, 0, 0, 0, 0, 513, 529, 468, 493, 497, 523, 537, 515, 0, 0, 0, 0, 0, 0, 0, 0, 505, 514, 510, 536, 526, 524, 521, 515, 0, 0, 0, 0, 0, 0, 0, 0, 519, 532, 512, 537, 499, 495, 500, 497, 17, 10, 0, 0, 0, 0, 0, 0, 542, 521, 509, 536, 543, 498, 489, 498, 11, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4061, 4051, 4041, 4075, 4151, 4118, 4157] avg=4096.0000 max=4157 max_rank=7 min=4041 min_rank=3 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16501 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[489, 528, 464, 486, 523, 521, 521, 484, 5, 4, 0, 0, 0, 0, 0, 0, 506, 508, 548, 469, 537, 515, 525, 535, 0, 0, 0, 0, 0, 0, 0, 0, 497, 500, 479, 534, 541, 492, 507, 511, 0, 0, 0, 0, 0, 0, 0, 0, 505, 506, 503, 511, 515, 531, 538, 495, 6, 0, 0, 0, 0, 0, 0, 0, 544, 510, 530, 503, 511, 529, 494, 540, 0, 0, 0, 0, 0, 0, 0, 0, 553, 549, 497, 491, 532, 515, 544, 512, 0, 0, 0, 0, 0, 0, 0, 0, 476, 528, 473, 527, 522, 512, 496, 504, 13, 5, 0, 0, 0, 0, 0, 0, 472, 523, 482, 498, 481, 502, 533, 517, 6, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4025, 4143, 4061, 4110, 4161, 4193, 4056, 4019] avg=4096.0000 max=4193 max_rank=5 min=4019 min_rank=7 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[515, 570, 520, 531, 522, 503, 514, 488, 5, 4, 0, 0, 0, 0, 0, 0, 526, 520, 493, 496, 492, 531, 474, 500, 0, 0, 0, 0, 0, 0, 0, 0, 493, 498, 539, 530, 502, 530, 500, 530, 0, 0, 0, 0, 0, 0, 0, 0, 507, 517, 503, 483, 515, 512, 520, 498, 6, 0, 0, 0, 0, 0, 0, 0, 500, 528, 514, 499, 541, 508, 479, 494, 0, 0, 0, 0, 0, 0, 0, 0, 531, 487, 525, 518, 520, 529, 509, 499, 0, 0, 0, 0, 0, 0, 0, 0, 560, 491, 509, 513, 521, 496, 547, 490, 12, 6, 0, 0, 0, 0, 0, 0, 516, 478, 496, 511, 519, 500, 522, 502, 6, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4172, 4032, 4122, 4061, 4063, 4118, 4145, 4055] avg=4096.0000 max=4172 max_rank=0 min=4032 min_rank=1 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[509, 503, 521, 494, 512, 507, 518, 503, 5, 4, 0, 0, 0, 0, 0, 0, 502, 499, 517, 494, 528, 509, 505, 505, 0, 0, 0, 0, 0, 0, 0, 0, 497, 544, 498, 483, 499, 504, 524, 530, 0, 0, 0, 0, 0, 0, 0, 0, 529, 534, 521, 509, 487, 499, 548, 510, 5, 0, 0, 0, 0, 0, 0, 0, 526, 493, 528, 511, 516, 479, 532, 504, 0, 0, 0, 0, 0, 0, 0, 0, 545, 559, 526, 463, 497, 523, 507, 465, 0, 0, 0, 0, 0, 0, 0, 0, 508, 500, 547, 531, 537, 532, 494, 479, 12, 6, 0, 0, 0, 0, 0, 0, 508, 554, 469, 516, 491, 473, 544, 525, 7, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4059, 4079, 4142, 4089, 4085, 4146, 4092] avg=4096.0000 max=4146 max_rank=6 min=4059 min_rank=1 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16412 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 3968, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 3968, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[509, 512, 508, 502, 530, 542, 527, 504, 503, 523, 484, 488, 512, 497, 512, 488, 495, 490, 508, 509, 564, 508, 484, 487, 534, 523, 520, 505, 517, 526, 532, 462, 511, 521, 497, 523, 479, 520, 520, 515, 477, 517, 549, 520, 528, 488, 533, 529, 527, 477, 512, 519, 527, 519, 508, 502, 516, 537, 517, 530, 489, 510, 511, 535] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[484, 514, 505, 527, 498, 509, 486, 515, 560, 518, 532, 531, 515, 509, 499, 491, 506, 508, 544, 539, 512, 508, 502, 490, 500, 493, 530, 530, 523, 478, 470, 544, 530, 499, 475, 510, 501, 508, 528, 509, 509, 483, 535, 542, 541, 524, 498, 460, 501, 532, 511, 506, 536, 555, 510, 492, 553, 466, 520, 527, 509, 506, 514, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[488, 553, 507, 476, 510, 488, 504, 487, 553, 528, 520, 530, 495, 480, 492, 515, 493, 499, 526, 469, 507, 537, 515, 507, 522, 528, 535, 531, 517, 507, 529, 524, 545, 519, 542, 482, 502, 504, 503, 505, 509, 546, 472, 516, 477, 492, 520, 515, 491, 533, 520, 540, 509, 491, 501, 491, 527, 550, 539, 517, 495, 495, 535, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[502, 516, 530, 518, 488, 505, 494, 502, 508, 519, 492, 544, 519, 495, 518, 503, 504, 499, 514, 470, 518, 521, 547, 530, 531, 525, 514, 509, 492, 512, 548, 497, 508, 562, 529, 494, 531, 510, 504, 485, 525, 507, 502, 533, 551, 508, 534, 503, 532, 489, 499, 477, 475, 517, 520, 479, 505, 506, 470, 527, 552, 524, 515, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[533, 497, 527, 515, 539, 484, 506, 519, 505, 562, 471, 486, 543, 490, 465, 497, 529, 485, 494, 497, 530, 546, 518, 460, 521, 498, 548, 527, 476, 560, 505, 546, 501, 526, 517, 512, 505, 500, 515, 514, 510, 504, 512, 520, 516, 503, 523, 522, 512, 513, 534, 494, 498, 503, 497, 519, 492, 502, 519, 486, 511, 548, 513, 548] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[497, 534, 526, 536, 513, 521, 474, 510, 494, 493, 525, 510, 525, 519, 531, 555, 485, 514, 515, 540, 502, 515, 524, 489, 540, 510, 495, 534, 525, 514, 516, 476, 491, 487, 487, 521, 531, 522, 532, 534, 503, 567, 467, 483, 485, 499, 479, 494, 543, 505, 493, 510, 523, 545, 516, 538, 506, 546, 506, 519, 489, 498, 512, 480] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[486, 499, 534, 496, 505, 516, 529, 512, 531, 497, 511, 519, 535, 557, 484, 533, 530, 514, 499, 540, 504, 550, 509, 503, 514, 502, 553, 518, 496, 530, 509, 516, 475, 519, 531, 500, 548, 531, 511, 505, 507, 489, 509, 499, 489, 477, 530, 497, 536, 508, 503, 509, 498, 496, 530, 500, 496, 522, 504, 512, 493, 532, 508, 473] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[517, 541, 507, 528, 547, 505, 504, 520, 478, 500, 550, 544, 498, 523, 501, 491, 520, 477, 471, 474, 496, 474, 518, 537, 513, 512, 524, 485, 530, 513, 480, 503, 510, 459, 450, 522, 499, 514, 523, 516, 514, 558, 518, 501, 522, 483, 549, 498, 542, 504, 504, 546, 480, 542, 506, 499, 548, 553, 498, 537, 553, 495, 511, 533] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[509, 512, 508, 502, 530, 542, 527, 504, 6, 0, 0, 0, 0, 0, 0, 0, 503, 523, 484, 486, 512, 497, 512, 488, 0, 0, 0, 0, 0, 0, 0, 0, 495, 490, 508, 509, 564, 508, 484, 487, 24, 0, 0, 0, 0, 0, 0, 0, 534, 523, 493, 505, 517, 526, 532, 462, 0, 0, 0, 0, 0, 0, 0, 0, 511, 521, 497, 523, 479, 520, 520, 515, 11, 0, 0, 0, 0, 0, 0, 0, 477, 517, 549, 520, 528, 488, 533, 529, 8, 3, 0, 0, 0, 0, 0, 0, 527, 477, 512, 519, 527, 519, 508, 502, 2, 0, 0, 0, 0, 0, 0, 0, 516, 512, 517, 530, 489, 510, 511, 535, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4140, 4005, 4069, 4092, 4097, 4152, 4093, 4120] avg=4096.0000 max=4152 max_rank=5 min=4005 min_rank=1 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16462 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[497, 534, 526, 536, 513, 521, 474, 510, 6, 0, 0, 0, 0, 0, 0, 0, 494, 493, 525, 508, 525, 519, 531, 555, 0, 0, 0, 0, 0, 0, 0, 0, 485, 514, 515, 540, 502, 515, 524, 489, 23, 0, 0, 0, 0, 0, 0, 0, 540, 510, 469, 534, 525, 514, 516, 476, 0, 0, 0, 0, 0, 0, 0, 0, 491, 487, 487, 521, 531, 522, 532, 534, 11, 0, 0, 0, 0, 0, 0, 0, 503, 567, 467, 483, 485, 499, 479, 494, 8, 3, 0, 0, 0, 0, 0, 0, 543, 505, 493, 510, 523, 545, 516, 538, 2, 0, 0, 0, 0, 0, 0, 0, 506, 521, 506, 519, 489, 498, 512, 480, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4117, 4150, 4107, 4084, 4116, 3988, 4175, 4031] avg=4096.0000 max=4175 max_rank=6 min=3988 min_rank=5 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16310 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[488, 553, 507, 476, 510, 488, 504, 487, 6, 0, 0, 0, 0, 0, 0, 0, 553, 528, 520, 527, 495, 480, 492, 515, 0, 0, 0, 0, 0, 0, 0, 0, 493, 499, 526, 469, 507, 537, 515, 507, 25, 0, 0, 0, 0, 0, 0, 0, 522, 528, 507, 531, 517, 507, 529, 524, 0, 0, 0, 0, 0, 0, 0, 0, 545, 519, 542, 482, 502, 504, 503, 505, 11, 0, 0, 0, 0, 0, 0, 0, 509, 546, 472, 516, 477, 492, 520, 515, 8, 3, 0, 0, 0, 0, 0, 0, 491, 533, 520, 540, 509, 491, 501, 491, 3, 0, 0, 0, 0, 0, 0, 0, 527, 525, 539, 517, 495, 495, 535, 513, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4019, 4110, 4078, 4165, 4113, 4058, 4079, 4146] avg=4096.0000 max=4165 max_rank=3 min=4019 min_rank=0 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16396 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[484, 514, 505, 527, 498, 509, 486, 515, 8, 4, 0, 0, 0, 0, 0, 0, 560, 518, 532, 521, 515, 509, 499, 491, 0, 0, 0, 0, 0, 0, 0, 0, 506, 508, 544, 539, 512, 508, 502, 490, 28, 0, 0, 0, 0, 0, 0, 0, 500, 493, 493, 530, 523, 478, 470, 544, 0, 0, 0, 0, 0, 0, 0, 0, 530, 499, 475, 510, 501, 508, 528, 509, 14, 0, 0, 0, 0, 0, 0, 0, 509, 483, 535, 542, 541, 524, 498, 460, 12, 9, 0, 0, 0, 0, 0, 0, 501, 532, 511, 506, 536, 555, 510, 492, 6, 0, 0, 0, 0, 0, 0, 0, 553, 432, 520, 527, 509, 506, 514, 508, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4050, 4145, 4137, 4031, 4074, 4113, 4149, 4069] avg=4096.0000 max=4149 max_rank=6 min=4031 min_rank=3 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16405 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[502, 516, 530, 518, 488, 505, 494, 502, 5, 0, 0, 0, 0, 0, 0, 0, 508, 519, 492, 541, 519, 495, 518, 503, 0, 0, 0, 0, 0, 0, 0, 0, 504, 499, 514, 470, 518, 521, 547, 530, 24, 0, 0, 0, 0, 0, 0, 0, 531, 525, 487, 509, 492, 512, 548, 497, 0, 0, 0, 0, 0, 0, 0, 0, 508, 562, 529, 494, 531, 510, 504, 485, 10, 0, 0, 0, 0, 0, 0, 0, 525, 507, 502, 533, 551, 508, 534, 503, 8, 3, 0, 0, 0, 0, 0, 0, 532, 489, 499, 477, 475, 517, 520, 479, 3, 0, 0, 0, 0, 0, 0, 0, 505, 483, 470, 527, 552, 524, 515, 511, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4095, 4127, 4101, 4133, 4174, 3991, 4087] avg=4096.0000 max=4174 max_rank=5 min=3991 min_rank=6 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[533, 497, 527, 515, 539, 484, 506, 519, 5, 0, 0, 0, 0, 0, 0, 0, 505, 562, 471, 484, 543, 490, 465, 497, 0, 0, 0, 0, 0, 0, 0, 0, 529, 485, 494, 497, 530, 546, 518, 460, 25, 0, 0, 0, 0, 0, 0, 0, 521, 498, 520, 527, 476, 560, 505, 546, 0, 0, 0, 0, 0, 0, 0, 0, 501, 526, 517, 512, 505, 500, 515, 514, 10, 0, 0, 0, 0, 0, 0, 0, 510, 504, 512, 520, 516, 503, 523, 522, 8, 3, 0, 0, 0, 0, 0, 0, 512, 513, 534, 494, 498, 503, 497, 519, 2, 0, 0, 0, 0, 0, 0, 0, 492, 479, 519, 486, 511, 548, 513, 548, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4017, 4084, 4153, 4100, 4121, 4072, 4096] avg=4096.0000 max=4153 max_rank=3 min=4017 min_rank=1 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[486, 499, 534, 496, 505, 516, 529, 512, 5, 0, 0, 0, 0, 0, 0, 0, 531, 497, 511, 516, 535, 557, 484, 533, 0, 0, 0, 0, 0, 0, 0, 0, 530, 514, 499, 540, 504, 550, 509, 503, 26, 0, 0, 0, 0, 0, 0, 0, 514, 502, 524, 518, 496, 530, 509, 516, 0, 0, 0, 0, 0, 0, 0, 0, 475, 519, 531, 500, 548, 531, 511, 505, 11, 0, 0, 0, 0, 0, 0, 0, 507, 489, 509, 499, 489, 477, 530, 497, 8, 3, 0, 0, 0, 0, 0, 0, 536, 508, 503, 509, 498, 496, 530, 500, 3, 0, 0, 0, 0, 0, 0, 0, 496, 498, 504, 512, 493, 532, 508, 473, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4164, 4175, 4109, 4131, 4008, 4083, 4016] avg=4096.0000 max=4175 max_rank=2 min=4008 min_rank=5 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16238 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[517, 541, 507, 528, 547, 505, 504, 520, 6, 0, 0, 0, 0, 0, 0, 0, 478, 500, 550, 541, 498, 523, 501, 491, 0, 0, 0, 0, 0, 0, 0, 0, 520, 477, 471, 474, 496, 474, 518, 537, 24, 0, 0, 0, 0, 0, 0, 0, 513, 512, 497, 485, 530, 513, 480, 503, 0, 0, 0, 0, 0, 0, 0, 0, 510, 459, 450, 522, 499, 514, 523, 516, 11, 0, 0, 0, 0, 0, 0, 0, 514, 558, 518, 501, 522, 483, 549, 498, 8, 3, 0, 0, 0, 0, 0, 0, 542, 504, 504, 546, 480, 542, 506, 499, 3, 0, 0, 0, 0, 0, 0, 0, 548, 528, 498, 537, 553, 495, 511, 533, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4175, 4082, 3991, 4033, 4004, 4154, 4126, 4203] avg=4096.0000 max=4203 max_rank=7 min=3991 min_rank=2 imbalance_ratio=1.026123 max_load_reduction=28565 max_load_reduction_pct=0.871735 moved_to_echo=16487 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[526, 495, 539, 541, 510, 513, 537, 478, 475, 530, 465, 510, 511, 542, 525, 514, 500, 495, 503, 515, 493, 519, 504, 517, 540, 524, 466, 511, 524, 505, 533, 506, 536, 526, 498, 519, 510, 519, 542, 509, 509, 497, 476, 520, 512, 546, 495, 480, 475, 484, 552, 500, 489, 511, 513, 517, 489, 531, 487, 544, 499, 575, 523, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[495, 521, 527, 504, 518, 483, 498, 537, 479, 539, 569, 520, 521, 547, 511, 496, 491, 485, 534, 534, 521, 523, 508, 535, 533, 511, 501, 495, 491, 509, 529, 510, 473, 511, 531, 498, 510, 495, 540, 509, 509, 486, 514, 481, 532, 512, 481, 531, 505, 522, 522, 522, 505, 491, 526, 507, 512, 529, 517, 480, 506, 521, 489, 526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[472, 505, 518, 549, 542, 512, 502, 509, 495, 552, 540, 526, 510, 510, 535, 521, 477, 480, 506, 504, 515, 509, 557, 523, 493, 507, 486, 521, 513, 502, 527, 508, 513, 458, 509, 527, 494, 558, 504, 510, 509, 531, 531, 523, 472, 522, 532, 519, 486, 512, 534, 525, 500, 510, 499, 507, 535, 503, 490, 528, 521, 495, 490, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[515, 523, 555, 548, 545, 535, 490, 491, 508, 462, 499, 524, 528, 509, 558, 529, 506, 559, 508, 498, 487, 522, 477, 521, 506, 482, 520, 541, 495, 501, 526, 508, 501, 521, 497, 496, 545, 496, 500, 513, 572, 514, 544, 529, 501, 498, 503, 512, 486, 542, 513, 512, 531, 489, 523, 471, 505, 502, 507, 524, 472, 479, 504, 490] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[486, 521, 504, 498, 535, 471, 483, 523, 469, 553, 485, 525, 568, 521, 499, 523, 548, 489, 493, 518, 509, 485, 484, 475, 474, 548, 509, 510, 549, 539, 507, 523, 536, 516, 507, 528, 523, 535, 537, 504, 500, 516, 494, 496, 498, 524, 525, 512, 500, 488, 526, 546, 546, 509, 536, 517, 511, 496, 506, 496, 536, 489, 505, 486] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[533, 553, 504, 513, 491, 500, 504, 521, 532, 541, 497, 544, 534, 504, 511, 498, 493, 508, 531, 524, 503, 548, 520, 495, 509, 479, 569, 493, 504, 539, 492, 487, 510, 493, 519, 534, 548, 486, 539, 557, 488, 506, 495, 553, 467, 505, 490, 518, 495, 545, 502, 504, 532, 478, 537, 451, 533, 490, 519, 468, 484, 536, 498, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[492, 495, 524, 435, 506, 507, 482, 517, 515, 505, 494, 486, 514, 506, 539, 533, 513, 522, 515, 523, 504, 501, 485, 533, 526, 541, 523, 529, 529, 487, 530, 450, 541, 505, 525, 509, 532, 480, 524, 502, 493, 493, 539, 521, 481, 520, 516, 502, 528, 539, 537, 512, 535, 497, 502, 550, 537, 486, 497, 508, 523, 523, 517, 533] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[535, 490, 511, 490, 535, 523, 489, 471, 505, 552, 505, 517, 517, 480, 517, 511, 480, 523, 507, 525, 493, 534, 516, 493, 513, 483, 533, 539, 551, 533, 538, 462, 520, 526, 495, 467, 536, 519, 534, 531, 532, 546, 509, 540, 506, 483, 488, 526, 492, 516, 508, 483, 560, 528, 520, 465, 520, 514, 504, 508, 519, 525, 489, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[526, 495, 539, 541, 510, 513, 537, 478, 11, 0, 0, 0, 0, 0, 0, 0, 475, 495, 465, 510, 511, 542, 525, 514, 0, 0, 0, 0, 0, 0, 0, 0, 500, 495, 503, 515, 493, 519, 504, 517, 10, 8, 0, 0, 0, 0, 0, 0, 540, 524, 466, 511, 524, 505, 527, 506, 0, 0, 0, 0, 0, 0, 0, 0, 536, 526, 498, 519, 510, 519, 516, 509, 0, 0, 0, 0, 0, 0, 0, 0, 509, 497, 476, 520, 512, 546, 495, 480, 6, 5, 0, 0, 0, 0, 0, 0, 475, 484, 552, 500, 489, 511, 513, 517, 0, 0, 0, 0, 0, 0, 0, 0, 489, 531, 487, 544, 499, 575, 523, 519, 27, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4150, 4037, 4064, 4103, 4133, 4046, 4041, 4194] avg=4096.0000 max=4194 max_rank=7 min=4037 min_rank=1 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16414 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[472, 505, 518, 549, 542, 512, 502, 509, 11, 0, 0, 0, 0, 0, 0, 0, 495, 515, 540, 526, 510, 510, 535, 521, 0, 0, 0, 0, 0, 0, 0, 0, 477, 480, 506, 504, 515, 509, 557, 523, 9, 8, 0, 0, 0, 0, 0, 0, 493, 507, 486, 521, 513, 502, 521, 508, 0, 0, 0, 0, 0, 0, 0, 0, 513, 458, 509, 527, 494, 558, 479, 510, 0, 0, 0, 0, 0, 0, 0, 0, 509, 531, 531, 523, 472, 522, 532, 519, 6, 5, 0, 0, 0, 0, 0, 0, 486, 512, 534, 525, 500, 510, 499, 507, 0, 0, 0, 0, 0, 0, 0, 0, 535, 503, 490, 528, 521, 495, 490, 495, 29, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4152, 4088, 4051, 4048, 4150, 4073, 4086] avg=4096.0000 max=4152 max_rank=1 min=4048 min_rank=4 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16357 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[495, 521, 527, 504, 518, 483, 498, 537, 11, 0, 0, 0, 0, 0, 0, 0, 479, 503, 569, 520, 521, 547, 511, 496, 0, 0, 0, 0, 0, 0, 0, 0, 491, 485, 534, 534, 521, 523, 508, 535, 10, 8, 0, 0, 0, 0, 0, 0, 533, 511, 501, 495, 491, 509, 523, 510, 0, 0, 0, 0, 0, 0, 0, 0, 473, 511, 531, 498, 510, 495, 514, 509, 0, 0, 0, 0, 0, 0, 0, 0, 509, 486, 514, 481, 532, 512, 481, 531, 6, 5, 0, 0, 0, 0, 0, 0, 505, 522, 522, 522, 505, 491, 526, 507, 0, 0, 0, 0, 0, 0, 0, 0, 512, 529, 517, 480, 506, 521, 489, 526, 28, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4146, 4149, 4073, 4041, 4057, 4100, 4108] avg=4096.0000 max=4149 max_rank=2 min=4041 min_rank=4 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16306 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[486, 521, 504, 498, 535, 471, 483, 523, 11, 0, 0, 0, 0, 0, 0, 0, 469, 515, 485, 525, 568, 521, 499, 523, 0, 0, 0, 0, 0, 0, 0, 0, 548, 489, 493, 518, 509, 485, 484, 475, 10, 9, 0, 0, 0, 0, 0, 0, 474, 548, 509, 510, 549, 539, 501, 523, 0, 0, 0, 0, 0, 0, 0, 0, 536, 516, 507, 528, 523, 535, 511, 504, 0, 0, 0, 0, 0, 0, 0, 0, 500, 516, 494, 496, 498, 524, 525, 512, 6, 5, 0, 0, 0, 0, 0, 0, 500, 488, 526, 546, 546, 509, 536, 517, 0, 0, 0, 0, 0, 0, 0, 0, 511, 496, 506, 496, 536, 489, 505, 486, 29, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4032, 4105, 4020, 4153, 4160, 4076, 4168, 4054] avg=4096.0000 max=4168 max_rank=6 min=4020 min_rank=2 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16458 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[515, 523, 555, 548, 545, 535, 490, 491, 16, 0, 0, 0, 0, 0, 0, 0, 508, 423, 499, 524, 528, 509, 558, 529, 0, 0, 0, 0, 0, 0, 0, 0, 506, 559, 508, 498, 487, 522, 477, 521, 14, 12, 0, 0, 0, 0, 0, 0, 506, 482, 520, 541, 495, 501, 514, 508, 0, 0, 0, 0, 0, 0, 0, 0, 501, 521, 497, 496, 545, 496, 463, 513, 0, 0, 0, 0, 0, 0, 0, 0, 572, 514, 544, 529, 501, 498, 503, 512, 9, 7, 0, 0, 0, 0, 0, 0, 486, 542, 513, 512, 531, 489, 523, 471, 3, 0, 0, 0, 0, 0, 0, 0, 505, 502, 507, 524, 472, 479, 504, 490, 27, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4218, 4078, 4104, 4067, 4032, 4189, 4070, 4010] avg=4096.0000 max=4218 max_rank=0 min=4010 min_rank=7 imbalance_ratio=1.029785 max_load_reduction=28550 max_load_reduction_pct=0.871277 moved_to_echo=16301 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[533, 553, 504, 513, 491, 500, 504, 521, 11, 0, 0, 0, 0, 0, 0, 0, 532, 505, 497, 544, 534, 504, 511, 498, 0, 0, 0, 0, 0, 0, 0, 0, 493, 508, 531, 524, 503, 548, 520, 495, 10, 8, 0, 0, 0, 0, 0, 0, 509, 479, 569, 493, 504, 539, 486, 487, 0, 0, 0, 0, 0, 0, 0, 0, 510, 493, 519, 534, 548, 486, 513, 557, 0, 0, 0, 0, 0, 0, 0, 0, 488, 506, 495, 553, 467, 505, 490, 518, 6, 5, 0, 0, 0, 0, 0, 0, 495, 545, 502, 504, 532, 478, 537, 451, 0, 0, 0, 0, 0, 0, 0, 0, 533, 490, 519, 468, 484, 536, 498, 514, 28, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4130, 4125, 4140, 4066, 4160, 4033, 4044, 4070] avg=4096.0000 max=4160 max_rank=4 min=4033 min_rank=5 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16307 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[492, 495, 524, 435, 506, 507, 482, 517, 11, 0, 0, 0, 0, 0, 0, 0, 515, 471, 494, 486, 514, 506, 539, 533, 0, 0, 0, 0, 0, 0, 0, 0, 513, 522, 515, 523, 504, 501, 485, 533, 10, 8, 0, 0, 0, 0, 0, 0, 526, 541, 523, 529, 529, 487, 524, 450, 0, 0, 0, 0, 0, 0, 0, 0, 541, 505, 525, 509, 532, 480, 498, 502, 0, 0, 0, 0, 0, 0, 0, 0, 493, 493, 539, 521, 481, 520, 516, 502, 6, 5, 0, 0, 0, 0, 0, 0, 528, 539, 537, 512, 535, 497, 502, 550, 0, 0, 0, 0, 0, 0, 0, 0, 537, 486, 497, 508, 523, 523, 517, 533, 26, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3969, 4058, 4114, 4109, 4092, 4076, 4200, 4150] avg=4096.0000 max=4200 max_rank=6 min=3969 min_rank=0 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16518 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[535, 490, 511, 490, 535, 523, 489, 471, 11, 0, 0, 0, 0, 0, 0, 0, 505, 515, 505, 517, 517, 480, 517, 511, 0, 0, 0, 0, 0, 0, 0, 0, 480, 523, 507, 525, 493, 534, 516, 493, 10, 8, 0, 0, 0, 0, 0, 0, 513, 483, 533, 539, 551, 533, 532, 462, 0, 0, 0, 0, 0, 0, 0, 0, 520, 526, 495, 467, 536, 519, 508, 531, 0, 0, 0, 0, 0, 0, 0, 0, 532, 546, 509, 540, 506, 483, 488, 526, 6, 5, 0, 0, 0, 0, 0, 0, 492, 516, 508, 483, 560, 528, 520, 465, 0, 0, 0, 0, 0, 0, 0, 0, 520, 514, 504, 508, 519, 525, 489, 488, 29, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4055, 4067, 4089, 4146, 4102, 4141, 4072, 4096] avg=4096.0000 max=4146 max_rank=3 min=4055 min_rank=0 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16411 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 3968, 4096, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4224, 3968, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3968, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[531, 500, 542, 530, 547, 522, 513, 488, 506, 532, 523, 477, 549, 543, 497, 497, 519, 545, 532, 536, 494, 464, 525, 469, 529, 487, 488, 533, 510, 522, 525, 538, 462, 522, 520, 481, 519, 539, 506, 495, 510, 507, 505, 502, 528, 546, 528, 488, 471, 528, 537, 478, 529, 491, 508, 491, 493, 475, 541, 514, 541, 531, 497, 472] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[496, 505, 521, 525, 518, 505, 482, 552, 509, 539, 509, 543, 515, 508, 512, 523, 541, 527, 543, 516, 495, 496, 469, 480, 512, 485, 540, 513, 512, 501, 516, 487, 544, 527, 478, 508, 525, 528, 547, 519, 513, 519, 454, 530, 517, 501, 475, 505, 504, 501, 509, 510, 513, 512, 511, 478, 501, 525, 531, 497, 471, 550, 540, 530] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[471, 518, 519, 510, 512, 548, 512, 482, 529, 502, 512, 525, 518, 471, 515, 491, 517, 505, 528, 521, 539, 499, 527, 563, 553, 499, 519, 509, 527, 484, 509, 468, 487, 503, 509, 487, 479, 516, 503, 541, 519, 517, 533, 515, 488, 537, 485, 504, 496, 517, 513, 513, 500, 521, 493, 513, 506, 520, 489, 526, 535, 533, 505, 563] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[550, 526, 502, 479, 508, 492, 517, 505, 491, 500, 498, 485, 529, 510, 498, 480, 512, 479, 540, 509, 495, 526, 522, 552, 511, 531, 541, 490, 504, 518, 527, 507, 515, 529, 513, 523, 517, 522, 527, 544, 525, 524, 504, 488, 534, 543, 518, 506, 498, 509, 527, 491, 492, 523, 511, 492, 546, 510, 481, 487, 547, 525, 483, 480] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[505, 493, 558, 500, 528, 521, 493, 518, 525, 483, 522, 545, 520, 515, 503, 526, 531, 499, 568, 502, 518, 508, 551, 541, 484, 507, 498, 497, 535, 479, 531, 513, 480, 533, 503, 497, 511, 530, 504, 506, 485, 485, 538, 514, 520, 534, 493, 484, 504, 546, 532, 497, 516, 531, 512, 499, 523, 488, 499, 469, 524, 492, 474, 528] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[544, 525, 486, 539, 506, 549, 496, 527, 503, 533, 519, 527, 515, 505, 511, 515, 537, 514, 493, 517, 522, 500, 556, 475, 486, 492, 526, 477, 509, 492, 521, 545, 521, 527, 523, 520, 495, 533, 516, 505, 523, 496, 470, 516, 519, 517, 503, 491, 493, 497, 516, 487, 476, 558, 528, 506, 501, 481, 519, 529, 491, 503, 526, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[505, 536, 514, 523, 515, 514, 470, 526, 569, 527, 483, 512, 525, 532, 521, 519, 508, 509, 495, 523, 517, 488, 521, 560, 517, 507, 526, 502, 475, 490, 483, 538, 563, 494, 504, 522, 490, 525, 487, 495, 545, 487, 529, 480, 504, 504, 520, 507, 489, 482, 517, 515, 483, 536, 491, 507, 507, 515, 555, 537, 507, 523, 506, 492] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[476, 536, 512, 492, 564, 532, 545, 488, 531, 527, 501, 497, 515, 516, 554, 481, 538, 534, 566, 493, 521, 508, 523, 535, 525, 483, 525, 520, 517, 500, 445, 469, 483, 498, 472, 501, 525, 537, 494, 516, 509, 527, 472, 497, 495, 519, 512, 497, 500, 533, 518, 541, 482, 497, 493, 486, 520, 522, 537, 547, 534, 493, 501, 541] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[531, 500, 542, 530, 523, 522, 513, 488, 0, 0, 0, 0, 0, 0, 0, 0, 506, 532, 523, 477, 530, 543, 497, 497, 0, 0, 0, 0, 0, 0, 0, 0, 519, 545, 485, 536, 494, 464, 525, 469, 0, 0, 0, 0, 0, 0, 0, 0, 529, 487, 488, 533, 510, 522, 525, 538, 24, 8, 0, 0, 0, 0, 0, 0, 462, 522, 520, 481, 519, 539, 506, 495, 2, 0, 0, 0, 0, 0, 0, 0, 510, 507, 505, 502, 528, 546, 528, 488, 19, 7, 0, 0, 0, 0, 0, 0, 471, 528, 537, 478, 529, 491, 508, 491, 39, 0, 0, 0, 0, 0, 0, 0, 493, 475, 532, 514, 541, 531, 497, 472, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4149, 4105, 4037, 4164, 4046, 4140, 4072, 4055] avg=4096.0000 max=4164 max_rank=3 min=4037 min_rank=2 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16313 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[550, 526, 502, 479, 485, 492, 517, 505, 0, 0, 0, 0, 0, 0, 0, 0, 491, 500, 498, 485, 511, 510, 498, 480, 0, 0, 0, 0, 0, 0, 0, 0, 512, 479, 492, 509, 495, 526, 522, 552, 0, 0, 0, 0, 0, 0, 0, 0, 511, 531, 541, 490, 504, 518, 527, 507, 23, 8, 0, 0, 0, 0, 0, 0, 515, 529, 513, 523, 517, 522, 527, 544, 2, 0, 0, 0, 0, 0, 0, 0, 525, 524, 504, 488, 534, 543, 518, 506, 18, 6, 0, 0, 0, 0, 0, 0, 498, 509, 527, 491, 492, 523, 511, 492, 40, 0, 0, 0, 0, 0, 0, 0, 546, 510, 473, 487, 547, 525, 483, 480, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4056, 3973, 4087, 4160, 4192, 4166, 4083, 4051] avg=4096.0000 max=4192 max_rank=4 min=3973 min_rank=1 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16492 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[496, 505, 521, 525, 495, 505, 482, 552, 0, 0, 0, 0, 0, 0, 0, 0, 509, 539, 509, 543, 498, 508, 512, 523, 0, 0, 0, 0, 0, 0, 0, 0, 541, 527, 495, 516, 495, 496, 469, 480, 0, 0, 0, 0, 0, 0, 0, 0, 512, 485, 540, 513, 512, 501, 516, 487, 23, 8, 0, 0, 0, 0, 0, 0, 544, 527, 478, 508, 525, 528, 547, 519, 2, 0, 0, 0, 0, 0, 0, 0, 513, 519, 454, 530, 517, 501, 475, 505, 17, 7, 0, 0, 0, 0, 0, 0, 504, 501, 509, 510, 513, 512, 511, 478, 40, 0, 0, 0, 0, 0, 0, 0, 501, 525, 522, 497, 471, 550, 540, 530, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4141, 4019, 4097, 4178, 4038, 4078, 4136] avg=4096.0000 max=4178 max_rank=4 min=4019 min_rank=2 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16430 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[505, 493, 558, 500, 504, 521, 493, 518, 0, 0, 0, 0, 0, 0, 0, 0, 525, 483, 522, 545, 502, 515, 503, 526, 0, 0, 0, 0, 0, 0, 0, 0, 531, 499, 517, 502, 518, 508, 551, 541, 0, 0, 0, 0, 0, 0, 0, 0, 484, 507, 498, 497, 535, 479, 531, 513, 24, 9, 0, 0, 0, 0, 0, 0, 480, 533, 503, 497, 511, 530, 504, 506, 2, 0, 0, 0, 0, 0, 0, 0, 485, 485, 538, 514, 520, 534, 493, 484, 18, 6, 0, 0, 0, 0, 0, 0, 504, 546, 532, 497, 516, 531, 512, 499, 42, 0, 0, 0, 0, 0, 0, 0, 523, 488, 491, 469, 524, 492, 474, 528, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4121, 4167, 4077, 4066, 4077, 4179, 3989] avg=4096.0000 max=4179 max_rank=6 min=3989 min_rank=7 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16311 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[471, 518, 519, 510, 489, 548, 512, 482, 0, 0, 0, 0, 0, 0, 0, 0, 529, 502, 512, 525, 501, 471, 515, 491, 0, 0, 0, 0, 0, 0, 0, 0, 517, 505, 481, 521, 539, 499, 527, 563, 0, 0, 0, 0, 0, 0, 0, 0, 553, 499, 519, 509, 527, 484, 509, 468, 23, 8, 0, 0, 0, 0, 0, 0, 487, 503, 509, 487, 479, 516, 503, 541, 2, 0, 0, 0, 0, 0, 0, 0, 519, 517, 533, 515, 488, 537, 485, 504, 17, 6, 0, 0, 0, 0, 0, 0, 496, 517, 513, 513, 500, 521, 493, 513, 39, 0, 0, 0, 0, 0, 0, 0, 506, 520, 481, 526, 535, 533, 505, 563, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4049, 4046, 4152, 4099, 4027, 4121, 4105, 4169] avg=4096.0000 max=4169 max_rank=7 min=4027 min_rank=4 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16422 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[505, 536, 514, 523, 484, 514, 470, 526, 0, 0, 0, 0, 0, 0, 0, 0, 569, 527, 483, 512, 503, 532, 521, 519, 0, 0, 0, 0, 0, 0, 0, 0, 508, 509, 442, 523, 517, 488, 521, 560, 0, 0, 0, 0, 0, 0, 0, 0, 517, 507, 526, 502, 475, 490, 483, 538, 26, 11, 0, 0, 0, 0, 0, 0, 563, 494, 504, 522, 490, 525, 487, 495, 9, 0, 0, 0, 0, 0, 0, 0, 545, 487, 529, 480, 504, 504, 520, 507, 22, 12, 5, 0, 0, 0, 0, 0, 489, 482, 517, 515, 483, 536, 491, 507, 42, 0, 0, 0, 0, 0, 0, 0, 507, 515, 534, 537, 507, 523, 506, 492, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4166, 4068, 4075, 4089, 4115, 4062, 4121] avg=4096.0000 max=4166 max_rank=1 min=4062 min_rank=6 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16387 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[544, 525, 486, 539, 483, 549, 496, 527, 0, 0, 0, 0, 0, 0, 0, 0, 503, 533, 519, 527, 498, 505, 511, 515, 0, 0, 0, 0, 0, 0, 0, 0, 537, 514, 450, 517, 522, 500, 556, 475, 0, 0, 0, 0, 0, 0, 0, 0, 486, 492, 526, 477, 509, 492, 521, 545, 23, 7, 0, 0, 0, 0, 0, 0, 521, 527, 523, 520, 495, 533, 516, 505, 2, 0, 0, 0, 0, 0, 0, 0, 523, 496, 470, 516, 519, 517, 503, 491, 17, 7, 0, 0, 0, 0, 0, 0, 493, 497, 516, 487, 476, 558, 528, 506, 36, 0, 0, 0, 0, 0, 0, 0, 501, 481, 510, 529, 491, 503, 526, 520, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4149, 4111, 4071, 4078, 4142, 4059, 4097, 4061] avg=4096.0000 max=4149 max_rank=0 min=4059 min_rank=5 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16359 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[476, 536, 512, 492, 539, 532, 545, 488, 0, 0, 0, 0, 0, 0, 0, 0, 531, 527, 501, 497, 498, 516, 554, 481, 0, 0, 0, 0, 0, 0, 0, 0, 538, 534, 515, 493, 521, 508, 523, 535, 0, 0, 0, 0, 0, 0, 0, 0, 525, 483, 525, 520, 517, 500, 445, 469, 25, 9, 0, 0, 0, 0, 0, 0, 483, 498, 472, 501, 525, 537, 494, 516, 2, 0, 0, 0, 0, 0, 0, 0, 509, 527, 472, 497, 495, 519, 512, 497, 17, 7, 0, 0, 0, 0, 0, 0, 500, 533, 518, 541, 482, 497, 493, 486, 42, 0, 0, 0, 0, 0, 0, 0, 520, 522, 528, 547, 534, 493, 501, 541, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4105, 4167, 4018, 4028, 4052, 4092, 4186] avg=4096.0000 max=4186 max_rank=7 min=4018 min_rank=3 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16358 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4352, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 3968, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3968, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[511, 500, 533, 535, 517, 530, 485, 521, 489, 504, 488, 534, 514, 482, 468, 512, 511, 517, 531, 523, 521, 541, 527, 507, 461, 505, 516, 547, 555, 508, 537, 499, 529, 532, 543, 546, 539, 508, 517, 512, 485, 511, 507, 472, 559, 459, 523, 483, 522, 485, 523, 521, 503, 495, 508, 509, 537, 474, 496, 492, 517, 480, 522, 530] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[530, 503, 498, 534, 532, 523, 518, 509, 544, 505, 499, 509, 517, 516, 498, 473, 536, 486, 540, 493, 482, 501, 512, 502, 517, 509, 517, 552, 520, 480, 509, 505, 524, 507, 495, 529, 486, 512, 466, 495, 555, 500, 541, 508, 490, 529, 503, 501, 511, 502, 477, 493, 520, 533, 513, 514, 491, 495, 543, 534, 543, 549, 518, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[494, 498, 478, 499, 467, 501, 459, 519, 527, 523, 548, 477, 540, 526, 516, 507, 510, 539, 532, 527, 525, 484, 520, 502, 481, 542, 476, 494, 485, 516, 536, 522, 509, 495, 527, 508, 513, 481, 527, 484, 524, 522, 566, 527, 493, 522, 512, 522, 527, 529, 531, 489, 529, 578, 488, 481, 536, 543, 501, 521, 513, 517, 507, 476] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[521, 503, 529, 480, 500, 467, 505, 505, 500, 529, 531, 515, 488, 530, 543, 499, 499, 513, 524, 506, 527, 517, 492, 530, 554, 545, 507, 498, 498, 549, 520, 523, 544, 454, 477, 506, 487, 504, 517, 481, 518, 508, 504, 521, 524, 530, 470, 519, 501, 495, 492, 527, 513, 491, 496, 555, 541, 520, 529, 491, 527, 561, 491, 527] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[528, 502, 507, 529, 479, 507, 478, 484, 518, 499, 544, 492, 509, 511, 547, 472, 532, 543, 476, 498, 472, 505, 518, 496, 513, 521, 535, 506, 515, 466, 493, 521, 526, 539, 519, 519, 495, 507, 497, 532, 528, 534, 528, 511, 488, 527, 498, 535, 526, 471, 530, 522, 514, 514, 508, 547, 500, 511, 502, 497, 515, 528, 536, 548] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[497, 482, 533, 515, 500, 514, 525, 538, 576, 500, 511, 512, 481, 520, 462, 521, 480, 504, 549, 511, 500, 523, 519, 525, 534, 487, 539, 502, 525, 519, 526, 525, 527, 532, 542, 547, 461, 526, 539, 466, 488, 517, 545, 530, 478, 515, 476, 446, 490, 515, 536, 496, 536, 529, 489, 500, 521, 501, 537, 515, 512, 509, 485, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[484, 494, 499, 542, 508, 495, 528, 527, 511, 485, 515, 515, 521, 495, 535, 484, 526, 545, 543, 522, 506, 517, 533, 501, 518, 518, 516, 501, 514, 535, 532, 511, 505, 512, 519, 515, 512, 520, 527, 522, 517, 518, 517, 534, 483, 525, 459, 490, 523, 515, 490, 503, 536, 501, 509, 484, 503, 519, 509, 437, 487, 529, 550, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[551, 526, 479, 487, 520, 495, 496, 549, 516, 509, 516, 514, 491, 525, 530, 502, 571, 494, 495, 506, 510, 489, 510, 537, 509, 523, 491, 469, 513, 533, 526, 513, 508, 552, 506, 467, 534, 529, 529, 478, 497, 527, 517, 526, 548, 518, 499, 515, 537, 523, 501, 480, 488, 484, 541, 534, 514, 516, 491, 468, 529, 492, 497, 528] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[530, 503, 498, 534, 532, 523, 518, 509, 21, 12, 0, 0, 0, 0, 0, 0, 544, 505, 499, 509, 517, 516, 498, 473, 7, 1, 0, 0, 0, 0, 0, 0, 536, 486, 519, 493, 482, 501, 512, 502, 0, 0, 0, 0, 0, 0, 0, 0, 517, 509, 517, 552, 520, 480, 490, 505, 0, 0, 0, 0, 0, 0, 0, 0, 524, 507, 495, 529, 486, 512, 466, 495, 0, 0, 0, 0, 0, 0, 0, 0, 555, 500, 541, 508, 490, 529, 503, 501, 3, 0, 0, 0, 0, 0, 0, 0, 511, 502, 477, 493, 520, 533, 513, 514, 5, 0, 0, 0, 0, 0, 0, 0, 491, 495, 543, 534, 543, 540, 518, 522, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4180, 4069, 4031, 4090, 4014, 4130, 4068, 4186] avg=4096.0000 max=4186 max_rank=7 min=4014 min_rank=4 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[521, 503, 529, 480, 500, 467, 505, 505, 20, 12, 0, 0, 0, 0, 0, 0, 500, 529, 531, 515, 488, 530, 543, 499, 7, 2, 0, 0, 0, 0, 0, 0, 499, 513, 504, 506, 527, 517, 492, 530, 0, 0, 0, 0, 0, 0, 0, 0, 554, 545, 507, 498, 498, 549, 501, 523, 0, 0, 0, 0, 0, 0, 0, 0, 544, 454, 477, 506, 487, 504, 517, 481, 0, 0, 0, 0, 0, 0, 0, 0, 518, 508, 504, 521, 524, 530, 470, 519, 3, 0, 0, 0, 0, 0, 0, 0, 501, 495, 492, 527, 513, 491, 496, 555, 6, 0, 0, 0, 0, 0, 0, 0, 541, 520, 529, 491, 527, 550, 491, 527, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4042, 4144, 4088, 4175, 3970, 4097, 4076, 4176] avg=4096.0000 max=4176 max_rank=7 min=3970 min_rank=4 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16319 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[511, 500, 533, 535, 517, 530, 485, 521, 20, 12, 0, 0, 0, 0, 0, 0, 489, 504, 488, 534, 514, 482, 468, 512, 8, 1, 0, 0, 0, 0, 0, 0, 511, 517, 511, 523, 521, 541, 527, 507, 0, 0, 0, 0, 0, 0, 0, 0, 461, 505, 516, 547, 555, 508, 517, 499, 0, 0, 0, 0, 0, 0, 0, 0, 529, 532, 543, 546, 539, 508, 517, 512, 0, 0, 0, 0, 0, 0, 0, 0, 485, 511, 507, 472, 559, 459, 523, 483, 2, 0, 0, 0, 0, 0, 0, 0, 522, 485, 523, 521, 503, 495, 508, 509, 5, 0, 0, 0, 0, 0, 0, 0, 537, 474, 496, 492, 517, 472, 522, 530, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4164, 4000, 4158, 4108, 4226, 4001, 4071, 4040] avg=4096.0000 max=4226 max_rank=4 min=4000 min_rank=1 imbalance_ratio=1.031738 max_load_reduction=28542 max_load_reduction_pct=0.871033 moved_to_echo=16338 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[497, 482, 533, 515, 500, 514, 525, 538, 21, 12, 0, 0, 0, 0, 0, 0, 576, 500, 511, 512, 481, 520, 462, 521, 7, 1, 0, 0, 0, 0, 0, 0, 480, 504, 528, 511, 500, 523, 519, 525, 0, 0, 0, 0, 0, 0, 0, 0, 534, 487, 539, 502, 525, 519, 507, 525, 0, 0, 0, 0, 0, 0, 0, 0, 527, 532, 542, 547, 461, 526, 539, 466, 0, 0, 0, 0, 0, 0, 0, 0, 488, 517, 545, 530, 478, 515, 476, 446, 3, 0, 0, 0, 0, 0, 0, 0, 490, 515, 536, 496, 536, 529, 489, 500, 5, 0, 0, 0, 0, 0, 0, 0, 521, 501, 537, 515, 512, 500, 485, 507, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4137, 4091, 4090, 4138, 4140, 3998, 4096, 4078] avg=4096.0000 max=4140 max_rank=4 min=3998 min_rank=5 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16312 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[494, 498, 478, 499, 467, 501, 459, 519, 20, 12, 0, 0, 0, 0, 0, 0, 527, 523, 548, 477, 540, 526, 516, 507, 8, 1, 0, 0, 0, 0, 0, 0, 510, 539, 512, 527, 525, 484, 520, 502, 0, 0, 0, 0, 0, 0, 0, 0, 481, 542, 476, 494, 485, 516, 516, 522, 0, 0, 0, 0, 0, 0, 0, 0, 509, 495, 527, 508, 513, 481, 527, 484, 0, 0, 0, 0, 0, 0, 0, 0, 524, 522, 566, 527, 493, 522, 512, 522, 3, 0, 0, 0, 0, 0, 0, 0, 527, 529, 531, 489, 529, 578, 488, 481, 5, 0, 0, 0, 0, 0, 0, 0, 536, 543, 501, 521, 513, 508, 507, 476, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3947, 4173, 4119, 4032, 4044, 4191, 4157, 4105] avg=4096.0000 max=4191 max_rank=5 min=3947 min_rank=0 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16497 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[528, 502, 507, 529, 479, 507, 478, 484, 18, 11, 0, 0, 0, 0, 0, 0, 518, 499, 544, 492, 509, 511, 547, 472, 7, 1, 0, 0, 0, 0, 0, 0, 532, 543, 458, 498, 472, 505, 518, 496, 0, 0, 0, 0, 0, 0, 0, 0, 513, 521, 535, 506, 515, 466, 475, 521, 0, 0, 0, 0, 0, 0, 0, 0, 526, 539, 519, 519, 495, 507, 497, 532, 0, 0, 0, 0, 0, 0, 0, 0, 528, 534, 528, 511, 488, 527, 498, 535, 3, 0, 0, 0, 0, 0, 0, 0, 526, 471, 530, 522, 514, 514, 508, 547, 5, 0, 0, 0, 0, 0, 0, 0, 500, 511, 502, 497, 515, 519, 536, 548, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4043, 4100, 4022, 4052, 4134, 4152, 4137, 4128] avg=4096.0000 max=4152 max_rank=5 min=4022 min_rank=2 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16551 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[484, 494, 499, 542, 508, 495, 528, 527, 26, 18, 0, 0, 0, 0, 0, 0, 511, 485, 515, 515, 521, 495, 535, 484, 12, 7, 0, 0, 0, 0, 0, 0, 526, 545, 517, 522, 506, 517, 533, 501, 0, 0, 0, 0, 0, 0, 0, 0, 518, 518, 516, 501, 514, 535, 502, 511, 0, 0, 0, 0, 0, 0, 0, 0, 505, 512, 519, 515, 512, 520, 527, 522, 5, 0, 0, 0, 0, 0, 0, 0, 517, 518, 517, 534, 483, 525, 459, 490, 6, 0, 0, 0, 0, 0, 0, 0, 523, 515, 490, 503, 536, 501, 509, 484, 9, 0, 0, 0, 0, 0, 0, 0, 503, 519, 509, 437, 487, 502, 550, 522, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4080, 4167, 4115, 4137, 4049, 4070, 4029] avg=4096.0000 max=4167 max_rank=2 min=4029 min_rank=7 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16285 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[551, 526, 479, 487, 520, 495, 496, 549, 19, 12, 0, 0, 0, 0, 0, 0, 516, 509, 516, 514, 491, 525, 530, 502, 7, 1, 0, 0, 0, 0, 0, 0, 571, 494, 476, 506, 510, 489, 510, 537, 0, 0, 0, 0, 0, 0, 0, 0, 509, 523, 491, 469, 513, 533, 507, 513, 0, 0, 0, 0, 0, 0, 0, 0, 508, 552, 506, 467, 534, 529, 529, 478, 0, 0, 0, 0, 0, 0, 0, 0, 497, 527, 517, 526, 548, 518, 499, 515, 3, 0, 0, 0, 0, 0, 0, 0, 537, 523, 501, 480, 488, 484, 541, 534, 5, 0, 0, 0, 0, 0, 0, 0, 514, 516, 491, 468, 529, 483, 497, 528, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4134, 4111, 4093, 4058, 4103, 4150, 4093, 4026] avg=4096.0000 max=4150 max_rank=5 min=4026 min_rank=7 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 3968, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4352, 4224, 4096, 4224, 3968, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[525, 498, 504, 528, 500, 501, 529, 476, 505, 539, 515, 540, 502, 494, 510, 492, 509, 508, 515, 516, 496, 483, 517, 538, 506, 511, 476, 521, 547, 535, 551, 517, 519, 492, 522, 475, 548, 472, 539, 520, 523, 508, 546, 523, 489, 528, 480, 497, 526, 493, 504, 551, 490, 525, 502, 546, 485, 520, 514, 513, 499, 504, 503, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[495, 525, 494, 489, 509, 521, 513, 537, 482, 537, 527, 530, 502, 535, 493, 531, 484, 487, 527, 532, 522, 518, 529, 519, 493, 517, 502, 498, 529, 487, 522, 538, 524, 511, 508, 507, 508, 500, 497, 476, 541, 549, 512, 501, 492, 537, 500, 527, 449, 543, 513, 514, 468, 524, 494, 524, 516, 505, 501, 491, 553, 532, 496, 531] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[493, 546, 562, 533, 521, 474, 549, 520, 530, 516, 506, 528, 521, 485, 486, 561, 495, 535, 518, 499, 525, 468, 517, 508, 498, 457, 482, 518, 517, 511, 492, 516, 518, 547, 514, 535, 487, 510, 508, 508, 518, 536, 518, 536, 507, 511, 498, 498, 504, 497, 512, 546, 498, 480, 511, 522, 537, 472, 492, 494, 506, 512, 532, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[478, 450, 530, 489, 516, 499, 495, 529, 532, 505, 545, 505, 510, 519, 513, 465, 485, 539, 519, 528, 505, 506, 465, 479, 510, 535, 514, 556, 537, 487, 476, 515, 549, 514, 496, 519, 501, 509, 566, 538, 536, 508, 490, 498, 505, 529, 534, 517, 524, 484, 551, 511, 492, 530, 539, 510, 530, 504, 514, 499, 498, 503, 540, 494] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[518, 490, 517, 479, 533, 513, 493, 541, 539, 507, 497, 502, 501, 524, 543, 483, 535, 490, 498, 484, 476, 504, 573, 531, 521, 526, 517, 508, 511, 550, 493, 558, 511, 501, 484, 533, 524, 497, 514, 544, 503, 486, 482, 461, 519, 522, 491, 482, 492, 504, 497, 501, 534, 491, 555, 514, 510, 562, 509, 527, 524, 529, 491, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[491, 520, 527, 527, 498, 468, 492, 516, 530, 500, 522, 518, 482, 550, 540, 513, 501, 530, 494, 493, 508, 519, 494, 520, 476, 486, 515, 487, 534, 487, 526, 489, 521, 533, 519, 503, 513, 524, 485, 522, 520, 551, 537, 481, 520, 519, 486, 488, 502, 513, 499, 531, 537, 541, 516, 548, 516, 516, 547, 484, 519, 502, 540, 482] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[540, 506, 516, 542, 522, 526, 519, 515, 510, 497, 519, 493, 453, 535, 526, 509, 501, 450, 533, 514, 499, 502, 516, 502, 518, 522, 512, 486, 537, 529, 489, 535, 515, 509, 517, 501, 482, 533, 566, 480, 514, 514, 540, 477, 501, 516, 484, 543, 513, 493, 494, 526, 514, 523, 542, 475, 524, 497, 513, 482, 560, 531, 507, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[508, 551, 533, 500, 519, 500, 526, 508, 532, 500, 487, 547, 521, 500, 517, 525, 541, 492, 497, 545, 540, 490, 485, 494, 503, 527, 468, 498, 498, 486, 488, 528, 497, 522, 513, 536, 530, 503, 510, 541, 500, 513, 540, 529, 529, 498, 502, 542, 530, 505, 513, 497, 503, 466, 523, 528, 477, 488, 481, 543, 511, 506, 515, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[495, 525, 493, 489, 509, 521, 513, 537, 0, 0, 0, 0, 0, 0, 0, 0, 482, 537, 527, 516, 502, 535, 493, 531, 0, 0, 0, 0, 0, 0, 0, 0, 484, 487, 527, 532, 522, 518, 529, 519, 18, 9, 0, 0, 0, 0, 0, 0, 493, 517, 502, 498, 529, 487, 522, 538, 5, 3, 1, 0, 0, 0, 0, 0, 524, 511, 508, 507, 508, 500, 479, 476, 0, 0, 0, 0, 0, 0, 0, 0, 541, 548, 512, 501, 492, 537, 500, 527, 0, 0, 0, 0, 0, 0, 0, 0, 449, 543, 513, 514, 468, 524, 491, 524, 0, 0, 0, 0, 0, 0, 0, 0, 516, 505, 501, 491, 553, 532, 496, 531, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4123, 4145, 4095, 4013, 4158, 4026, 4126] avg=4096.0000 max=4158 max_rank=5 min=4013 min_rank=4 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16323 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[525, 498, 503, 528, 500, 501, 529, 476, 0, 0, 0, 0, 0, 0, 0, 0, 505, 539, 515, 526, 502, 494, 510, 492, 0, 0, 0, 0, 0, 0, 0, 0, 509, 508, 515, 516, 496, 483, 517, 538, 19, 9, 0, 0, 0, 0, 0, 0, 506, 511, 476, 521, 547, 535, 551, 517, 5, 3, 1, 0, 0, 0, 0, 0, 519, 492, 522, 475, 548, 472, 520, 520, 0, 0, 0, 0, 0, 0, 0, 0, 523, 507, 546, 523, 489, 528, 480, 497, 0, 0, 0, 0, 0, 0, 0, 0, 526, 493, 504, 551, 490, 525, 499, 546, 0, 0, 0, 0, 0, 0, 0, 0, 485, 520, 514, 513, 499, 504, 503, 508, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4083, 4110, 4173, 4068, 4093, 4134, 4047] avg=4096.0000 max=4173 max_rank=3 min=4047 min_rank=7 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16342 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[478, 450, 529, 489, 516, 499, 495, 529, 0, 0, 0, 0, 0, 0, 0, 0, 532, 505, 545, 493, 510, 519, 513, 465, 0, 0, 0, 0, 0, 0, 0, 0, 485, 539, 519, 528, 505, 506, 465, 479, 20, 8, 0, 0, 0, 0, 0, 0, 510, 535, 514, 556, 537, 487, 476, 515, 4, 3, 1, 0, 0, 0, 0, 0, 549, 514, 496, 519, 501, 509, 546, 538, 0, 0, 0, 0, 0, 0, 0, 0, 536, 507, 490, 498, 505, 529, 534, 517, 0, 0, 0, 0, 0, 0, 0, 0, 524, 484, 551, 511, 492, 530, 536, 510, 0, 0, 0, 0, 0, 0, 0, 0, 530, 504, 514, 499, 498, 503, 540, 494, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3985, 4082, 4054, 4138, 4172, 4116, 4138, 4083] avg=4096.0000 max=4172 max_rank=4 min=3985 min_rank=0 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16509 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[493, 546, 555, 533, 521, 474, 549, 520, 0, 0, 0, 0, 0, 0, 0, 0, 530, 516, 506, 505, 521, 485, 486, 561, 0, 0, 0, 0, 0, 0, 0, 0, 495, 535, 518, 499, 525, 468, 517, 508, 22, 14, 0, 0, 0, 0, 0, 0, 498, 457, 482, 518, 517, 511, 492, 516, 9, 8, 7, 5, 0, 0, 0, 0, 518, 547, 514, 535, 487, 510, 486, 508, 0, 0, 0, 0, 0, 0, 0, 0, 518, 529, 518, 536, 507, 511, 498, 498, 0, 0, 0, 0, 0, 0, 0, 0, 504, 497, 512, 546, 498, 480, 503, 522, 0, 0, 0, 0, 0, 0, 0, 0, 537, 472, 492, 494, 506, 512, 532, 517, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4191, 4110, 4101, 4020, 4105, 4115, 4062, 4064] avg=4096.0000 max=4191 max_rank=0 min=4020 min_rank=3 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16346 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[518, 490, 516, 479, 533, 513, 493, 541, 0, 0, 0, 0, 0, 0, 0, 0, 539, 507, 497, 490, 501, 524, 543, 483, 0, 0, 0, 0, 0, 0, 0, 0, 535, 490, 498, 484, 476, 504, 573, 531, 18, 8, 0, 0, 0, 0, 0, 0, 521, 526, 517, 508, 511, 550, 493, 558, 4, 3, 1, 0, 0, 0, 0, 0, 511, 501, 484, 533, 524, 497, 496, 544, 0, 0, 0, 0, 0, 0, 0, 0, 503, 485, 482, 461, 519, 522, 491, 482, 0, 0, 0, 0, 0, 0, 0, 0, 492, 504, 497, 501, 534, 491, 552, 514, 0, 0, 0, 0, 0, 0, 0, 0, 510, 562, 509, 527, 524, 529, 491, 519, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4084, 4117, 4192, 4090, 3945, 4085, 4172] avg=4096.0000 max=4192 max_rank=3 min=3945 min_rank=5 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16292 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[540, 506, 515, 542, 522, 526, 519, 515, 0, 0, 0, 0, 0, 0, 0, 0, 510, 497, 519, 481, 453, 535, 526, 509, 0, 0, 0, 0, 0, 0, 0, 0, 501, 450, 533, 514, 499, 502, 516, 502, 20, 8, 0, 0, 0, 0, 0, 0, 518, 522, 512, 486, 537, 529, 489, 535, 4, 3, 1, 0, 0, 0, 0, 0, 515, 509, 517, 501, 482, 533, 546, 480, 0, 0, 0, 0, 0, 0, 0, 0, 514, 513, 540, 477, 501, 516, 484, 543, 0, 0, 0, 0, 0, 0, 0, 0, 513, 493, 494, 526, 514, 523, 539, 475, 0, 0, 0, 0, 0, 0, 0, 0, 524, 497, 513, 482, 560, 531, 507, 509, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4185, 4030, 4045, 4136, 4083, 4088, 4077, 4124] avg=4096.0000 max=4185 max_rank=0 min=4030 min_rank=1 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[491, 520, 526, 527, 498, 468, 492, 516, 0, 0, 0, 0, 0, 0, 0, 0, 530, 500, 522, 504, 482, 550, 540, 513, 0, 0, 0, 0, 0, 0, 0, 0, 501, 530, 494, 493, 508, 519, 494, 520, 17, 9, 0, 0, 0, 0, 0, 0, 476, 486, 515, 487, 534, 487, 526, 489, 5, 3, 1, 0, 0, 0, 0, 0, 521, 533, 519, 503, 513, 524, 468, 522, 0, 0, 0, 0, 0, 0, 0, 0, 520, 550, 537, 481, 520, 519, 486, 488, 0, 0, 0, 0, 0, 0, 0, 0, 502, 513, 499, 531, 537, 541, 513, 548, 0, 0, 0, 0, 0, 0, 0, 0, 516, 516, 547, 484, 519, 502, 540, 482, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4038, 4141, 4085, 4009, 4103, 4101, 4184, 4107] avg=4096.0000 max=4184 max_rank=6 min=4009 min_rank=3 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16495 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[508, 551, 532, 500, 519, 500, 526, 508, 0, 0, 0, 0, 0, 0, 0, 0, 532, 500, 487, 533, 521, 500, 517, 525, 0, 0, 0, 0, 0, 0, 0, 0, 541, 492, 497, 545, 540, 490, 485, 494, 18, 9, 0, 0, 0, 0, 0, 0, 503, 527, 468, 498, 498, 486, 488, 528, 5, 3, 1, 0, 0, 0, 0, 0, 497, 522, 513, 536, 530, 503, 492, 541, 0, 0, 0, 0, 0, 0, 0, 0, 500, 512, 540, 529, 529, 498, 502, 542, 0, 0, 0, 0, 0, 0, 0, 0, 530, 505, 513, 497, 503, 466, 520, 528, 0, 0, 0, 0, 0, 0, 0, 0, 477, 488, 481, 543, 511, 506, 515, 523, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4144, 4115, 4111, 4005, 4134, 4152, 4062, 4045] avg=4096.0000 max=4152 max_rank=5 min=4005 min_rank=3 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 128, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[510, 511, 515, 536, 515, 502, 534, 515, 523, 477, 535, 523, 487, 510, 513, 516, 519, 537, 525, 491, 514, 482, 555, 553, 502, 522, 531, 505, 515, 511, 499, 516, 529, 492, 449, 508, 462, 527, 526, 509, 527, 459, 481, 525, 511, 523, 535, 516, 499, 528, 493, 504, 522, 509, 553, 528, 533, 512, 511, 489, 533, 487, 507, 482] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[490, 532, 531, 504, 552, 498, 514, 523, 529, 523, 507, 522, 518, 522, 538, 486, 524, 547, 521, 484, 517, 516, 537, 537, 524, 480, 514, 483, 509, 493, 473, 490, 528, 505, 471, 514, 463, 525, 490, 452, 503, 563, 545, 483, 496, 508, 513, 495, 499, 521, 522, 528, 507, 495, 489, 523, 509, 529, 488, 568, 535, 509, 528, 526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[520, 526, 506, 523, 503, 531, 521, 496, 477, 485, 561, 499, 526, 498, 558, 520, 545, 490, 512, 528, 496, 508, 535, 503, 481, 527, 524, 506, 504, 510, 509, 483, 539, 478, 511, 500, 481, 541, 507, 531, 525, 523, 510, 514, 551, 520, 481, 487, 499, 503, 518, 489, 490, 479, 547, 504, 499, 557, 470, 530, 498, 517, 517, 541] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[512, 535, 505, 483, 536, 504, 498, 535, 528, 514, 487, 528, 511, 502, 520, 501, 534, 500, 496, 532, 511, 519, 488, 479, 493, 485, 529, 485, 541, 530, 499, 513, 520, 479, 541, 507, 546, 495, 499, 495, 491, 543, 533, 545, 507, 506, 525, 522, 537, 510, 457, 534, 524, 507, 487, 512, 533, 528, 516, 479, 515, 528, 520, 494] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[498, 482, 522, 516, 510, 541, 500, 496, 540, 528, 501, 505, 509, 528, 535, 494, 502, 521, 538, 480, 503, 543, 518, 537, 470, 523, 524, 508, 516, 498, 488, 522, 496, 493, 471, 558, 550, 524, 545, 511, 485, 512, 511, 513, 508, 486, 488, 493, 482, 508, 524, 477, 527, 536, 513, 503, 456, 548, 523, 513, 540, 515, 534, 529] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[508, 504, 547, 521, 526, 508, 535, 540, 487, 524, 538, 532, 493, 525, 579, 555, 518, 449, 498, 540, 540, 499, 523, 540, 502, 474, 498, 478, 545, 518, 489, 504, 537, 505, 485, 514, 486, 494, 536, 507, 520, 506, 503, 523, 517, 546, 483, 510, 470, 507, 542, 515, 499, 514, 530, 500, 499, 475, 512, 483, 504, 514, 502, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[530, 521, 494, 513, 483, 541, 523, 515, 535, 533, 487, 496, 497, 513, 507, 512, 547, 504, 512, 524, 553, 489, 566, 481, 520, 467, 539, 504, 516, 498, 528, 511, 488, 548, 561, 503, 487, 484, 513, 491, 519, 521, 511, 520, 506, 493, 477, 515, 500, 522, 514, 513, 550, 503, 528, 485, 483, 520, 520, 486, 523, 540, 475, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[502, 536, 535, 487, 492, 522, 497, 510, 508, 494, 546, 535, 546, 505, 496, 491, 533, 492, 510, 469, 505, 506, 496, 495, 540, 488, 528, 513, 500, 491, 512, 555, 505, 483, 496, 522, 517, 542, 523, 523, 558, 517, 529, 511, 523, 505, 517, 501, 541, 480, 491, 531, 509, 506, 501, 496, 532, 482, 525, 520, 514, 513, 494, 526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[510, 511, 490, 536, 515, 502, 534, 515, 0, 0, 0, 0, 0, 0, 0, 0, 523, 477, 535, 523, 487, 510, 483, 516, 0, 0, 0, 0, 0, 0, 0, 0, 496, 537, 525, 491, 514, 482, 555, 553, 0, 0, 0, 0, 0, 0, 0, 0, 502, 522, 531, 505, 515, 511, 499, 516, 30, 8, 0, 0, 0, 0, 0, 0, 529, 492, 449, 508, 462, 527, 526, 509, 17, 13, 0, 0, 0, 0, 0, 0, 527, 457, 481, 525, 511, 523, 535, 516, 0, 0, 0, 0, 0, 0, 0, 0, 499, 528, 493, 504, 522, 509, 553, 528, 10, 2, 2, 0, 0, 0, 0, 0, 533, 512, 511, 489, 531, 487, 507, 482, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4054, 4153, 4139, 4032, 4075, 4150, 4052] avg=4096.0000 max=4153 max_rank=2 min=4032 min_rank=4 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16309 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[490, 532, 506, 504, 552, 498, 514, 523, 0, 0, 0, 0, 0, 0, 0, 0, 529, 523, 507, 522, 518, 522, 507, 486, 0, 0, 0, 0, 0, 0, 0, 0, 501, 547, 521, 484, 517, 516, 537, 537, 0, 0, 0, 0, 0, 0, 0, 0, 524, 480, 514, 483, 509, 493, 473, 490, 31, 8, 0, 0, 0, 0, 0, 0, 528, 505, 471, 514, 463, 525, 490, 452, 17, 13, 0, 0, 0, 0, 0, 0, 503, 560, 545, 483, 496, 508, 513, 495, 0, 0, 0, 0, 0, 0, 0, 0, 499, 521, 522, 528, 507, 495, 489, 523, 10, 3, 2, 0, 0, 0, 0, 0, 509, 529, 488, 568, 533, 509, 528, 526, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4114, 4160, 4005, 3978, 4103, 4099, 4190] avg=4096.0000 max=4190 max_rank=7 min=3978 min_rank=4 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16370 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[498, 482, 497, 516, 510, 541, 500, 496, 0, 0, 0, 0, 0, 0, 0, 0, 540, 528, 501, 505, 509, 528, 504, 494, 0, 0, 0, 0, 0, 0, 0, 0, 479, 521, 538, 480, 503, 543, 518, 537, 0, 0, 0, 0, 0, 0, 0, 0, 470, 523, 524, 508, 516, 498, 488, 522, 31, 8, 0, 0, 0, 0, 0, 0, 496, 493, 471, 558, 550, 524, 545, 511, 17, 13, 0, 0, 0, 0, 0, 0, 485, 509, 511, 513, 508, 486, 488, 493, 0, 0, 0, 0, 0, 0, 0, 0, 482, 508, 524, 477, 527, 536, 513, 503, 10, 3, 2, 0, 0, 0, 0, 0, 456, 548, 523, 513, 538, 515, 534, 529, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4040, 4109, 4119, 4088, 4178, 3993, 4085, 4156] avg=4096.0000 max=4178 max_rank=4 min=3993 min_rank=5 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16412 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[508, 504, 521, 521, 526, 508, 535, 540, 0, 0, 0, 0, 0, 0, 0, 0, 487, 524, 538, 532, 493, 525, 545, 555, 0, 0, 0, 0, 0, 0, 0, 0, 495, 449, 498, 540, 540, 499, 523, 540, 0, 0, 0, 0, 0, 0, 0, 0, 502, 474, 498, 478, 545, 518, 489, 504, 34, 8, 0, 0, 0, 0, 0, 0, 537, 505, 485, 514, 486, 494, 536, 507, 18, 13, 0, 0, 0, 0, 0, 0, 520, 503, 503, 523, 517, 546, 483, 510, 0, 0, 0, 0, 0, 0, 0, 0, 470, 507, 542, 515, 499, 514, 530, 500, 10, 3, 2, 0, 0, 0, 0, 0, 499, 475, 512, 483, 502, 514, 502, 493, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4163, 4199, 4084, 4050, 4095, 4105, 4092, 3980] avg=4096.0000 max=4199 max_rank=1 min=3980 min_rank=7 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16272 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[512, 535, 474, 483, 536, 504, 498, 535, 0, 0, 0, 0, 0, 0, 0, 0, 528, 514, 487, 528, 511, 502, 486, 501, 0, 0, 0, 0, 0, 0, 0, 0, 502, 500, 496, 532, 511, 519, 488, 479, 0, 0, 0, 0, 0, 0, 0, 0, 493, 485, 529, 485, 541, 530, 499, 513, 34, 11, 0, 0, 0, 0, 0, 0, 520, 479, 541, 507, 546, 495, 499, 495, 20, 18, 0, 0, 0, 0, 0, 0, 491, 538, 533, 545, 507, 506, 525, 522, 0, 0, 0, 0, 0, 0, 0, 0, 537, 510, 457, 534, 524, 507, 487, 512, 14, 5, 9, 0, 0, 0, 0, 0, 533, 528, 516, 479, 506, 528, 520, 494, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4057, 4027, 4120, 4120, 4167, 4096, 4104] avg=4096.0000 max=4167 max_rank=5 min=4027 min_rank=2 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16487 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[520, 526, 483, 523, 503, 531, 521, 496, 0, 0, 0, 0, 0, 0, 0, 0, 477, 485, 561, 499, 526, 498, 526, 520, 0, 0, 0, 0, 0, 0, 0, 0, 520, 490, 512, 528, 496, 508, 535, 503, 0, 0, 0, 0, 0, 0, 0, 0, 481, 527, 524, 506, 504, 510, 509, 483, 32, 7, 0, 0, 0, 0, 0, 0, 539, 478, 511, 500, 481, 541, 507, 531, 16, 14, 0, 0, 0, 0, 0, 0, 525, 520, 510, 514, 551, 520, 481, 487, 0, 0, 0, 0, 0, 0, 0, 0, 499, 503, 518, 489, 490, 479, 547, 504, 11, 3, 2, 0, 0, 0, 0, 0, 499, 557, 470, 530, 496, 517, 517, 541, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4092, 4092, 4083, 4118, 4108, 4045, 4127] avg=4096.0000 max=4127 max_rank=7 min=4045 min_rank=6 imbalance_ratio=1.007568 max_load_reduction=28641 max_load_reduction_pct=0.874054 moved_to_echo=16398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[530, 521, 471, 513, 483, 541, 523, 515, 0, 0, 0, 0, 0, 0, 0, 0, 535, 533, 487, 496, 497, 513, 478, 512, 0, 0, 0, 0, 0, 0, 0, 0, 522, 504, 512, 524, 553, 489, 566, 481, 0, 0, 0, 0, 0, 0, 0, 0, 520, 467, 539, 504, 516, 498, 528, 511, 29, 7, 0, 0, 0, 0, 0, 0, 488, 548, 561, 503, 487, 484, 513, 491, 16, 14, 0, 0, 0, 0, 0, 0, 519, 518, 511, 520, 506, 493, 477, 515, 0, 0, 0, 0, 0, 0, 0, 0, 500, 522, 514, 513, 550, 503, 528, 485, 11, 3, 2, 0, 0, 0, 0, 0, 483, 520, 520, 486, 521, 540, 475, 510, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4051, 4151, 4119, 4105, 4059, 4131, 4055] avg=4096.0000 max=4151 max_rank=2 min=4051 min_rank=1 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16350 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[502, 536, 510, 487, 492, 522, 497, 510, 0, 0, 0, 0, 0, 0, 0, 0, 508, 494, 546, 535, 546, 505, 467, 491, 0, 0, 0, 0, 0, 0, 0, 0, 509, 492, 510, 469, 505, 506, 496, 495, 0, 0, 0, 0, 0, 0, 0, 0, 540, 488, 528, 513, 500, 491, 512, 555, 29, 8, 0, 0, 0, 0, 0, 0, 505, 483, 496, 522, 517, 542, 523, 523, 17, 14, 0, 0, 0, 0, 0, 0, 558, 514, 529, 511, 523, 505, 517, 501, 0, 0, 0, 0, 0, 0, 0, 0, 541, 480, 491, 531, 509, 506, 501, 496, 10, 3, 2, 0, 0, 0, 0, 0, 532, 482, 525, 520, 512, 513, 494, 526, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4056, 4092, 3982, 4164, 4142, 4158, 4070, 4104] avg=4096.0000 max=4164 max_rank=3 min=3982 min_rank=2 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16474 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 3968, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 3968, 4224, 4096, 4224, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[455, 535, 490, 541, 515, 525, 516, 503, 516, 510, 515, 525, 548, 513, 516, 528, 537, 542, 490, 490, 514, 511, 519, 452, 543, 504, 518, 505, 543, 537, 517, 523, 517, 498, 509, 489, 476, 504, 534, 471, 534, 506, 517, 501, 528, 535, 505, 543, 531, 530, 500, 511, 506, 477, 539, 503, 526, 468, 508, 491, 492, 480, 509, 534] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[526, 521, 481, 510, 508, 475, 520, 552, 509, 491, 517, 521, 544, 496, 563, 497, 506, 507, 497, 508, 486, 520, 511, 553, 504, 506, 475, 510, 528, 504, 520, 506, 500, 513, 501, 514, 508, 522, 455, 507, 518, 535, 529, 532, 517, 540, 556, 517, 480, 518, 540, 502, 484, 497, 488, 541, 529, 488, 506, 524, 510, 490, 515, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[517, 519, 519, 556, 471, 495, 543, 527, 486, 506, 550, 487, 484, 514, 475, 522, 517, 490, 557, 497, 486, 503, 504, 466, 529, 505, 492, 502, 526, 500, 513, 496, 514, 534, 544, 517, 543, 531, 505, 523, 543, 486, 495, 558, 527, 502, 493, 510, 529, 531, 499, 487, 497, 537, 544, 499, 514, 517, 511, 537, 491, 492, 507, 497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[511, 552, 532, 489, 514, 521, 488, 513, 506, 525, 542, 494, 489, 514, 514, 511, 547, 504, 524, 542, 544, 499, 484, 516, 516, 483, 486, 529, 511, 499, 521, 476, 522, 496, 526, 515, 500, 517, 531, 537, 498, 504, 522, 517, 507, 512, 511, 467, 537, 494, 512, 523, 502, 519, 582, 519, 527, 490, 498, 468, 513, 502, 511, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[545, 531, 501, 503, 538, 502, 505, 504, 501, 484, 556, 505, 471, 538, 533, 485, 535, 517, 509, 482, 511, 496, 513, 506, 490, 507, 516, 534, 520, 524, 501, 528, 555, 520, 503, 492, 517, 550, 486, 503, 460, 517, 529, 541, 525, 511, 468, 538, 508, 491, 487, 517, 503, 561, 504, 510, 517, 499, 504, 501, 517, 491, 542, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[515, 486, 525, 565, 511, 504, 505, 553, 543, 538, 512, 513, 510, 501, 489, 523, 548, 507, 486, 514, 516, 477, 517, 530, 525, 528, 509, 539, 512, 472, 500, 501, 529, 481, 540, 511, 511, 542, 503, 519, 483, 492, 521, 508, 513, 476, 495, 555, 498, 484, 518, 518, 480, 487, 516, 516, 540, 515, 502, 478, 546, 520, 527, 470] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[505, 505, 505, 504, 558, 542, 509, 562, 513, 500, 491, 486, 492, 511, 492, 481, 477, 531, 548, 515, 506, 487, 487, 512, 517, 538, 522, 504, 484, 520, 483, 524, 511, 528, 525, 505, 533, 517, 516, 461, 493, 523, 499, 529, 537, 528, 541, 472, 497, 498, 503, 468, 506, 498, 527, 529, 552, 555, 472, 545, 538, 516, 500, 535] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[487, 512, 487, 532, 503, 486, 513, 511, 509, 521, 481, 511, 506, 529, 516, 528, 536, 546, 504, 521, 525, 490, 515, 513, 513, 505, 489, 511, 495, 473, 518, 537, 513, 475, 495, 495, 487, 512, 533, 516, 512, 515, 520, 519, 465, 527, 527, 507, 530, 526, 514, 482, 539, 540, 511, 529, 531, 491, 502, 525, 535, 539, 530, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[455, 535, 490, 541, 515, 525, 516, 482, 0, 0, 0, 0, 0, 0, 0, 0, 516, 510, 515, 525, 548, 513, 516, 528, 8, 0, 0, 0, 0, 0, 0, 0, 537, 542, 490, 490, 514, 511, 519, 452, 6, 1, 0, 0, 0, 0, 0, 0, 543, 504, 518, 505, 543, 537, 517, 523, 12, 0, 0, 0, 0, 0, 0, 0, 517, 498, 509, 489, 476, 504, 534, 471, 1, 0, 0, 0, 0, 0, 0, 0, 534, 506, 517, 486, 528, 535, 505, 543, 0, 0, 0, 0, 0, 0, 0, 0, 531, 530, 500, 511, 506, 477, 539, 503, 1, 0, 0, 0, 0, 0, 0, 0, 526, 468, 508, 491, 492, 480, 509, 534, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4059, 4179, 4062, 4202, 3999, 4154, 4098, 4015] avg=4096.0000 max=4202 max_rank=3 min=3999 min_rank=4 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16266 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[511, 552, 532, 489, 514, 521, 488, 492, 0, 0, 0, 0, 0, 0, 0, 0, 506, 525, 542, 494, 489, 514, 514, 511, 8, 0, 0, 0, 0, 0, 0, 0, 547, 504, 524, 542, 544, 499, 484, 516, 6, 1, 0, 0, 0, 0, 0, 0, 516, 483, 486, 529, 511, 499, 521, 476, 12, 0, 0, 0, 0, 0, 0, 0, 522, 496, 526, 515, 500, 517, 531, 537, 1, 0, 0, 0, 0, 0, 0, 0, 498, 504, 522, 502, 507, 512, 511, 467, 0, 0, 0, 0, 0, 0, 0, 0, 537, 494, 512, 523, 502, 519, 582, 519, 1, 0, 0, 0, 0, 0, 0, 0, 527, 490, 498, 468, 513, 502, 511, 493, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4103, 4167, 4033, 4145, 4023, 4189, 4009] avg=4096.0000 max=4189 max_rank=6 min=4009 min_rank=7 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[526, 521, 481, 510, 508, 475, 520, 529, 0, 0, 0, 0, 0, 0, 0, 0, 509, 491, 517, 521, 544, 496, 563, 497, 9, 0, 0, 0, 0, 0, 0, 0, 506, 507, 497, 508, 486, 520, 511, 553, 6, 1, 0, 0, 0, 0, 0, 0, 504, 506, 475, 510, 528, 504, 520, 506, 13, 0, 0, 0, 0, 0, 0, 0, 500, 513, 501, 514, 508, 522, 455, 507, 1, 0, 0, 0, 0, 0, 0, 0, 518, 535, 529, 517, 517, 540, 556, 517, 0, 0, 0, 0, 0, 0, 0, 0, 480, 518, 540, 502, 484, 497, 488, 541, 1, 0, 0, 0, 0, 0, 0, 0, 529, 488, 506, 524, 510, 490, 515, 520, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4070, 4147, 4095, 4066, 4021, 4229, 4051, 4089] avg=4096.0000 max=4229 max_rank=5 min=4021 min_rank=4 imbalance_ratio=1.032471 max_load_reduction=28539 max_load_reduction_pct=0.870941 moved_to_echo=16390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[505, 505, 505, 504, 558, 542, 509, 539, 0, 0, 0, 0, 0, 0, 0, 0, 513, 500, 491, 486, 492, 511, 492, 481, 9, 0, 0, 0, 0, 0, 0, 0, 477, 531, 548, 515, 506, 487, 487, 512, 6, 1, 0, 0, 0, 0, 0, 0, 517, 538, 522, 504, 484, 520, 483, 524, 13, 0, 0, 0, 0, 0, 0, 0, 511, 528, 525, 505, 533, 517, 516, 461, 1, 0, 0, 0, 0, 0, 0, 0, 493, 523, 499, 514, 537, 528, 541, 472, 0, 0, 0, 0, 0, 0, 0, 0, 497, 498, 503, 468, 506, 498, 527, 529, 1, 0, 0, 0, 0, 0, 0, 0, 552, 555, 472, 545, 538, 516, 500, 535, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4167, 3975, 4070, 4105, 4097, 4107, 4027, 4220] avg=4096.0000 max=4220 max_rank=7 min=3975 min_rank=1 imbalance_ratio=1.030273 max_load_reduction=28548 max_load_reduction_pct=0.871216 moved_to_echo=16451 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[545, 531, 501, 503, 538, 502, 505, 471, 0, 0, 0, 0, 0, 0, 0, 0, 501, 484, 556, 505, 471, 538, 533, 485, 12, 0, 0, 0, 0, 0, 0, 0, 535, 517, 509, 482, 511, 496, 513, 506, 12, 6, 0, 0, 0, 0, 0, 0, 490, 507, 516, 534, 520, 524, 501, 528, 15, 0, 0, 0, 0, 0, 0, 0, 555, 520, 503, 492, 517, 550, 486, 503, 4, 0, 0, 0, 0, 0, 0, 0, 460, 517, 529, 505, 525, 511, 468, 538, 0, 0, 0, 0, 0, 0, 0, 0, 508, 491, 487, 517, 503, 561, 504, 510, 8, 0, 0, 0, 0, 0, 0, 0, 517, 499, 504, 501, 517, 491, 542, 510, 12, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4085, 4087, 4135, 4130, 4053, 4089, 4093] avg=4096.0000 max=4135 max_rank=3 min=4053 min_rank=5 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16365 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[515, 486, 525, 565, 511, 504, 505, 530, 0, 0, 0, 0, 0, 0, 0, 0, 543, 538, 512, 513, 510, 501, 489, 523, 9, 0, 0, 0, 0, 0, 0, 0, 548, 507, 486, 514, 516, 477, 517, 530, 6, 1, 0, 0, 0, 0, 0, 0, 525, 528, 509, 539, 512, 472, 500, 501, 13, 0, 0, 0, 0, 0, 0, 0, 529, 481, 540, 511, 511, 542, 503, 519, 1, 0, 0, 0, 0, 0, 0, 0, 483, 492, 521, 493, 513, 476, 495, 555, 0, 0, 0, 0, 0, 0, 0, 0, 498, 484, 518, 518, 480, 487, 516, 516, 1, 0, 0, 0, 0, 0, 0, 0, 540, 515, 502, 478, 546, 520, 527, 470, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4141, 4138, 4102, 4099, 4137, 4028, 4018, 4105] avg=4096.0000 max=4141 max_rank=0 min=4018 min_rank=6 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16288 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[487, 512, 487, 532, 503, 486, 513, 490, 0, 0, 0, 0, 0, 0, 0, 0, 509, 521, 481, 511, 506, 529, 516, 528, 8, 0, 0, 0, 0, 0, 0, 0, 536, 546, 504, 521, 525, 490, 515, 513, 6, 1, 0, 0, 0, 0, 0, 0, 513, 505, 489, 511, 495, 473, 518, 537, 12, 0, 0, 0, 0, 0, 0, 0, 513, 475, 495, 495, 487, 512, 533, 516, 1, 0, 0, 0, 0, 0, 0, 0, 512, 515, 520, 504, 465, 527, 527, 507, 0, 0, 0, 0, 0, 0, 0, 0, 530, 526, 514, 482, 539, 540, 511, 529, 1, 0, 0, 0, 0, 0, 0, 0, 531, 491, 502, 525, 535, 539, 530, 503, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4010, 4109, 4157, 4053, 4027, 4077, 4172, 4163] avg=4096.0000 max=4172 max_rank=6 min=4010 min_rank=0 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16439 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[517, 519, 519, 556, 471, 495, 543, 506, 0, 0, 0, 0, 0, 0, 0, 0, 486, 506, 550, 487, 484, 514, 475, 522, 8, 0, 0, 0, 0, 0, 0, 0, 517, 490, 557, 497, 486, 503, 504, 466, 7, 1, 0, 0, 0, 0, 0, 0, 529, 505, 492, 502, 526, 500, 513, 496, 12, 0, 0, 0, 0, 0, 0, 0, 514, 534, 544, 517, 543, 531, 505, 523, 1, 0, 0, 0, 0, 0, 0, 0, 543, 486, 495, 541, 527, 502, 493, 510, 0, 0, 0, 0, 0, 0, 0, 0, 529, 531, 499, 487, 497, 537, 544, 499, 1, 0, 0, 0, 0, 0, 0, 0, 514, 517, 511, 537, 491, 492, 507, 497, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4032, 4028, 4075, 4212, 4097, 4124, 4074] avg=4096.0000 max=4212 max_rank=4 min=4028 min_rank=2 imbalance_ratio=1.028320 max_load_reduction=28556 max_load_reduction_pct=0.871460 moved_to_echo=16507 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4352, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True + [2026-09-21 08:26:48] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 605.4 | throughput per GPU (TFLOP/s/GPU): 53.2 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 1.016369E+01 | loss scale: 1.0 | grad norm: 117.673 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[517, 503, 507, 504, 535, 514, 485, 510, 493, 505, 525, 542, 516, 503, 482, 496, 523, 489, 501, 499, 512, 497, 494, 535, 546, 519, 537, 505, 522, 520, 496, 506, 514, 519, 518, 525, 518, 508, 559, 552, 562, 550, 483, 511, 536, 518, 483, 476, 550, 490, 517, 522, 509, 495, 503, 495, 497, 493, 484, 532, 478, 517, 525, 491] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[477, 534, 503, 494, 495, 547, 511, 527, 490, 486, 511, 473, 571, 506, 504, 523, 531, 496, 506, 557, 519, 479, 521, 490, 509, 500, 517, 520, 507, 492, 511, 537, 518, 505, 474, 507, 494, 510, 545, 498, 505, 525, 552, 529, 522, 524, 462, 532, 484, 511, 543, 520, 495, 511, 523, 527, 532, 542, 495, 485, 516, 524, 515, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[516, 529, 530, 532, 484, 542, 483, 508, 555, 507, 515, 538, 500, 514, 506, 502, 510, 482, 538, 505, 491, 518, 526, 504, 541, 530, 536, 514, 541, 506, 484, 478, 532, 519, 520, 518, 551, 499, 510, 512, 500, 460, 513, 523, 474, 503, 511, 521, 496, 495, 549, 494, 473, 500, 501, 497, 524, 550, 506, 531, 502, 512, 479, 528] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[504, 517, 534, 508, 536, 525, 536, 534, 568, 512, 505, 516, 492, 526, 509, 509, 503, 477, 481, 533, 503, 477, 523, 515, 469, 506, 535, 507, 524, 532, 478, 546, 497, 478, 526, 522, 492, 492, 511, 503, 536, 521, 541, 528, 479, 504, 485, 528, 484, 525, 528, 523, 494, 502, 536, 514, 508, 497, 513, 502, 511, 494, 539, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[509, 532, 496, 526, 485, 503, 522, 504, 512, 526, 544, 526, 537, 521, 498, 568, 516, 484, 490, 484, 481, 539, 520, 501, 529, 495, 494, 507, 555, 500, 502, 505, 514, 478, 489, 487, 486, 528, 534, 486, 508, 546, 534, 508, 522, 500, 477, 522, 535, 535, 501, 529, 467, 519, 564, 524, 468, 498, 503, 527, 509, 544, 495, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[508, 486, 513, 502, 519, 536, 497, 522, 488, 512, 515, 513, 514, 523, 547, 496, 510, 523, 540, 502, 504, 493, 536, 519, 570, 525, 485, 504, 515, 490, 529, 461, 494, 516, 521, 518, 560, 538, 522, 532, 531, 501, 525, 527, 530, 547, 546, 473, 509, 539, 507, 447, 483, 469, 481, 554, 469, 507, 514, 522, 505, 475, 520, 489] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[504, 508, 526, 511, 521, 512, 504, 497, 498, 472, 503, 526, 525, 509, 494, 548, 522, 498, 498, 466, 521, 480, 496, 503, 560, 544, 531, 503, 502, 526, 520, 513, 513, 503, 499, 475, 517, 530, 513, 519, 485, 459, 583, 539, 503, 528, 542, 490, 491, 512, 461, 502, 498, 511, 493, 541, 547, 513, 517, 523, 555, 503, 553, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[510, 512, 530, 514, 497, 526, 528, 512, 509, 515, 502, 494, 534, 515, 524, 510, 523, 543, 517, 493, 498, 517, 501, 512, 504, 488, 496, 492, 504, 490, 482, 471, 501, 507, 510, 519, 493, 489, 520, 519, 514, 536, 532, 511, 480, 526, 498, 536, 495, 507, 543, 522, 511, 506, 513, 541, 518, 528, 504, 556, 516, 532, 468, 554] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[517, 503, 507, 504, 535, 504, 485, 510, 0, 0, 0, 0, 0, 0, 0, 0, 493, 505, 525, 542, 498, 503, 482, 496, 0, 0, 0, 0, 0, 0, 0, 0, 523, 489, 501, 499, 512, 497, 494, 535, 21, 14, 0, 0, 0, 0, 0, 0, 546, 519, 537, 505, 522, 520, 496, 506, 0, 0, 0, 0, 0, 0, 0, 0, 514, 519, 518, 525, 518, 508, 559, 552, 0, 0, 0, 0, 0, 0, 0, 0, 562, 550, 462, 511, 536, 518, 483, 476, 0, 0, 0, 0, 0, 0, 0, 0, 550, 490, 517, 522, 509, 495, 503, 495, 10, 4, 3, 0, 0, 0, 0, 0, 497, 493, 484, 529, 478, 517, 525, 491, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4065, 4044, 4085, 4151, 4213, 4098, 4098, 4014] avg=4096.0000 max=4213 max_rank=4 min=4014 min_rank=7 imbalance_ratio=1.028564 max_load_reduction=28555 max_load_reduction_pct=0.871429 moved_to_echo=16423 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[477, 534, 503, 494, 495, 536, 511, 527, 0, 0, 0, 0, 0, 0, 0, 0, 490, 486, 511, 473, 552, 506, 504, 523, 0, 0, 0, 0, 0, 0, 0, 0, 531, 496, 506, 557, 519, 479, 521, 490, 24, 15, 0, 0, 0, 0, 0, 0, 509, 500, 517, 520, 507, 492, 511, 537, 0, 0, 0, 0, 0, 0, 0, 0, 518, 505, 474, 507, 494, 510, 545, 498, 0, 0, 0, 0, 0, 0, 0, 0, 505, 525, 528, 529, 522, 524, 462, 532, 0, 0, 0, 0, 0, 0, 0, 0, 484, 511, 543, 520, 495, 511, 523, 527, 11, 4, 3, 0, 0, 0, 0, 0, 532, 542, 495, 482, 516, 524, 515, 499, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4045, 4138, 4093, 4051, 4127, 4132, 4105] avg=4096.0000 max=4138 max_rank=2 min=4045 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16415 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[504, 517, 534, 508, 536, 515, 536, 534, 0, 0, 0, 0, 0, 0, 0, 0, 568, 512, 505, 516, 475, 526, 509, 509, 0, 0, 0, 0, 0, 0, 0, 0, 503, 477, 481, 533, 503, 477, 523, 515, 23, 13, 0, 0, 0, 0, 0, 0, 469, 506, 535, 507, 524, 532, 478, 546, 0, 0, 0, 0, 0, 0, 0, 0, 497, 478, 526, 522, 492, 492, 511, 503, 0, 0, 0, 0, 0, 0, 0, 0, 536, 521, 518, 528, 479, 504, 485, 528, 0, 0, 0, 0, 0, 0, 0, 0, 484, 525, 528, 523, 494, 502, 536, 514, 10, 4, 3, 0, 0, 0, 0, 0, 508, 497, 513, 499, 511, 494, 539, 515, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4184, 4120, 4048, 4097, 4021, 4099, 4123, 4076] avg=4096.0000 max=4184 max_rank=0 min=4021 min_rank=4 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16319 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[516, 529, 530, 532, 484, 532, 483, 508, 0, 0, 0, 0, 0, 0, 0, 0, 555, 507, 515, 538, 483, 514, 506, 502, 0, 0, 0, 0, 0, 0, 0, 0, 510, 482, 538, 505, 491, 518, 526, 504, 22, 13, 0, 0, 0, 0, 0, 0, 541, 530, 536, 514, 541, 506, 484, 478, 0, 0, 0, 0, 0, 0, 0, 0, 532, 519, 520, 518, 551, 499, 510, 512, 0, 0, 0, 0, 0, 0, 0, 0, 500, 460, 491, 523, 474, 503, 511, 521, 0, 0, 0, 0, 0, 0, 0, 0, 496, 495, 549, 494, 473, 500, 501, 497, 10, 4, 3, 0, 0, 0, 0, 0, 524, 550, 506, 528, 502, 512, 479, 528, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4120, 4109, 4130, 4161, 3983, 4022, 4129] avg=4096.0000 max=4161 max_rank=4 min=3983 min_rank=5 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16295 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[509, 532, 496, 526, 485, 493, 522, 504, 0, 0, 0, 0, 0, 0, 0, 0, 512, 526, 544, 526, 519, 521, 498, 568, 0, 0, 0, 0, 0, 0, 0, 0, 516, 484, 490, 484, 481, 539, 520, 501, 23, 14, 0, 0, 0, 0, 0, 0, 529, 495, 494, 507, 555, 500, 502, 505, 0, 0, 0, 0, 0, 0, 0, 0, 514, 478, 489, 487, 486, 528, 534, 486, 0, 0, 0, 0, 0, 0, 0, 0, 508, 546, 511, 508, 522, 500, 477, 522, 0, 0, 0, 0, 0, 0, 0, 0, 535, 535, 501, 529, 467, 519, 564, 524, 10, 4, 3, 0, 0, 0, 0, 0, 468, 498, 503, 524, 509, 544, 495, 520, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4214, 4052, 4087, 4002, 4094, 4191, 4061] avg=4096.0000 max=4214 max_rank=1 min=4002 min_rank=4 imbalance_ratio=1.028809 max_load_reduction=28554 max_load_reduction_pct=0.871399 moved_to_echo=16348 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[508, 486, 513, 502, 519, 522, 497, 522, 0, 0, 0, 0, 0, 0, 0, 0, 488, 512, 515, 513, 489, 523, 547, 496, 0, 0, 0, 0, 0, 0, 0, 0, 510, 523, 540, 502, 504, 493, 536, 519, 27, 18, 0, 0, 0, 0, 0, 0, 570, 525, 485, 504, 515, 490, 529, 461, 5, 0, 0, 0, 0, 0, 0, 0, 494, 516, 521, 518, 560, 538, 514, 532, 0, 0, 0, 0, 0, 0, 0, 0, 531, 501, 498, 527, 530, 547, 546, 473, 0, 0, 0, 0, 0, 0, 0, 0, 509, 539, 507, 447, 483, 469, 481, 554, 14, 7, 8, 3, 0, 0, 0, 0, 469, 507, 514, 514, 505, 475, 520, 489, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4083, 4172, 4084, 4193, 4153, 4021, 3993] avg=4096.0000 max=4193 max_rank=4 min=3993 min_rank=7 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16360 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[504, 508, 526, 511, 521, 502, 504, 497, 0, 0, 0, 0, 0, 0, 0, 0, 498, 472, 503, 526, 507, 509, 494, 548, 0, 0, 0, 0, 0, 0, 0, 0, 522, 498, 498, 466, 521, 480, 496, 503, 25, 14, 0, 0, 0, 0, 0, 0, 560, 544, 531, 503, 502, 526, 520, 513, 0, 0, 0, 0, 0, 0, 0, 0, 513, 503, 499, 475, 517, 530, 513, 519, 0, 0, 0, 0, 0, 0, 0, 0, 485, 459, 558, 539, 503, 528, 542, 490, 0, 0, 0, 0, 0, 0, 0, 0, 491, 512, 461, 502, 498, 511, 493, 541, 10, 4, 3, 0, 0, 0, 0, 0, 547, 513, 517, 520, 555, 503, 553, 509, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4057, 4023, 4199, 4069, 4104, 4026, 4217] avg=4096.0000 max=4217 max_rank=7 min=4023 min_rank=2 imbalance_ratio=1.029541 max_load_reduction=28551 max_load_reduction_pct=0.871307 moved_to_echo=16416 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[510, 512, 530, 514, 497, 516, 528, 512, 0, 0, 0, 0, 0, 0, 0, 0, 509, 515, 502, 494, 516, 515, 524, 510, 0, 0, 0, 0, 0, 0, 0, 0, 523, 543, 517, 493, 498, 517, 501, 512, 23, 14, 0, 0, 0, 0, 0, 0, 504, 488, 496, 492, 504, 490, 482, 471, 0, 0, 0, 0, 0, 0, 0, 0, 501, 507, 510, 519, 493, 489, 520, 519, 0, 0, 0, 0, 0, 0, 0, 0, 514, 536, 509, 511, 480, 526, 498, 536, 0, 0, 0, 0, 0, 0, 0, 0, 495, 507, 543, 522, 511, 506, 513, 541, 10, 4, 3, 0, 0, 0, 0, 0, 518, 528, 504, 553, 516, 532, 468, 554, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4085, 4141, 3927, 4058, 4110, 4155, 4173] avg=4096.0000 max=4173 max_rank=7 min=3927 min_rank=3 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16496 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4352, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4096, 3968, 4096, 4224, 4224, 128, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[537, 503, 501, 493, 494, 532, 510, 489, 531, 505, 542, 497, 543, 549, 540, 532, 505, 507, 508, 526, 500, 520, 495, 478, 499, 508, 510, 529, 476, 510, 464, 519, 473, 520, 486, 520, 530, 502, 521, 543, 502, 522, 538, 504, 488, 514, 498, 502, 516, 476, 512, 507, 492, 553, 517, 525, 481, 515, 498, 529, 524, 539, 534, 535] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[505, 499, 516, 526, 492, 514, 557, 509, 497, 495, 498, 500, 522, 553, 524, 511, 516, 519, 450, 470, 550, 476, 499, 503, 528, 491, 530, 478, 537, 480, 545, 453, 508, 495, 542, 514, 487, 519, 523, 519, 503, 524, 494, 537, 504, 500, 507, 499, 520, 497, 577, 511, 561, 535, 494, 528, 530, 507, 516, 515, 513, 523, 493, 530] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[520, 513, 518, 519, 488, 528, 480, 524, 525, 526, 542, 473, 499, 482, 545, 520, 525, 502, 507, 501, 518, 540, 488, 493, 482, 511, 473, 497, 529, 483, 540, 508, 478, 501, 498, 504, 519, 510, 529, 536, 548, 502, 497, 540, 522, 524, 517, 480, 498, 503, 494, 500, 512, 523, 501, 532, 535, 564, 484, 513, 542, 532, 512, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[515, 515, 528, 502, 529, 530, 523, 548, 542, 543, 485, 494, 499, 505, 539, 566, 502, 506, 488, 499, 485, 494, 544, 484, 514, 494, 502, 511, 472, 512, 533, 549, 547, 500, 521, 501, 522, 515, 518, 527, 491, 486, 494, 526, 488, 505, 513, 535, 495, 517, 523, 498, 539, 549, 494, 519, 496, 533, 516, 532, 479, 501, 478, 458] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[514, 521, 511, 519, 510, 512, 556, 520, 497, 499, 526, 510, 551, 504, 479, 539, 521, 460, 518, 529, 518, 503, 545, 544, 493, 526, 539, 511, 543, 510, 500, 547, 502, 510, 524, 515, 529, 483, 527, 525, 515, 498, 474, 505, 523, 505, 492, 501, 525, 514, 491, 488, 519, 521, 511, 477, 488, 491, 485, 492, 515, 515, 523, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[499, 506, 529, 526, 507, 519, 536, 474, 502, 482, 484, 518, 501, 539, 510, 536, 499, 554, 527, 498, 484, 536, 508, 486, 538, 532, 474, 507, 522, 528, 493, 517, 528, 512, 476, 510, 516, 518, 522, 515, 535, 529, 507, 519, 538, 501, 546, 518, 509, 533, 498, 525, 499, 477, 554, 516, 488, 495, 482, 502, 499, 547, 465, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[516, 521, 490, 515, 500, 510, 515, 513, 517, 492, 507, 534, 511, 504, 497, 510, 508, 476, 518, 470, 469, 487, 500, 523, 528, 536, 533, 547, 517, 508, 518, 518, 525, 522, 498, 488, 533, 479, 524, 500, 549, 529, 550, 540, 475, 490, 508, 515, 533, 508, 488, 504, 514, 522, 492, 486, 533, 506, 514, 494, 530, 525, 541, 545] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[538, 514, 510, 521, 529, 531, 495, 521, 528, 512, 471, 513, 511, 481, 531, 478, 479, 539, 486, 521, 477, 499, 506, 520, 486, 499, 550, 518, 539, 515, 519, 535, 486, 498, 499, 547, 493, 517, 523, 487, 502, 540, 486, 503, 563, 519, 491, 507, 482, 534, 549, 488, 514, 534, 471, 506, 534, 525, 507, 560, 522, 480, 504, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[515, 515, 528, 502, 529, 502, 523, 548, 0, 0, 0, 0, 0, 0, 0, 0, 542, 543, 485, 494, 499, 505, 539, 544, 0, 0, 0, 0, 0, 0, 0, 0, 502, 506, 488, 499, 485, 494, 544, 484, 28, 22, 13, 11, 8, 0, 0, 0, 514, 494, 502, 511, 472, 512, 533, 538, 0, 0, 0, 0, 0, 0, 0, 0, 547, 500, 521, 501, 522, 515, 518, 527, 2, 0, 0, 0, 0, 0, 0, 0, 491, 486, 494, 524, 488, 505, 513, 535, 0, 0, 0, 0, 0, 0, 0, 0, 495, 517, 523, 498, 539, 541, 494, 519, 0, 0, 0, 0, 0, 0, 0, 0, 496, 533, 516, 532, 479, 488, 478, 458, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4162, 4151, 4084, 4076, 4153, 4036, 4126, 3980] avg=4096.0000 max=4162 max_rank=0 min=3980 min_rank=7 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16295 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[499, 506, 529, 526, 507, 496, 536, 474, 0, 0, 0, 0, 0, 0, 0, 0, 502, 482, 484, 518, 501, 539, 510, 520, 0, 0, 0, 0, 0, 0, 0, 0, 499, 554, 527, 498, 484, 536, 508, 486, 23, 16, 8, 5, 3, 0, 0, 0, 538, 532, 474, 507, 522, 528, 493, 512, 0, 0, 0, 0, 0, 0, 0, 0, 528, 512, 476, 510, 516, 518, 522, 515, 1, 0, 0, 0, 0, 0, 0, 0, 535, 529, 507, 518, 538, 501, 546, 518, 0, 0, 0, 0, 0, 0, 0, 0, 509, 533, 498, 525, 499, 474, 554, 516, 0, 0, 0, 0, 0, 0, 0, 0, 488, 495, 482, 502, 499, 539, 465, 518, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4056, 4147, 4106, 4098, 4192, 4108, 3988] avg=4096.0000 max=4192 max_rank=5 min=3988 min_rank=7 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16386 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[514, 521, 511, 519, 510, 490, 556, 520, 0, 0, 0, 0, 0, 0, 0, 0, 497, 499, 526, 510, 551, 504, 479, 523, 0, 0, 0, 0, 0, 0, 0, 0, 521, 460, 518, 529, 518, 503, 545, 544, 22, 16, 8, 5, 3, 0, 0, 0, 493, 526, 539, 511, 543, 510, 500, 542, 0, 0, 0, 0, 0, 0, 0, 0, 502, 510, 524, 515, 529, 483, 527, 525, 1, 0, 0, 0, 0, 0, 0, 0, 515, 498, 474, 504, 523, 505, 492, 501, 0, 0, 0, 0, 0, 0, 0, 0, 525, 514, 491, 488, 519, 518, 511, 477, 0, 0, 0, 0, 0, 0, 0, 0, 488, 491, 485, 492, 515, 507, 523, 510, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4141, 4089, 4192, 4164, 4116, 4012, 4043, 4011] avg=4096.0000 max=4192 max_rank=2 min=4011 min_rank=7 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16182 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[520, 513, 518, 519, 488, 505, 480, 524, 0, 0, 0, 0, 0, 0, 0, 0, 525, 526, 542, 473, 499, 482, 545, 504, 0, 0, 0, 0, 0, 0, 0, 0, 525, 502, 507, 501, 518, 540, 488, 493, 23, 16, 8, 5, 3, 0, 0, 0, 482, 511, 473, 497, 529, 483, 540, 503, 0, 0, 0, 0, 0, 0, 0, 0, 478, 501, 498, 504, 519, 510, 529, 536, 1, 0, 0, 0, 0, 0, 0, 0, 548, 502, 497, 539, 522, 524, 517, 480, 0, 0, 0, 0, 0, 0, 0, 0, 498, 503, 494, 500, 512, 520, 501, 532, 0, 0, 0, 0, 0, 0, 0, 0, 535, 564, 484, 513, 542, 524, 512, 519, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4096, 4129, 4018, 4076, 4129, 4060, 4193] avg=4096.0000 max=4193 max_rank=7 min=4018 min_rank=3 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16458 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[537, 503, 501, 493, 494, 509, 510, 489, 0, 0, 0, 0, 0, 0, 0, 0, 531, 505, 542, 497, 543, 549, 540, 516, 0, 0, 0, 0, 0, 0, 0, 0, 505, 507, 508, 526, 500, 520, 495, 478, 23, 16, 8, 5, 4, 0, 0, 0, 499, 508, 510, 529, 476, 510, 464, 514, 0, 0, 0, 0, 0, 0, 0, 0, 473, 520, 486, 520, 530, 502, 521, 543, 1, 0, 0, 0, 0, 0, 0, 0, 502, 522, 538, 503, 488, 514, 498, 502, 0, 0, 0, 0, 0, 0, 0, 0, 516, 476, 512, 507, 492, 549, 517, 525, 0, 0, 0, 0, 0, 0, 0, 0, 481, 515, 498, 529, 524, 531, 534, 535, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4036, 4223, 4095, 4010, 4096, 4067, 4094, 4147] avg=4096.0000 max=4223 max_rank=1 min=4010 min_rank=3 imbalance_ratio=1.031006 max_load_reduction=28545 max_load_reduction_pct=0.871124 moved_to_echo=16404 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[516, 521, 490, 515, 500, 488, 515, 513, 0, 0, 0, 0, 0, 0, 0, 0, 517, 492, 507, 534, 511, 504, 497, 495, 0, 0, 0, 0, 0, 0, 0, 0, 508, 476, 518, 470, 469, 487, 500, 523, 22, 15, 8, 5, 3, 0, 0, 0, 528, 536, 533, 547, 517, 508, 518, 513, 0, 0, 0, 0, 0, 0, 0, 0, 525, 522, 498, 488, 533, 479, 524, 500, 1, 0, 0, 0, 0, 0, 0, 0, 549, 529, 550, 539, 475, 490, 508, 515, 0, 0, 0, 0, 0, 0, 0, 0, 533, 508, 488, 504, 514, 519, 492, 486, 0, 0, 0, 0, 0, 0, 0, 0, 533, 506, 514, 494, 530, 517, 541, 545, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4058, 4057, 4004, 4200, 4070, 4155, 4044, 4180] avg=4096.0000 max=4200 max_rank=3 min=4004 min_rank=2 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16449 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[538, 514, 510, 521, 529, 508, 495, 521, 0, 0, 0, 0, 0, 0, 0, 0, 528, 512, 471, 513, 511, 481, 531, 464, 0, 0, 0, 0, 0, 0, 0, 0, 479, 539, 486, 521, 477, 499, 506, 520, 23, 14, 7, 5, 4, 0, 0, 0, 486, 499, 550, 518, 539, 515, 519, 530, 0, 0, 0, 0, 0, 0, 0, 0, 486, 498, 499, 547, 493, 517, 523, 487, 1, 0, 0, 0, 0, 0, 0, 0, 502, 540, 486, 502, 563, 519, 491, 507, 0, 0, 0, 0, 0, 0, 0, 0, 482, 534, 549, 488, 514, 530, 471, 506, 0, 0, 0, 0, 0, 0, 0, 0, 534, 525, 507, 560, 522, 473, 504, 525, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4136, 4011, 4080, 4156, 4051, 4110, 4074, 4150] avg=4096.0000 max=4156 max_rank=3 min=4011 min_rank=1 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16385 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[505, 499, 516, 526, 492, 491, 557, 509, 0, 0, 0, 0, 0, 0, 0, 0, 497, 495, 498, 500, 522, 553, 524, 496, 0, 0, 0, 0, 0, 0, 0, 0, 516, 519, 450, 470, 550, 476, 499, 503, 23, 15, 8, 4, 4, 0, 0, 0, 528, 491, 530, 478, 537, 480, 545, 449, 0, 0, 0, 0, 0, 0, 0, 0, 508, 495, 542, 514, 487, 519, 523, 519, 1, 0, 0, 0, 0, 0, 0, 0, 503, 524, 494, 536, 504, 500, 507, 499, 0, 0, 0, 0, 0, 0, 0, 0, 520, 497, 577, 511, 561, 531, 494, 528, 0, 0, 0, 0, 0, 0, 0, 0, 530, 507, 516, 515, 513, 515, 493, 530, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4085, 4037, 4038, 4108, 4067, 4219, 4119] avg=4096.0000 max=4219 max_rank=6 min=4037 min_rank=2 imbalance_ratio=1.030029 max_load_reduction=28549 max_load_reduction_pct=0.871246 moved_to_echo=16513 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 256, 256, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[547, 493, 515, 494, 519, 517, 515, 529, 534, 535, 528, 527, 494, 496, 504, 506, 505, 496, 517, 500, 521, 471, 509, 522, 497, 539, 513, 511, 480, 511, 540, 532, 489, 507, 551, 542, 520, 514, 532, 542, 532, 478, 495, 488, 524, 520, 517, 518, 463, 484, 478, 542, 528, 498, 507, 522, 493, 505, 497, 518, 526, 503, 512, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[497, 525, 504, 508, 526, 496, 507, 518, 558, 505, 504, 511, 537, 488, 522, 522, 493, 495, 491, 515, 509, 477, 486, 494, 546, 531, 519, 506, 525, 454, 555, 533, 502, 512, 545, 464, 535, 510, 507, 495, 500, 514, 513, 521, 516, 528, 525, 468, 504, 524, 554, 479, 537, 501, 514, 516, 495, 537, 531, 525, 509, 531, 473, 526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[513, 524, 546, 494, 503, 545, 526, 476, 463, 509, 493, 520, 490, 502, 518, 533, 524, 524, 505, 531, 497, 516, 484, 502, 527, 522, 517, 494, 505, 528, 544, 503, 559, 501, 520, 489, 538, 502, 465, 486, 510, 485, 504, 510, 524, 522, 483, 532, 548, 549, 498, 507, 512, 505, 503, 521, 524, 526, 484, 505, 504, 534, 505, 535] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[519, 506, 484, 529, 501, 457, 522, 540, 521, 508, 530, 463, 497, 560, 512, 541, 475, 510, 484, 486, 531, 511, 527, 515, 538, 550, 518, 513, 506, 518, 484, 512, 507, 543, 471, 509, 527, 511, 526, 492, 538, 573, 496, 501, 500, 522, 530, 495, 491, 491, 552, 491, 495, 536, 481, 527, 519, 532, 518, 495, 519, 474, 492, 546] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[528, 491, 491, 528, 512, 501, 540, 541, 526, 544, 524, 504, 550, 519, 510, 482, 526, 496, 507, 484, 514, 459, 515, 511, 493, 519, 481, 529, 518, 536, 524, 505, 510, 488, 482, 524, 513, 515, 500, 499, 513, 523, 512, 513, 491, 509, 509, 516, 508, 518, 509, 511, 496, 567, 518, 532, 506, 499, 495, 509, 509, 517, 514, 535] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[501, 487, 528, 517, 517, 483, 504, 510, 525, 555, 503, 513, 492, 529, 516, 493, 505, 525, 478, 513, 514, 533, 493, 530, 532, 517, 522, 505, 498, 524, 553, 546, 527, 519, 499, 485, 523, 515, 511, 518, 515, 484, 490, 508, 514, 470, 524, 524, 490, 544, 501, 525, 498, 461, 519, 490, 486, 559, 550, 503, 539, 500, 525, 491] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[505, 476, 463, 511, 504, 450, 561, 537, 510, 507, 521, 519, 544, 509, 507, 521, 545, 515, 531, 518, 491, 497, 521, 509, 494, 550, 511, 530, 516, 514, 505, 546, 503, 535, 528, 499, 472, 492, 533, 494, 515, 506, 510, 509, 512, 542, 485, 521, 526, 481, 503, 491, 532, 523, 512, 501, 522, 521, 493, 521, 491, 556, 464, 537] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[489, 551, 520, 490, 518, 512, 541, 514, 496, 482, 522, 483, 510, 491, 484, 496, 517, 519, 537, 496, 487, 541, 508, 539, 545, 543, 491, 546, 479, 513, 507, 527, 531, 534, 472, 518, 508, 533, 543, 514, 510, 512, 524, 519, 496, 534, 517, 497, 508, 505, 540, 472, 484, 523, 518, 521, 500, 518, 517, 501, 503, 489, 491, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[497, 525, 504, 508, 526, 496, 507, 518, 5, 0, 0, 0, 0, 0, 0, 0, 558, 500, 504, 511, 537, 488, 522, 522, 0, 0, 0, 0, 0, 0, 0, 0, 493, 495, 491, 515, 509, 477, 486, 494, 33, 0, 0, 0, 0, 0, 0, 0, 546, 481, 519, 506, 525, 454, 555, 533, 0, 0, 0, 0, 0, 0, 0, 0, 502, 512, 545, 464, 535, 510, 507, 495, 1, 0, 0, 0, 0, 0, 0, 0, 500, 514, 513, 521, 516, 528, 525, 468, 7, 0, 0, 0, 0, 0, 0, 0, 504, 524, 554, 479, 537, 501, 514, 516, 10, 0, 0, 0, 0, 0, 0, 0, 495, 537, 531, 525, 509, 531, 473, 525, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4142, 3993, 4119, 4071, 4092, 4139, 4126] avg=4096.0000 max=4142 max_rank=1 min=3993 min_rank=2 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16428 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[547, 493, 515, 494, 519, 517, 515, 529, 5, 0, 0, 0, 0, 0, 0, 0, 534, 530, 528, 527, 494, 496, 504, 506, 0, 0, 0, 0, 0, 0, 0, 0, 505, 496, 517, 500, 521, 471, 509, 522, 34, 0, 0, 0, 0, 0, 0, 0, 497, 488, 513, 511, 480, 511, 540, 532, 0, 0, 0, 0, 0, 0, 0, 0, 489, 507, 551, 542, 520, 514, 532, 542, 1, 0, 0, 0, 0, 0, 0, 0, 532, 478, 495, 488, 524, 520, 517, 518, 7, 0, 0, 0, 0, 0, 0, 0, 463, 484, 478, 542, 528, 498, 507, 522, 10, 0, 0, 0, 0, 0, 0, 0, 493, 505, 497, 518, 526, 503, 512, 505, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4134, 4119, 4075, 4072, 4198, 4079, 4032, 4059] avg=4096.0000 max=4198 max_rank=4 min=4032 min_rank=6 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[528, 491, 491, 528, 512, 501, 540, 541, 6, 0, 0, 0, 0, 0, 0, 0, 526, 538, 524, 504, 550, 519, 510, 482, 0, 0, 0, 0, 0, 0, 0, 0, 526, 496, 507, 484, 514, 459, 515, 511, 32, 0, 0, 0, 0, 0, 0, 0, 493, 470, 481, 529, 518, 536, 524, 505, 0, 0, 0, 0, 0, 0, 0, 0, 510, 488, 482, 524, 513, 515, 500, 499, 1, 0, 0, 0, 0, 0, 0, 0, 513, 523, 512, 513, 491, 509, 509, 516, 7, 0, 0, 0, 0, 0, 0, 0, 508, 518, 509, 511, 496, 567, 518, 532, 10, 0, 0, 0, 0, 0, 0, 0, 506, 499, 495, 509, 509, 517, 514, 534, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4138, 4153, 4044, 4056, 4032, 4093, 4169, 4083] avg=4096.0000 max=4169 max_rank=6 min=4032 min_rank=4 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[501, 487, 528, 517, 517, 483, 504, 510, 6, 0, 0, 0, 0, 0, 0, 0, 525, 549, 503, 513, 492, 529, 516, 493, 0, 0, 0, 0, 0, 0, 0, 0, 505, 525, 478, 513, 514, 533, 493, 530, 32, 0, 0, 0, 0, 0, 0, 0, 532, 468, 522, 505, 498, 524, 553, 546, 0, 0, 0, 0, 0, 0, 0, 0, 527, 519, 499, 485, 523, 515, 511, 518, 1, 0, 0, 0, 0, 0, 0, 0, 515, 484, 490, 508, 514, 470, 524, 524, 7, 0, 0, 0, 0, 0, 0, 0, 490, 544, 501, 525, 498, 461, 519, 490, 10, 0, 0, 0, 0, 0, 0, 0, 486, 559, 550, 503, 539, 500, 525, 490, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4053, 4120, 4123, 4148, 4098, 4036, 4038, 4152] avg=4096.0000 max=4152 max_rank=7 min=4036 min_rank=5 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16324 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[505, 476, 463, 511, 504, 450, 561, 537, 9, 6, 0, 0, 0, 0, 0, 0, 510, 494, 521, 519, 544, 509, 507, 521, 0, 0, 0, 0, 0, 0, 0, 0, 545, 515, 531, 518, 491, 497, 521, 509, 39, 0, 0, 0, 0, 0, 0, 0, 494, 479, 511, 530, 516, 514, 505, 546, 0, 0, 0, 0, 0, 0, 0, 0, 503, 535, 528, 499, 472, 492, 533, 494, 7, 4, 0, 0, 0, 0, 0, 0, 515, 506, 510, 509, 512, 542, 485, 521, 13, 0, 0, 0, 0, 0, 0, 0, 526, 481, 503, 491, 532, 523, 512, 501, 13, 0, 0, 0, 0, 0, 0, 0, 522, 521, 493, 521, 491, 556, 464, 530, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4022, 4125, 4166, 4095, 4067, 4113, 4082, 4098] avg=4096.0000 max=4166 max_rank=2 min=4022 min_rank=0 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16360 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[513, 524, 546, 494, 503, 545, 526, 476, 5, 0, 0, 0, 0, 0, 0, 0, 463, 504, 493, 520, 490, 502, 518, 533, 0, 0, 0, 0, 0, 0, 0, 0, 524, 524, 505, 531, 497, 516, 484, 502, 33, 0, 0, 0, 0, 0, 0, 0, 527, 472, 517, 494, 505, 528, 544, 503, 0, 0, 0, 0, 0, 0, 0, 0, 559, 501, 520, 489, 538, 502, 465, 486, 1, 0, 0, 0, 0, 0, 0, 0, 510, 485, 504, 510, 524, 522, 483, 532, 7, 0, 0, 0, 0, 0, 0, 0, 548, 549, 498, 507, 512, 505, 503, 521, 10, 0, 0, 0, 0, 0, 0, 0, 524, 526, 484, 505, 504, 534, 505, 534, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4132, 4023, 4116, 4090, 4061, 4077, 4153, 4116] avg=4096.0000 max=4153 max_rank=6 min=4023 min_rank=1 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16407 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[489, 551, 520, 490, 518, 512, 541, 514, 5, 0, 0, 0, 0, 0, 0, 0, 496, 477, 522, 483, 510, 491, 484, 496, 0, 0, 0, 0, 0, 0, 0, 0, 517, 519, 537, 496, 487, 541, 508, 539, 34, 0, 0, 0, 0, 0, 0, 0, 545, 492, 491, 546, 479, 513, 507, 527, 0, 0, 0, 0, 0, 0, 0, 0, 531, 534, 472, 518, 508, 533, 543, 514, 1, 0, 0, 0, 0, 0, 0, 0, 510, 512, 524, 519, 496, 534, 517, 497, 7, 0, 0, 0, 0, 0, 0, 0, 508, 505, 540, 472, 484, 523, 518, 521, 10, 0, 0, 0, 0, 0, 0, 0, 500, 518, 517, 501, 503, 489, 491, 521, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4140, 3959, 4178, 4100, 4154, 4116, 4081, 4040] avg=4096.0000 max=4178 max_rank=2 min=3959 min_rank=1 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[519, 506, 484, 529, 501, 457, 522, 540, 5, 0, 0, 0, 0, 0, 0, 0, 521, 503, 530, 463, 497, 560, 512, 541, 0, 0, 0, 0, 0, 0, 0, 0, 475, 510, 484, 486, 531, 511, 527, 515, 34, 0, 0, 0, 0, 0, 0, 0, 538, 499, 518, 513, 506, 518, 484, 512, 0, 0, 0, 0, 0, 0, 0, 0, 507, 543, 471, 509, 527, 511, 526, 492, 1, 0, 0, 0, 0, 0, 0, 0, 538, 573, 496, 501, 500, 522, 530, 495, 7, 0, 0, 0, 0, 0, 0, 0, 491, 491, 552, 491, 495, 536, 481, 527, 10, 0, 0, 0, 0, 0, 0, 0, 519, 532, 518, 495, 519, 474, 492, 545, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4127, 4073, 4088, 4087, 4162, 4074, 4094] avg=4096.0000 max=4162 max_rank=5 min=4063 min_rank=0 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16417 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 3968, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 3968, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[546, 472, 508, 542, 528, 502, 513, 505, 499, 503, 502, 527, 534, 539, 519, 516, 547, 509, 508, 516, 495, 498, 524, 538, 510, 525, 472, 527, 497, 508, 504, 478, 510, 541, 533, 491, 485, 525, 495, 497, 520, 512, 521, 497, 513, 531, 528, 500, 495, 487, 538, 536, 507, 503, 516, 478, 511, 498, 538, 501, 490, 542, 519, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[510, 510, 520, 501, 514, 497, 501, 489, 490, 478, 527, 483, 544, 515, 511, 482, 545, 494, 533, 539, 502, 543, 530, 503, 514, 535, 517, 511, 533, 520, 486, 531, 500, 475, 527, 501, 507, 507, 526, 487, 491, 533, 510, 515, 562, 527, 536, 497, 482, 520, 521, 524, 492, 531, 516, 511, 505, 530, 506, 536, 489, 530, 508, 458] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[524, 493, 513, 498, 533, 483, 499, 501, 531, 518, 500, 521, 509, 499, 492, 470, 514, 518, 526, 509, 512, 518, 511, 499, 498, 495, 504, 529, 530, 494, 531, 511, 488, 559, 496, 527, 542, 524, 530, 511, 508, 535, 532, 505, 515, 507, 519, 490, 520, 533, 503, 522, 515, 468, 484, 528, 516, 539, 516, 506, 504, 517, 525, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[517, 515, 496, 509, 529, 550, 527, 507, 479, 545, 489, 519, 509, 525, 500, 501, 510, 530, 506, 510, 481, 508, 521, 500, 546, 505, 517, 535, 505, 526, 518, 528, 524, 527, 524, 510, 479, 491, 512, 513, 489, 518, 494, 516, 468, 513, 536, 517, 520, 549, 520, 496, 513, 515, 471, 547, 508, 501, 527, 523, 483, 501, 499, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[507, 562, 519, 555, 538, 494, 513, 526, 516, 535, 494, 485, 521, 462, 527, 516, 510, 504, 505, 504, 515, 525, 548, 536, 522, 497, 517, 493, 489, 537, 503, 532, 550, 535, 510, 490, 554, 520, 502, 511, 503, 534, 529, 523, 512, 496, 490, 516, 506, 528, 494, 495, 517, 483, 483, 487, 491, 488, 498, 522, 472, 478, 512, 532] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[526, 507, 520, 514, 526, 493, 562, 537, 508, 501, 497, 496, 500, 519, 501, 464, 514, 470, 529, 505, 545, 466, 526, 509, 514, 523, 533, 543, 533, 571, 502, 511, 545, 508, 503, 500, 525, 501, 479, 526, 505, 479, 496, 524, 515, 494, 489, 568, 486, 508, 482, 523, 477, 523, 531, 481, 521, 551, 493, 509, 481, 521, 503, 556] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[526, 511, 547, 530, 485, 475, 496, 540, 486, 520, 551, 526, 524, 541, 497, 502, 533, 506, 495, 510, 516, 534, 523, 507, 543, 506, 497, 504, 499, 494, 528, 499, 495, 509, 512, 473, 474, 520, 531, 522, 540, 502, 543, 509, 476, 513, 560, 539, 515, 492, 512, 512, 475, 543, 504, 499, 549, 497, 499, 503, 498, 501, 509, 491] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[464, 499, 497, 515, 505, 516, 522, 496, 522, 488, 500, 521, 502, 505, 524, 555, 522, 533, 494, 489, 545, 517, 499, 499, 529, 544, 517, 517, 500, 529, 502, 513, 519, 538, 532, 517, 483, 531, 490, 478, 497, 527, 529, 545, 505, 552, 516, 554, 518, 510, 493, 517, 506, 498, 501, 493, 509, 503, 503, 522, 493, 498, 514, 497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[510, 510, 520, 489, 514, 497, 501, 489, 0, 0, 0, 0, 0, 0, 0, 0, 490, 478, 527, 483, 544, 515, 511, 482, 12, 10, 0, 0, 0, 0, 0, 0, 525, 494, 533, 539, 502, 543, 530, 503, 0, 0, 0, 0, 0, 0, 0, 0, 514, 535, 517, 511, 533, 495, 486, 531, 0, 0, 0, 0, 0, 0, 0, 0, 500, 475, 527, 501, 507, 507, 526, 487, 2, 0, 0, 0, 0, 0, 0, 0, 491, 533, 510, 515, 562, 527, 536, 466, 0, 0, 0, 0, 0, 0, 0, 0, 482, 520, 521, 524, 492, 531, 516, 511, 31, 5, 0, 0, 0, 0, 0, 0, 505, 530, 506, 536, 489, 530, 508, 458, 20, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4030, 4052, 4169, 4122, 4032, 4140, 4133, 4090] avg=4096.0000 max=4169 max_rank=2 min=4030 min_rank=0 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16395 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[546, 472, 508, 529, 528, 502, 513, 505, 0, 0, 0, 0, 0, 0, 0, 0, 499, 503, 502, 527, 534, 539, 519, 516, 13, 11, 0, 0, 0, 0, 0, 0, 526, 509, 508, 516, 495, 498, 524, 538, 0, 0, 0, 0, 0, 0, 0, 0, 510, 525, 472, 527, 497, 483, 504, 478, 0, 0, 0, 0, 0, 0, 0, 0, 510, 541, 533, 491, 485, 525, 495, 497, 2, 0, 0, 0, 0, 0, 0, 0, 520, 512, 521, 497, 513, 531, 528, 469, 0, 0, 0, 0, 0, 0, 0, 0, 495, 487, 538, 536, 507, 503, 516, 478, 31, 5, 0, 0, 0, 0, 0, 0, 511, 498, 538, 501, 490, 542, 519, 499, 20, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4163, 4114, 3996, 4079, 4091, 4096, 4126] avg=4096.0000 max=4163 max_rank=1 min=3996 min_rank=3 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[517, 515, 496, 497, 529, 550, 527, 507, 0, 0, 0, 0, 0, 0, 0, 0, 479, 545, 489, 519, 509, 525, 500, 501, 12, 10, 0, 0, 0, 0, 0, 0, 491, 530, 506, 510, 481, 508, 521, 500, 0, 0, 0, 0, 0, 0, 0, 0, 546, 505, 517, 535, 505, 500, 518, 528, 0, 0, 0, 0, 0, 0, 0, 0, 524, 527, 524, 510, 479, 491, 512, 513, 2, 0, 0, 0, 0, 0, 0, 0, 489, 518, 494, 516, 468, 513, 536, 484, 0, 0, 0, 0, 0, 0, 0, 0, 520, 549, 520, 496, 513, 515, 471, 547, 33, 6, 0, 0, 0, 0, 0, 0, 508, 501, 527, 523, 483, 501, 499, 501, 20, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4138, 4089, 4047, 4154, 4082, 4018, 4170, 4070] avg=4096.0000 max=4170 max_rank=6 min=4018 min_rank=5 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16340 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[524, 493, 513, 486, 533, 483, 499, 501, 0, 0, 0, 0, 0, 0, 0, 0, 531, 518, 500, 521, 509, 499, 492, 470, 12, 10, 0, 0, 0, 0, 0, 0, 495, 518, 526, 509, 512, 518, 511, 499, 0, 0, 0, 0, 0, 0, 0, 0, 498, 495, 504, 529, 530, 470, 531, 511, 0, 0, 0, 0, 0, 0, 0, 0, 488, 559, 496, 527, 542, 524, 530, 511, 2, 0, 0, 0, 0, 0, 0, 0, 508, 535, 532, 505, 515, 507, 519, 459, 0, 0, 0, 0, 0, 0, 0, 0, 520, 533, 503, 522, 515, 468, 484, 528, 31, 5, 0, 0, 0, 0, 0, 0, 516, 539, 516, 506, 504, 517, 525, 501, 19, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4032, 4062, 4088, 4068, 4179, 4080, 4109, 4150] avg=4096.0000 max=4179 max_rank=4 min=4032 min_rank=0 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16518 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[526, 507, 520, 502, 526, 493, 562, 537, 0, 0, 0, 0, 0, 0, 0, 0, 508, 501, 497, 496, 500, 519, 501, 464, 12, 10, 0, 0, 0, 0, 0, 0, 495, 470, 529, 505, 545, 466, 526, 509, 0, 0, 0, 0, 0, 0, 0, 0, 514, 523, 533, 543, 533, 543, 502, 511, 0, 0, 0, 0, 0, 0, 0, 0, 545, 508, 503, 500, 525, 501, 479, 526, 2, 0, 0, 0, 0, 0, 0, 0, 505, 479, 496, 524, 515, 494, 489, 532, 0, 0, 0, 0, 0, 0, 0, 0, 486, 508, 482, 523, 477, 523, 531, 481, 36, 6, 0, 0, 0, 0, 0, 0, 521, 551, 493, 509, 481, 521, 503, 556, 22, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4173, 4008, 4045, 4202, 4089, 4034, 4053, 4164] avg=4096.0000 max=4202 max_rank=3 min=4008 min_rank=1 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16340 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[464, 499, 497, 503, 505, 516, 522, 496, 0, 0, 0, 0, 0, 0, 0, 0, 522, 488, 500, 521, 502, 505, 524, 555, 12, 10, 0, 0, 0, 0, 0, 0, 503, 533, 494, 489, 545, 517, 499, 499, 0, 0, 0, 0, 0, 0, 0, 0, 529, 544, 517, 517, 500, 503, 502, 513, 0, 0, 0, 0, 0, 0, 0, 0, 519, 538, 532, 517, 483, 531, 490, 478, 2, 0, 0, 0, 0, 0, 0, 0, 497, 527, 529, 545, 505, 552, 516, 519, 0, 0, 0, 0, 0, 0, 0, 0, 518, 510, 493, 517, 506, 498, 501, 493, 35, 6, 0, 0, 0, 0, 0, 0, 509, 503, 503, 522, 493, 498, 514, 497, 20, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4002, 4139, 4079, 4125, 4090, 4190, 4077, 4066] avg=4096.0000 max=4190 max_rank=5 min=4002 min_rank=0 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16423 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[526, 511, 547, 509, 485, 475, 496, 540, 0, 0, 0, 0, 0, 0, 0, 0, 486, 520, 551, 526, 524, 541, 497, 502, 15, 14, 0, 0, 0, 0, 0, 0, 507, 506, 495, 510, 516, 534, 523, 507, 0, 0, 0, 0, 0, 0, 0, 0, 543, 506, 497, 504, 499, 462, 528, 499, 0, 0, 0, 0, 0, 0, 0, 0, 495, 509, 512, 473, 474, 520, 531, 522, 7, 0, 0, 0, 0, 0, 0, 0, 540, 502, 543, 509, 476, 513, 560, 501, 0, 0, 0, 0, 0, 0, 0, 0, 515, 492, 512, 512, 475, 543, 504, 499, 38, 9, 0, 0, 0, 0, 0, 0, 549, 497, 499, 503, 498, 501, 509, 491, 23, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4176, 4098, 4038, 4043, 4144, 4099, 4081] avg=4096.0000 max=4176 max_rank=1 min=4038 min_rank=3 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[507, 562, 519, 542, 538, 494, 513, 526, 0, 0, 0, 0, 0, 0, 0, 0, 516, 535, 494, 485, 521, 462, 527, 516, 12, 11, 0, 0, 0, 0, 0, 0, 491, 504, 505, 504, 515, 525, 548, 536, 0, 0, 0, 0, 0, 0, 0, 0, 522, 497, 517, 493, 489, 510, 503, 532, 0, 0, 0, 0, 0, 0, 0, 0, 550, 535, 510, 490, 554, 520, 502, 511, 2, 0, 0, 0, 0, 0, 0, 0, 503, 534, 529, 523, 512, 496, 490, 484, 0, 0, 0, 0, 0, 0, 0, 0, 506, 528, 494, 495, 517, 483, 483, 487, 32, 6, 0, 0, 0, 0, 0, 0, 491, 488, 498, 522, 472, 478, 512, 532, 21, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4201, 4079, 4128, 4063, 4174, 4071, 4031, 4021] avg=4096.0000 max=4201 max_rank=0 min=4021 min_rank=7 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16297 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4224, 3968, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 3968, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 3968, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[487, 513, 517, 540, 504, 527, 500, 541, 528, 513, 563, 517, 493, 476, 500, 525, 520, 512, 518, 508, 467, 518, 515, 523, 503, 505, 537, 531, 522, 526, 472, 523, 484, 523, 558, 519, 450, 550, 541, 516, 503, 503, 494, 515, 518, 516, 508, 471, 542, 510, 519, 540, 526, 515, 509, 500, 504, 508, 530, 509, 420, 512, 517, 494] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[510, 527, 501, 506, 539, 490, 543, 483, 545, 500, 512, 517, 497, 525, 543, 523, 481, 480, 541, 527, 500, 496, 515, 536, 526, 507, 510, 488, 549, 535, 484, 509, 520, 533, 528, 511, 471, 484, 547, 461, 486, 519, 519, 510, 503, 508, 521, 537, 521, 536, 529, 512, 522, 535, 494, 502, 499, 501, 482, 530, 483, 524, 519, 476] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[547, 525, 511, 488, 552, 502, 458, 489, 505, 527, 503, 515, 487, 480, 491, 518, 522, 517, 539, 512, 515, 502, 499, 492, 521, 520, 535, 529, 496, 521, 476, 491, 501, 526, 536, 517, 541, 525, 538, 519, 493, 538, 504, 521, 527, 495, 520, 526, 517, 536, 512, 479, 511, 516, 505, 499, 494, 508, 525, 487, 524, 521, 518, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[552, 534, 489, 501, 517, 515, 488, 517, 529, 551, 484, 496, 483, 506, 532, 509, 514, 506, 514, 538, 505, 503, 495, 486, 490, 524, 557, 548, 524, 519, 501, 538, 519, 545, 505, 495, 476, 467, 527, 497, 538, 478, 545, 503, 514, 500, 521, 518, 540, 529, 524, 547, 517, 490, 522, 496, 474, 545, 499, 492, 505, 506, 462, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[491, 527, 543, 493, 529, 504, 527, 498, 515, 559, 540, 502, 503, 496, 507, 513, 561, 513, 486, 516, 480, 521, 507, 516, 517, 505, 501, 544, 484, 492, 515, 543, 474, 523, 528, 491, 513, 528, 526, 538, 527, 478, 520, 528, 506, 547, 501, 504, 487, 515, 503, 514, 512, 488, 541, 523, 453, 501, 510, 507, 510, 514, 484, 526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[513, 546, 538, 524, 477, 518, 542, 513, 457, 481, 515, 493, 485, 494, 519, 527, 511, 523, 528, 479, 507, 523, 517, 526, 500, 473, 554, 537, 520, 520, 493, 501, 500, 539, 501, 505, 492, 504, 506, 502, 514, 520, 535, 485, 555, 496, 522, 488, 491, 528, 515, 534, 515, 499, 486, 537, 529, 496, 508, 504, 535, 522, 534, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[498, 512, 574, 505, 503, 497, 494, 528, 524, 520, 494, 531, 514, 515, 503, 522, 506, 494, 552, 493, 485, 494, 496, 492, 531, 534, 513, 512, 484, 520, 521, 524, 530, 513, 513, 507, 545, 502, 519, 483, 463, 547, 539, 519, 500, 527, 526, 535, 519, 508, 491, 533, 464, 462, 512, 486, 523, 496, 523, 529, 512, 525, 499, 533] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[470, 531, 516, 501, 498, 520, 509, 529, 540, 487, 508, 533, 512, 522, 516, 503, 508, 493, 518, 534, 551, 533, 513, 486, 541, 511, 510, 501, 536, 495, 494, 520, 538, 494, 437, 504, 482, 506, 531, 519, 523, 508, 508, 491, 499, 510, 505, 519, 530, 504, 549, 522, 518, 519, 502, 504, 480, 547, 482, 531, 521, 516, 537, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[487, 501, 517, 540, 504, 527, 500, 541, 0, 0, 0, 0, 0, 0, 0, 0, 528, 513, 563, 517, 493, 476, 500, 525, 2, 0, 0, 0, 0, 0, 0, 0, 520, 512, 518, 508, 467, 518, 515, 523, 9, 1, 0, 0, 0, 0, 0, 0, 503, 505, 513, 531, 522, 526, 472, 523, 0, 0, 0, 0, 0, 0, 0, 0, 484, 523, 558, 519, 450, 550, 541, 516, 2, 2, 0, 0, 0, 0, 0, 0, 503, 503, 490, 515, 518, 516, 508, 471, 0, 0, 0, 0, 0, 0, 0, 0, 542, 510, 519, 529, 526, 515, 509, 500, 0, 0, 0, 0, 0, 0, 0, 0, 504, 508, 530, 509, 420, 512, 517, 494, 24, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4117, 4117, 4091, 4095, 4145, 4024, 4150, 4029] avg=4096.0000 max=4150 max_rank=6 min=4024 min_rank=5 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16348 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[510, 513, 501, 506, 539, 490, 543, 483, 0, 0, 0, 0, 0, 0, 0, 0, 545, 500, 512, 517, 497, 525, 543, 523, 3, 0, 0, 0, 0, 0, 0, 0, 481, 480, 541, 527, 500, 496, 515, 536, 9, 2, 0, 0, 0, 0, 0, 0, 526, 507, 487, 488, 549, 535, 484, 509, 0, 0, 0, 0, 0, 0, 0, 0, 520, 533, 528, 511, 471, 484, 547, 461, 2, 2, 0, 0, 0, 0, 0, 0, 486, 519, 514, 510, 503, 508, 521, 537, 0, 0, 0, 0, 0, 0, 0, 0, 521, 536, 529, 501, 522, 535, 494, 502, 0, 0, 0, 0, 0, 0, 0, 0, 499, 501, 482, 530, 483, 524, 519, 476, 23, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4165, 4087, 4085, 4059, 4098, 4140, 4049] avg=4096.0000 max=4165 max_rank=1 min=4049 min_rank=7 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16346 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[498, 500, 574, 505, 503, 497, 494, 528, 0, 0, 0, 0, 0, 0, 0, 0, 524, 520, 494, 531, 514, 515, 503, 522, 3, 0, 0, 0, 0, 0, 0, 0, 506, 494, 552, 493, 485, 494, 496, 492, 9, 1, 0, 0, 0, 0, 0, 0, 531, 534, 490, 512, 484, 520, 521, 524, 0, 0, 0, 0, 0, 0, 0, 0, 530, 513, 513, 507, 545, 502, 519, 483, 3, 2, 0, 0, 0, 0, 0, 0, 463, 547, 533, 519, 500, 527, 526, 535, 0, 0, 0, 0, 0, 0, 0, 0, 519, 508, 491, 522, 464, 462, 512, 486, 0, 0, 0, 0, 0, 0, 0, 0, 523, 496, 523, 529, 512, 525, 499, 533, 23, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4126, 4022, 4116, 4117, 4150, 3964, 4174] avg=4096.0000 max=4174 max_rank=7 min=3964 min_rank=6 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16405 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[552, 520, 489, 501, 517, 515, 488, 517, 0, 0, 0, 0, 0, 0, 0, 0, 529, 551, 484, 496, 483, 506, 532, 509, 3, 0, 0, 0, 0, 0, 0, 0, 514, 506, 514, 538, 505, 503, 495, 486, 9, 2, 0, 0, 0, 0, 0, 0, 490, 524, 532, 548, 524, 519, 501, 538, 0, 0, 0, 0, 0, 0, 0, 0, 519, 545, 505, 495, 476, 467, 527, 497, 3, 2, 0, 0, 0, 0, 0, 0, 538, 478, 539, 503, 514, 500, 521, 518, 0, 0, 0, 0, 0, 0, 0, 0, 540, 529, 524, 536, 517, 490, 522, 496, 0, 0, 0, 0, 0, 0, 0, 0, 474, 545, 499, 492, 505, 506, 462, 507, 25, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4093, 4072, 4176, 4036, 4111, 4154, 4027] avg=4096.0000 max=4176 max_rank=3 min=4027 min_rank=7 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16328 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[547, 512, 511, 488, 552, 502, 458, 489, 0, 0, 0, 0, 0, 0, 0, 0, 505, 527, 503, 515, 487, 480, 491, 518, 3, 0, 0, 0, 0, 0, 0, 0, 522, 517, 539, 512, 515, 502, 499, 492, 8, 1, 0, 0, 0, 0, 0, 0, 521, 520, 511, 529, 496, 521, 476, 491, 0, 0, 0, 0, 0, 0, 0, 0, 501, 526, 536, 517, 541, 525, 538, 519, 2, 2, 0, 0, 0, 0, 0, 0, 493, 538, 499, 521, 527, 495, 520, 526, 0, 0, 0, 0, 0, 0, 0, 0, 517, 536, 512, 469, 511, 516, 505, 499, 0, 0, 0, 0, 0, 0, 0, 0, 494, 508, 525, 487, 524, 521, 518, 504, 24, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4059, 4029, 4107, 4065, 4207, 4119, 4065, 4117] avg=4096.0000 max=4207 max_rank=4 min=4029 min_rank=1 imbalance_ratio=1.027100 max_load_reduction=28561 max_load_reduction_pct=0.871613 moved_to_echo=16508 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[491, 507, 543, 493, 529, 504, 527, 498, 0, 0, 0, 0, 0, 0, 0, 0, 515, 559, 540, 502, 503, 496, 507, 513, 5, 0, 0, 0, 0, 0, 0, 0, 561, 513, 486, 516, 480, 521, 507, 516, 12, 5, 0, 0, 0, 0, 0, 0, 517, 505, 473, 544, 484, 492, 515, 543, 0, 0, 0, 0, 0, 0, 0, 0, 474, 523, 528, 491, 513, 528, 526, 538, 7, 7, 0, 0, 0, 0, 0, 0, 527, 478, 508, 528, 506, 547, 501, 504, 0, 0, 0, 0, 0, 0, 0, 0, 487, 515, 503, 495, 512, 488, 541, 523, 0, 0, 0, 0, 0, 0, 0, 0, 453, 501, 510, 507, 510, 514, 484, 526, 28, 15, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4140, 4117, 4073, 4135, 4099, 4064, 4048] avg=4096.0000 max=4140 max_rank=1 min=4048 min_rank=7 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16346 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[513, 532, 538, 524, 477, 518, 542, 513, 0, 0, 0, 0, 0, 0, 0, 0, 457, 481, 515, 493, 485, 494, 519, 527, 3, 0, 0, 0, 0, 0, 0, 0, 511, 523, 528, 479, 507, 523, 517, 526, 9, 2, 0, 0, 0, 0, 0, 0, 500, 473, 529, 537, 520, 520, 493, 501, 0, 0, 0, 0, 0, 0, 0, 0, 500, 539, 501, 505, 492, 504, 506, 502, 3, 2, 0, 0, 0, 0, 0, 0, 514, 520, 529, 485, 555, 496, 522, 488, 0, 0, 0, 0, 0, 0, 0, 0, 491, 528, 515, 523, 515, 499, 486, 537, 0, 0, 0, 0, 0, 0, 0, 0, 529, 496, 508, 504, 535, 522, 534, 517, 25, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4157, 3974, 4125, 4073, 4054, 4109, 4094, 4182] avg=4096.0000 max=4182 max_rank=7 min=3974 min_rank=1 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16439 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[470, 517, 516, 501, 498, 520, 509, 529, 0, 0, 0, 0, 0, 0, 0, 0, 540, 487, 508, 533, 512, 522, 516, 503, 3, 0, 0, 0, 0, 0, 0, 0, 508, 493, 518, 534, 551, 533, 513, 486, 9, 2, 0, 0, 0, 0, 0, 0, 541, 511, 487, 501, 536, 495, 494, 520, 0, 0, 0, 0, 0, 0, 0, 0, 538, 494, 437, 504, 482, 506, 531, 519, 2, 2, 0, 0, 0, 0, 0, 0, 523, 508, 503, 491, 499, 510, 505, 519, 0, 0, 0, 0, 0, 0, 0, 0, 530, 504, 549, 511, 518, 519, 502, 504, 0, 0, 0, 0, 0, 0, 0, 0, 480, 547, 482, 531, 521, 516, 537, 493, 23, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4124, 4147, 4085, 4015, 4058, 4137, 4142] avg=4096.0000 max=4147 max_rank=2 min=4015 min_rank=4 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16352 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3968, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4352, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[541, 496, 456, 502, 519, 544, 503, 548, 516, 516, 491, 545, 514, 496, 512, 503, 545, 527, 491, 505, 487, 514, 517, 461, 526, 514, 523, 567, 529, 466, 534, 508, 499, 525, 488, 493, 511, 493, 573, 535, 498, 515, 507, 516, 486, 514, 494, 533, 496, 512, 518, 524, 537, 506, 493, 515, 497, 495, 527, 520, 506, 518, 481, 527] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[497, 531, 520, 523, 524, 493, 520, 547, 530, 531, 508, 547, 497, 499, 522, 501, 510, 507, 523, 529, 499, 509, 526, 497, 522, 490, 543, 481, 533, 505, 516, 483, 505, 495, 501, 502, 510, 543, 503, 550, 543, 493, 519, 503, 510, 509, 485, 505, 516, 505, 523, 485, 546, 531, 532, 469, 510, 510, 515, 489, 529, 488, 489, 492] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[514, 523, 471, 522, 487, 524, 522, 491, 475, 504, 519, 510, 504, 496, 540, 496, 499, 513, 500, 516, 478, 478, 532, 491, 516, 544, 509, 520, 492, 509, 525, 519, 503, 520, 500, 537, 566, 503, 522, 511, 509, 513, 516, 466, 511, 526, 557, 519, 535, 487, 517, 504, 487, 494, 538, 563, 518, 470, 515, 533, 544, 507, 477, 561] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[501, 535, 533, 523, 554, 550, 519, 499, 501, 506, 520, 545, 521, 522, 494, 518, 538, 507, 522, 489, 493, 546, 504, 523, 519, 530, 507, 483, 506, 514, 500, 502, 506, 473, 512, 531, 512, 534, 528, 517, 503, 473, 479, 456, 486, 517, 511, 499, 533, 536, 485, 478, 518, 535, 497, 498, 507, 512, 534, 525, 486, 500, 549, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[524, 498, 509, 554, 487, 506, 492, 544, 511, 514, 503, 494, 494, 488, 481, 498, 492, 528, 540, 513, 478, 526, 486, 493, 546, 480, 521, 561, 467, 513, 537, 532, 519, 497, 502, 522, 520, 525, 517, 533, 554, 499, 517, 532, 540, 501, 551, 493, 516, 479, 532, 497, 515, 526, 500, 500, 526, 536, 520, 518, 518, 498, 465, 490] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[506, 518, 508, 500, 530, 485, 511, 522, 488, 507, 515, 527, 499, 532, 506, 518, 513, 516, 500, 490, 507, 562, 491, 532, 520, 533, 525, 507, 520, 500, 488, 519, 507, 524, 535, 529, 493, 541, 500, 542, 519, 529, 510, 490, 529, 540, 498, 495, 522, 507, 456, 546, 522, 481, 500, 514, 526, 494, 479, 503, 511, 521, 504, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[520, 515, 521, 535, 541, 512, 490, 506, 499, 544, 498, 484, 510, 548, 559, 486, 514, 474, 514, 525, 508, 511, 514, 504, 496, 538, 530, 560, 506, 536, 499, 525, 505, 492, 519, 503, 504, 521, 501, 501, 512, 507, 548, 449, 532, 498, 502, 537, 501, 497, 523, 485, 494, 527, 519, 529, 467, 510, 517, 502, 494, 519, 517, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[485, 547, 516, 556, 517, 523, 536, 532, 489, 536, 499, 491, 558, 494, 480, 527, 525, 532, 488, 478, 547, 497, 513, 520, 503, 506, 514, 544, 522, 513, 484, 535, 488, 511, 530, 496, 491, 528, 476, 492, 503, 541, 520, 522, 540, 510, 505, 528, 495, 512, 526, 505, 505, 513, 464, 511, 480, 507, 541, 467, 500, 526, 511, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[497, 531, 520, 485, 524, 493, 520, 547, 0, 0, 0, 0, 0, 0, 0, 0, 530, 531, 508, 547, 497, 499, 522, 501, 6, 4, 0, 0, 0, 0, 0, 0, 510, 507, 523, 529, 499, 509, 526, 497, 11, 7, 0, 0, 0, 0, 0, 0, 522, 490, 543, 455, 533, 505, 516, 483, 0, 0, 0, 0, 0, 0, 0, 0, 505, 495, 501, 502, 510, 532, 503, 550, 0, 0, 0, 0, 0, 0, 0, 0, 543, 493, 519, 503, 510, 509, 485, 505, 5, 0, 0, 0, 0, 0, 0, 0, 516, 505, 523, 485, 546, 531, 532, 469, 15, 0, 0, 0, 0, 0, 0, 0, 510, 510, 515, 489, 529, 488, 489, 492, 27, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4117, 4145, 4118, 4047, 4098, 4072, 4122, 4049] avg=4096.0000 max=4145 max_rank=1 min=4047 min_rank=3 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16341 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[514, 523, 471, 484, 487, 524, 522, 491, 0, 0, 0, 0, 0, 0, 0, 0, 475, 504, 519, 510, 504, 496, 540, 496, 6, 5, 0, 0, 0, 0, 0, 0, 499, 513, 500, 516, 478, 478, 532, 491, 11, 8, 0, 0, 0, 0, 0, 0, 516, 544, 509, 491, 492, 509, 525, 519, 0, 0, 0, 0, 0, 0, 0, 0, 503, 520, 500, 537, 566, 492, 522, 511, 0, 0, 0, 0, 0, 0, 0, 0, 509, 513, 516, 466, 511, 526, 557, 519, 5, 0, 0, 0, 0, 0, 0, 0, 535, 487, 517, 504, 487, 494, 538, 563, 16, 0, 0, 0, 0, 0, 0, 0, 518, 470, 515, 533, 544, 507, 477, 561, 27, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4016, 4055, 4026, 4105, 4151, 4122, 4141, 4152] avg=4096.0000 max=4152 max_rank=7 min=4016 min_rank=0 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16566 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[524, 498, 509, 515, 487, 506, 492, 544, 0, 0, 0, 0, 0, 0, 0, 0, 511, 514, 503, 494, 494, 488, 481, 498, 6, 5, 0, 0, 0, 0, 0, 0, 492, 528, 540, 513, 478, 526, 486, 493, 11, 9, 0, 0, 0, 0, 0, 0, 546, 480, 521, 529, 467, 513, 537, 532, 0, 0, 0, 0, 0, 0, 0, 0, 519, 497, 502, 522, 520, 514, 517, 533, 0, 0, 0, 0, 0, 0, 0, 0, 554, 499, 517, 532, 540, 501, 551, 493, 5, 0, 0, 0, 0, 0, 0, 0, 516, 479, 532, 497, 515, 526, 500, 500, 18, 0, 0, 0, 0, 0, 0, 0, 526, 536, 520, 518, 518, 498, 465, 490, 28, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 3994, 4076, 4125, 4124, 4192, 4083, 4099] avg=4096.0000 max=4192 max_rank=5 min=3994 min_rank=1 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16498 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[541, 496, 456, 458, 519, 544, 503, 548, 0, 0, 0, 0, 0, 0, 0, 0, 516, 516, 491, 545, 514, 496, 512, 503, 9, 9, 0, 0, 0, 0, 0, 0, 545, 527, 491, 505, 487, 514, 517, 461, 14, 12, 0, 0, 0, 0, 0, 0, 526, 514, 523, 525, 529, 466, 534, 508, 0, 0, 0, 0, 0, 0, 0, 0, 499, 525, 488, 493, 511, 473, 573, 535, 0, 0, 0, 0, 0, 0, 0, 0, 498, 515, 507, 516, 486, 514, 494, 533, 11, 0, 0, 0, 0, 0, 0, 0, 496, 512, 518, 524, 537, 506, 493, 515, 21, 0, 0, 0, 0, 0, 0, 0, 497, 495, 527, 520, 506, 518, 481, 527, 30, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4065, 4111, 4073, 4125, 4097, 4074, 4122, 4101] avg=4096.0000 max=4125 max_rank=3 min=4065 min_rank=0 imbalance_ratio=1.007080 max_load_reduction=28643 max_load_reduction_pct=0.874115 moved_to_echo=16394 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[501, 535, 533, 485, 554, 550, 519, 499, 0, 0, 0, 0, 0, 0, 0, 0, 501, 506, 520, 545, 521, 522, 494, 518, 6, 4, 0, 0, 0, 0, 0, 0, 538, 507, 522, 489, 493, 546, 504, 523, 11, 8, 0, 0, 0, 0, 0, 0, 519, 530, 507, 456, 506, 514, 500, 502, 0, 0, 0, 0, 0, 0, 0, 0, 506, 473, 512, 531, 512, 523, 528, 517, 0, 0, 0, 0, 0, 0, 0, 0, 503, 473, 479, 456, 486, 517, 511, 499, 5, 0, 0, 0, 0, 0, 0, 0, 533, 536, 485, 478, 518, 535, 497, 498, 15, 0, 0, 0, 0, 0, 0, 0, 507, 512, 534, 525, 486, 500, 549, 514, 27, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4176, 4137, 4141, 4034, 4102, 3929, 4095, 4154] avg=4096.0000 max=4176 max_rank=0 min=3929 min_rank=5 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16280 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[506, 518, 508, 465, 530, 485, 511, 522, 0, 0, 0, 0, 0, 0, 0, 0, 488, 507, 515, 527, 499, 532, 506, 518, 6, 4, 0, 0, 0, 0, 0, 0, 513, 516, 500, 490, 507, 562, 491, 532, 10, 8, 0, 0, 0, 0, 0, 0, 520, 533, 525, 479, 520, 500, 488, 519, 0, 0, 0, 0, 0, 0, 0, 0, 507, 524, 535, 529, 493, 530, 500, 542, 0, 0, 0, 0, 0, 0, 0, 0, 519, 529, 510, 490, 529, 540, 498, 495, 5, 0, 0, 0, 0, 0, 0, 0, 522, 507, 456, 546, 522, 481, 500, 514, 16, 0, 0, 0, 0, 0, 0, 0, 526, 494, 479, 503, 511, 521, 504, 506, 25, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4045, 4102, 4129, 4084, 4160, 4115, 4064, 4069] avg=4096.0000 max=4160 max_rank=4 min=4045 min_rank=0 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16408 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[520, 515, 521, 497, 541, 512, 490, 506, 0, 0, 0, 0, 0, 0, 0, 0, 499, 544, 498, 484, 510, 548, 559, 486, 6, 5, 0, 0, 0, 0, 0, 0, 514, 474, 514, 525, 508, 511, 514, 504, 11, 9, 0, 0, 0, 0, 0, 0, 496, 538, 530, 528, 506, 536, 499, 525, 0, 0, 0, 0, 0, 0, 0, 0, 505, 492, 519, 503, 504, 510, 501, 501, 0, 0, 0, 0, 0, 0, 0, 0, 512, 507, 548, 449, 532, 498, 502, 537, 5, 0, 0, 0, 0, 0, 0, 0, 501, 497, 523, 485, 494, 527, 519, 529, 18, 0, 0, 0, 0, 0, 0, 0, 467, 510, 517, 502, 494, 519, 517, 514, 27, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4139, 4084, 4158, 4035, 4090, 4093, 4067] avg=4096.0000 max=4158 max_rank=3 min=4035 min_rank=4 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16285 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[485, 547, 516, 516, 517, 523, 536, 532, 0, 0, 0, 0, 0, 0, 0, 0, 489, 536, 499, 491, 558, 494, 480, 527, 6, 5, 0, 0, 0, 0, 0, 0, 525, 532, 488, 478, 547, 497, 513, 520, 12, 9, 0, 0, 0, 0, 0, 0, 503, 506, 514, 513, 522, 513, 484, 535, 0, 0, 0, 0, 0, 0, 0, 0, 488, 511, 530, 496, 491, 517, 476, 492, 0, 0, 0, 0, 0, 0, 0, 0, 503, 541, 520, 522, 540, 510, 505, 528, 5, 0, 0, 0, 0, 0, 0, 0, 495, 512, 526, 505, 505, 513, 464, 511, 17, 0, 0, 0, 0, 0, 0, 0, 480, 507, 541, 467, 500, 526, 511, 517, 28, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4172, 4085, 4121, 4090, 4001, 4174, 4048, 4077] avg=4096.0000 max=4174 max_rank=5 min=4001 min_rank=4 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16300 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 3968, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 3968, 4224, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[507, 533, 519, 512, 521, 541, 519, 523, 484, 495, 528, 535, 508, 488, 500, 533, 538, 523, 525, 552, 465, 514, 478, 489, 503, 511, 492, 485, 499, 474, 538, 517, 474, 533, 557, 502, 521, 521, 560, 539, 468, 480, 531, 508, 480, 506, 511, 564, 553, 451, 538, 500, 472, 558, 503, 484, 490, 533, 533, 496, 490, 537, 488, 538] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[504, 515, 533, 457, 489, 495, 525, 505, 541, 515, 513, 478, 521, 515, 492, 500, 532, 475, 500, 508, 509, 492, 512, 517, 524, 547, 492, 502, 511, 521, 520, 534, 513, 492, 509, 532, 525, 495, 506, 509, 527, 512, 524, 542, 537, 534, 511, 554, 497, 468, 532, 556, 500, 487, 506, 556, 511, 527, 501, 539, 470, 519, 493, 490] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[480, 507, 498, 519, 502, 525, 510, 489, 495, 505, 515, 501, 513, 545, 531, 523, 489, 518, 535, 549, 571, 508, 523, 507, 513, 516, 502, 538, 526, 500, 521, 500, 496, 505, 534, 498, 499, 467, 531, 529, 504, 497, 521, 541, 478, 518, 513, 504, 511, 532, 497, 528, 505, 452, 526, 495, 519, 499, 503, 530, 475, 506, 550, 531] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[514, 512, 516, 516, 534, 491, 511, 517, 535, 487, 514, 470, 518, 515, 513, 480, 500, 470, 508, 516, 488, 517, 518, 567, 566, 573, 518, 522, 514, 496, 476, 509, 519, 507, 537, 508, 532, 511, 536, 502, 482, 523, 490, 511, 518, 511, 515, 507, 527, 488, 526, 526, 492, 474, 507, 554, 483, 538, 528, 523, 525, 493, 499, 475] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[511, 513, 536, 546, 535, 525, 528, 541, 521, 497, 547, 503, 514, 498, 479, 540, 540, 460, 507, 508, 522, 555, 475, 517, 486, 503, 501, 482, 508, 521, 513, 529, 526, 497, 510, 506, 506, 501, 527, 512, 486, 489, 522, 508, 540, 535, 477, 503, 460, 504, 515, 484, 538, 526, 493, 520, 565, 537, 489, 517, 498, 520, 489, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[525, 501, 514, 531, 474, 500, 476, 517, 515, 546, 517, 510, 474, 493, 491, 486, 506, 460, 512, 514, 522, 523, 482, 473, 553, 516, 516, 505, 516, 557, 490, 514, 507, 535, 551, 511, 534, 554, 494, 491, 516, 511, 546, 498, 525, 519, 540, 551, 533, 515, 503, 492, 467, 515, 491, 523, 494, 480, 537, 556, 495, 516, 516, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[490, 497, 521, 509, 514, 497, 525, 499, 487, 526, 485, 547, 518, 536, 522, 520, 503, 511, 502, 481, 538, 492, 517, 496, 524, 530, 528, 501, 532, 518, 488, 510, 551, 508, 520, 526, 479, 498, 530, 504, 494, 501, 551, 518, 510, 471, 506, 543, 509, 490, 536, 505, 505, 554, 516, 509, 493, 535, 493, 532, 521, 519, 473, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[516, 504, 480, 496, 499, 539, 528, 515, 548, 509, 522, 518, 513, 528, 513, 541, 530, 526, 513, 525, 497, 543, 511, 495, 493, 491, 517, 493, 509, 559, 517, 506, 511, 474, 500, 495, 508, 516, 535, 530, 484, 509, 532, 492, 500, 463, 477, 524, 525, 493, 475, 547, 515, 508, 505, 475, 548, 498, 525, 520, 508, 530, 518, 534] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[507, 533, 519, 512, 521, 541, 519, 523, 3, 0, 0, 0, 0, 0, 0, 0, 484, 495, 528, 535, 508, 488, 500, 533, 0, 0, 0, 0, 0, 0, 0, 0, 538, 523, 525, 552, 465, 514, 478, 489, 9, 2, 0, 0, 0, 0, 0, 0, 503, 497, 492, 485, 499, 474, 538, 517, 0, 0, 0, 0, 0, 0, 0, 0, 474, 533, 557, 502, 521, 521, 537, 539, 0, 0, 0, 0, 0, 0, 0, 0, 468, 480, 531, 508, 480, 506, 511, 564, 0, 0, 0, 0, 0, 0, 0, 0, 553, 451, 538, 500, 472, 558, 503, 484, 23, 5, 0, 0, 0, 0, 0, 0, 490, 533, 533, 491, 490, 537, 488, 538, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4178, 4071, 4095, 4005, 4184, 4048, 4087, 4100] avg=4096.0000 max=4184 max_rank=4 min=4005 min_rank=3 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16419 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[480, 507, 498, 519, 502, 525, 510, 489, 3, 0, 0, 0, 0, 0, 0, 0, 495, 505, 515, 501, 513, 545, 531, 523, 0, 0, 0, 0, 0, 0, 0, 0, 489, 518, 535, 549, 571, 508, 523, 507, 9, 3, 0, 0, 0, 0, 0, 0, 513, 502, 502, 538, 526, 500, 521, 500, 0, 0, 0, 0, 0, 0, 0, 0, 496, 505, 534, 498, 499, 467, 509, 529, 0, 0, 0, 0, 0, 0, 0, 0, 504, 497, 521, 541, 478, 518, 513, 504, 0, 0, 0, 0, 0, 0, 0, 0, 511, 532, 497, 528, 505, 452, 526, 495, 22, 5, 0, 0, 0, 0, 0, 0, 519, 499, 503, 524, 475, 506, 550, 531, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4033, 4128, 4212, 4102, 4037, 4076, 4073, 4107] avg=4096.0000 max=4212 max_rank=2 min=4033 min_rank=0 imbalance_ratio=1.028320 max_load_reduction=28556 max_load_reduction_pct=0.871460 moved_to_echo=16293 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[514, 512, 516, 516, 534, 491, 511, 517, 3, 0, 0, 0, 0, 0, 0, 0, 535, 487, 514, 470, 518, 515, 513, 480, 0, 0, 0, 0, 0, 0, 0, 0, 500, 470, 508, 516, 488, 517, 518, 567, 10, 2, 0, 0, 0, 0, 0, 0, 566, 558, 518, 522, 514, 496, 476, 509, 0, 0, 0, 0, 0, 0, 0, 0, 519, 507, 537, 508, 532, 511, 514, 502, 0, 0, 0, 0, 0, 0, 0, 0, 482, 523, 490, 511, 518, 511, 515, 507, 0, 0, 0, 0, 0, 0, 0, 0, 527, 488, 526, 526, 492, 474, 507, 554, 22, 5, 0, 0, 0, 0, 0, 0, 483, 538, 528, 518, 525, 493, 499, 475, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4032, 4096, 4159, 4130, 4057, 4121, 4059] avg=4096.0000 max=4159 max_rank=3 min=4032 min_rank=1 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[511, 513, 536, 546, 535, 525, 528, 541, 6, 0, 0, 0, 0, 0, 0, 0, 521, 497, 545, 503, 514, 498, 479, 540, 0, 0, 0, 0, 0, 0, 0, 0, 540, 460, 507, 508, 522, 555, 475, 517, 12, 6, 0, 0, 0, 0, 0, 0, 486, 483, 501, 482, 508, 521, 513, 529, 0, 0, 0, 0, 0, 0, 0, 0, 526, 497, 510, 506, 506, 501, 501, 512, 0, 0, 0, 0, 0, 0, 0, 0, 486, 489, 522, 508, 540, 535, 477, 503, 3, 2, 0, 0, 0, 0, 0, 0, 460, 504, 515, 484, 538, 526, 493, 520, 26, 8, 0, 0, 0, 0, 0, 0, 565, 537, 489, 502, 498, 520, 489, 507, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4241, 4097, 4102, 4023, 4059, 4065, 4074, 4107] avg=4096.0000 max=4241 max_rank=0 min=4023 min_rank=3 imbalance_ratio=1.035400 max_load_reduction=28527 max_load_reduction_pct=0.870575 moved_to_echo=16305 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[525, 501, 514, 531, 474, 500, 476, 517, 3, 0, 0, 0, 0, 0, 0, 0, 515, 546, 517, 510, 474, 493, 491, 486, 0, 0, 0, 0, 0, 0, 0, 0, 506, 460, 512, 514, 522, 523, 482, 473, 9, 3, 0, 0, 0, 0, 0, 0, 553, 502, 516, 505, 516, 557, 490, 514, 0, 0, 0, 0, 0, 0, 0, 0, 507, 535, 551, 511, 534, 554, 474, 491, 0, 0, 0, 0, 0, 0, 0, 0, 516, 511, 546, 498, 525, 519, 540, 551, 0, 0, 0, 0, 0, 0, 0, 0, 533, 515, 503, 492, 467, 515, 491, 523, 20, 5, 0, 0, 0, 0, 0, 0, 494, 480, 537, 550, 495, 516, 516, 523, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4041, 4032, 4004, 4153, 4157, 4206, 4064, 4111] avg=4096.0000 max=4206 max_rank=5 min=4004 min_rank=2 imbalance_ratio=1.026855 max_load_reduction=28562 max_load_reduction_pct=0.871643 moved_to_echo=16538 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[504, 515, 533, 457, 489, 495, 525, 505, 3, 0, 0, 0, 0, 0, 0, 0, 541, 515, 513, 478, 521, 515, 492, 500, 0, 0, 0, 0, 0, 0, 0, 0, 532, 475, 500, 508, 509, 492, 512, 517, 9, 3, 0, 0, 0, 0, 0, 0, 524, 533, 492, 502, 511, 521, 520, 534, 0, 0, 0, 0, 0, 0, 0, 0, 513, 492, 509, 532, 525, 495, 485, 509, 0, 0, 0, 0, 0, 0, 0, 0, 527, 512, 524, 542, 537, 534, 511, 554, 0, 0, 0, 0, 0, 0, 0, 0, 497, 468, 532, 556, 500, 487, 506, 556, 21, 5, 0, 0, 0, 0, 0, 0, 511, 527, 501, 533, 470, 519, 493, 490, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4026, 4075, 4057, 4137, 4060, 4241, 4128, 4044] avg=4096.0000 max=4241 max_rank=5 min=4026 min_rank=0 imbalance_ratio=1.035400 max_load_reduction=28527 max_load_reduction_pct=0.870575 moved_to_echo=16473 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[490, 497, 521, 509, 514, 497, 525, 499, 3, 0, 0, 0, 0, 0, 0, 0, 487, 526, 485, 547, 518, 536, 522, 520, 0, 0, 0, 0, 0, 0, 0, 0, 503, 511, 502, 481, 538, 492, 517, 496, 9, 3, 0, 0, 0, 0, 0, 0, 524, 516, 528, 501, 532, 518, 488, 510, 0, 0, 0, 0, 0, 0, 0, 0, 551, 508, 520, 526, 479, 498, 508, 504, 0, 0, 0, 0, 0, 0, 0, 0, 494, 501, 551, 518, 510, 471, 506, 543, 0, 0, 0, 0, 0, 0, 0, 0, 509, 490, 536, 505, 505, 554, 516, 509, 22, 5, 0, 0, 0, 0, 0, 0, 493, 535, 493, 526, 521, 519, 473, 504, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4055, 4141, 4052, 4117, 4094, 4094, 4151, 4064] avg=4096.0000 max=4151 max_rank=6 min=4052 min_rank=2 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16403 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[516, 504, 480, 496, 499, 539, 528, 515, 3, 0, 0, 0, 0, 0, 0, 0, 548, 509, 522, 518, 513, 528, 513, 541, 0, 0, 0, 0, 0, 0, 0, 0, 530, 526, 513, 525, 497, 543, 511, 495, 8, 2, 0, 0, 0, 0, 0, 0, 493, 478, 517, 493, 509, 559, 517, 506, 0, 0, 0, 0, 0, 0, 0, 0, 511, 474, 500, 495, 508, 516, 513, 530, 0, 0, 0, 0, 0, 0, 0, 0, 484, 509, 532, 492, 500, 463, 477, 524, 0, 0, 0, 0, 0, 0, 0, 0, 525, 493, 475, 547, 515, 508, 505, 475, 22, 5, 0, 0, 0, 0, 0, 0, 548, 498, 525, 515, 508, 530, 518, 534, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4192, 4150, 4072, 4047, 3981, 4070, 4176] avg=4096.0000 max=4192 max_rank=1 min=3981 min_rank=5 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16274 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 3968, 4224, 4224, 4096, 4096, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 3968, 4224, 4224, 4224, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3968, 4096, 4224, 4224, 4096, 4096, 4096, 4352, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[479, 549, 534, 565, 482, 518, 518, 532, 488, 523, 496, 483, 506, 521, 509, 552, 492, 529, 495, 494, 517, 500, 552, 528, 536, 522, 493, 518, 510, 503, 511, 517, 525, 555, 500, 484, 488, 499, 492, 524, 491, 531, 531, 529, 508, 496, 492, 545, 507, 469, 499, 526, 502, 532, 521, 503, 526, 499, 498, 511, 511, 533, 473, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[512, 488, 507, 501, 532, 485, 520, 517, 530, 481, 526, 495, 520, 512, 479, 491, 479, 535, 536, 516, 541, 494, 541, 542, 540, 490, 563, 527, 552, 494, 504, 496, 493, 513, 531, 487, 485, 511, 528, 513, 540, 533, 509, 526, 493, 515, 503, 528, 511, 501, 545, 491, 512, 532, 477, 504, 513, 481, 512, 500, 517, 538, 508, 472] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[523, 528, 497, 529, 513, 535, 518, 474, 540, 528, 534, 495, 518, 478, 538, 503, 501, 504, 511, 517, 507, 525, 528, 572, 509, 514, 538, 478, 531, 523, 511, 496, 458, 501, 494, 480, 512, 501, 477, 496, 539, 514, 510, 525, 533, 500, 477, 502, 517, 539, 519, 500, 500, 532, 527, 512, 514, 531, 486, 493, 505, 522, 512, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[550, 503, 514, 482, 511, 497, 542, 539, 516, 534, 513, 519, 536, 517, 521, 466, 539, 529, 493, 515, 495, 530, 546, 485, 546, 495, 494, 519, 506, 509, 481, 480, 496, 478, 581, 515, 493, 488, 524, 513, 476, 542, 491, 531, 536, 508, 516, 457, 486, 504, 474, 517, 532, 545, 511, 521, 514, 501, 512, 502, 539, 508, 515, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[484, 512, 494, 499, 480, 509, 462, 496, 545, 572, 535, 500, 507, 532, 535, 507, 516, 538, 523, 549, 526, 530, 534, 510, 530, 506, 529, 509, 539, 477, 481, 514, 547, 536, 512, 510, 487, 510, 501, 503, 523, 518, 502, 491, 488, 473, 492, 526, 508, 512, 557, 510, 525, 505, 492, 507, 499, 509, 504, 486, 482, 511, 547, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[525, 519, 515, 494, 507, 574, 477, 538, 471, 531, 486, 495, 463, 520, 506, 549, 473, 505, 528, 499, 530, 473, 485, 504, 528, 479, 516, 491, 533, 529, 502, 570, 493, 524, 525, 510, 489, 488, 530, 513, 501, 509, 501, 529, 497, 513, 523, 540, 534, 520, 509, 509, 499, 493, 539, 534, 507, 518, 541, 514, 521, 545, 473, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[472, 523, 527, 487, 531, 456, 556, 495, 522, 533, 484, 503, 535, 521, 503, 494, 502, 525, 507, 534, 514, 516, 464, 509, 502, 537, 515, 491, 518, 502, 520, 533, 512, 500, 525, 489, 509, 535, 538, 514, 495, 524, 527, 538, 516, 500, 510, 478, 484, 540, 537, 524, 545, 504, 500, 493, 491, 548, 520, 535, 514, 493, 517, 482] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[564, 496, 529, 502, 503, 521, 525, 494, 503, 507, 502, 541, 505, 522, 492, 527, 532, 523, 528, 487, 506, 551, 533, 515, 511, 512, 488, 501, 488, 501, 538, 500, 507, 516, 515, 493, 475, 496, 491, 509, 491, 504, 523, 536, 515, 503, 494, 503, 510, 535, 517, 550, 505, 506, 528, 506, 506, 503, 504, 510, 506, 486, 541, 537] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[512, 488, 507, 501, 532, 485, 520, 517, 1, 0, 0, 0, 0, 0, 0, 0, 530, 476, 526, 495, 520, 512, 479, 491, 0, 0, 0, 0, 0, 0, 0, 0, 479, 499, 536, 516, 541, 494, 541, 542, 0, 0, 0, 0, 0, 0, 0, 0, 537, 490, 563, 527, 552, 494, 504, 496, 0, 0, 0, 0, 0, 0, 0, 0, 493, 513, 531, 487, 485, 511, 528, 513, 36, 5, 0, 0, 0, 0, 0, 0, 540, 533, 509, 526, 493, 515, 503, 528, 4, 2, 0, 0, 0, 0, 0, 0, 511, 501, 528, 491, 512, 532, 477, 504, 0, 0, 0, 0, 0, 0, 0, 0, 513, 481, 512, 500, 517, 538, 508, 472, 12, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4029, 4148, 4163, 4102, 4153, 4056, 4054] avg=4096.0000 max=4163 max_rank=3 min=4029 min_rank=1 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16365 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[479, 549, 534, 565, 482, 518, 518, 532, 1, 0, 0, 0, 0, 0, 0, 0, 488, 518, 496, 483, 506, 521, 509, 552, 0, 0, 0, 0, 0, 0, 0, 0, 492, 493, 495, 494, 517, 500, 552, 528, 0, 0, 0, 0, 0, 0, 0, 0, 533, 522, 493, 518, 510, 503, 511, 517, 0, 0, 0, 0, 0, 0, 0, 0, 525, 555, 500, 484, 488, 499, 492, 524, 36, 5, 0, 0, 0, 0, 0, 0, 491, 531, 531, 529, 508, 496, 492, 545, 4, 2, 0, 0, 0, 0, 0, 0, 507, 469, 483, 526, 502, 532, 521, 503, 0, 0, 0, 0, 0, 0, 0, 0, 526, 499, 498, 511, 511, 533, 473, 496, 11, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4178, 4073, 4071, 4107, 4108, 4129, 4043, 4059] avg=4096.0000 max=4178 max_rank=0 min=4043 min_rank=6 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16339 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[523, 528, 497, 529, 513, 535, 518, 474, 0, 0, 0, 0, 0, 0, 0, 0, 540, 523, 534, 495, 518, 478, 538, 503, 0, 0, 0, 0, 0, 0, 0, 0, 501, 470, 511, 517, 507, 525, 528, 572, 0, 0, 0, 0, 0, 0, 0, 0, 507, 514, 538, 478, 531, 523, 511, 496, 0, 0, 0, 0, 0, 0, 0, 0, 458, 501, 494, 480, 512, 501, 477, 496, 34, 5, 0, 0, 0, 0, 0, 0, 539, 514, 510, 525, 533, 500, 477, 502, 4, 2, 0, 0, 0, 0, 0, 0, 517, 539, 503, 500, 500, 532, 527, 512, 0, 0, 0, 0, 0, 0, 0, 0, 514, 531, 486, 493, 505, 522, 512, 524, 11, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4117, 4129, 4131, 4098, 3958, 4106, 4130, 4099] avg=4096.0000 max=4131 max_rank=2 min=3958 min_rank=4 imbalance_ratio=1.008545 max_load_reduction=28637 max_load_reduction_pct=0.873932 moved_to_echo=16293 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[550, 503, 514, 482, 511, 497, 542, 539, 4, 0, 0, 0, 0, 0, 0, 0, 516, 522, 513, 519, 536, 517, 521, 466, 0, 0, 0, 0, 0, 0, 0, 0, 539, 489, 493, 515, 495, 530, 546, 485, 0, 0, 0, 0, 0, 0, 0, 0, 536, 495, 494, 519, 506, 509, 481, 480, 0, 0, 0, 0, 0, 0, 0, 0, 496, 478, 581, 515, 493, 488, 524, 513, 40, 7, 0, 0, 0, 0, 0, 0, 476, 542, 491, 531, 536, 508, 516, 457, 9, 6, 0, 0, 0, 0, 0, 0, 486, 504, 452, 517, 532, 545, 511, 521, 0, 0, 0, 0, 0, 0, 0, 0, 514, 501, 512, 502, 539, 508, 515, 520, 15, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4142, 4110, 4092, 4020, 4135, 4072, 4068, 4129] avg=4096.0000 max=4142 max_rank=0 min=4020 min_rank=3 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16404 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[525, 519, 515, 494, 507, 574, 477, 538, 1, 0, 0, 0, 0, 0, 0, 0, 471, 526, 486, 495, 463, 520, 506, 549, 0, 0, 0, 0, 0, 0, 0, 0, 473, 471, 528, 499, 530, 473, 485, 504, 0, 0, 0, 0, 0, 0, 0, 0, 525, 479, 516, 491, 533, 529, 502, 570, 0, 0, 0, 0, 0, 0, 0, 0, 493, 524, 525, 510, 489, 488, 530, 513, 34, 5, 0, 0, 0, 0, 0, 0, 501, 509, 501, 529, 497, 513, 523, 540, 4, 2, 0, 0, 0, 0, 0, 0, 534, 520, 493, 509, 499, 493, 539, 534, 0, 0, 0, 0, 0, 0, 0, 0, 507, 518, 541, 514, 521, 545, 473, 512, 11, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4150, 4016, 3963, 4145, 4111, 4119, 4121, 4143] avg=4096.0000 max=4150 max_rank=0 min=3963 min_rank=2 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16494 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[484, 512, 494, 499, 480, 509, 462, 496, 1, 0, 0, 0, 0, 0, 0, 0, 545, 566, 535, 500, 507, 532, 535, 507, 0, 0, 0, 0, 0, 0, 0, 0, 516, 501, 523, 549, 526, 530, 534, 510, 0, 0, 0, 0, 0, 0, 0, 0, 527, 506, 529, 509, 539, 477, 481, 514, 0, 0, 0, 0, 0, 0, 0, 0, 547, 536, 512, 510, 487, 510, 501, 503, 37, 5, 0, 0, 0, 0, 0, 0, 523, 518, 502, 491, 488, 473, 492, 526, 5, 2, 0, 0, 0, 0, 0, 0, 508, 512, 540, 510, 525, 505, 492, 507, 0, 0, 0, 0, 0, 0, 0, 0, 499, 509, 504, 486, 482, 511, 547, 515, 12, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3937, 4227, 4189, 4082, 4148, 4020, 4099, 4066] avg=4096.0000 max=4227 max_rank=1 min=3937 min_rank=0 imbalance_ratio=1.031982 max_load_reduction=28541 max_load_reduction_pct=0.871002 moved_to_echo=16333 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[472, 523, 527, 487, 531, 456, 556, 495, 0, 0, 0, 0, 0, 0, 0, 0, 522, 528, 484, 503, 535, 521, 503, 494, 0, 0, 0, 0, 0, 0, 0, 0, 502, 489, 507, 534, 514, 516, 464, 509, 0, 0, 0, 0, 0, 0, 0, 0, 500, 537, 515, 491, 518, 502, 520, 533, 0, 0, 0, 0, 0, 0, 0, 0, 512, 500, 525, 489, 509, 535, 538, 514, 36, 5, 0, 0, 0, 0, 0, 0, 495, 524, 527, 538, 516, 500, 510, 478, 4, 2, 0, 0, 0, 0, 0, 0, 484, 540, 520, 524, 545, 504, 500, 493, 0, 0, 0, 0, 0, 0, 0, 0, 491, 548, 520, 535, 514, 493, 517, 482, 12, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4047, 4090, 4035, 4116, 4163, 4094, 4110, 4113] avg=4096.0000 max=4163 max_rank=4 min=4035 min_rank=2 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16480 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[564, 496, 529, 502, 503, 521, 525, 494, 0, 0, 0, 0, 0, 0, 0, 0, 503, 502, 502, 541, 505, 522, 492, 527, 0, 0, 0, 0, 0, 0, 0, 0, 532, 487, 528, 487, 506, 551, 533, 515, 0, 0, 0, 0, 0, 0, 0, 0, 509, 512, 488, 501, 488, 501, 538, 500, 0, 0, 0, 0, 0, 0, 0, 0, 507, 516, 515, 493, 475, 496, 491, 509, 36, 5, 0, 0, 0, 0, 0, 0, 491, 504, 523, 536, 515, 503, 494, 503, 4, 2, 0, 0, 0, 0, 0, 0, 510, 535, 501, 550, 505, 506, 528, 506, 0, 0, 0, 0, 0, 0, 0, 0, 506, 503, 504, 510, 506, 486, 541, 537, 11, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4134, 4094, 4139, 4037, 4043, 4075, 4141, 4105] avg=4096.0000 max=4141 max_rank=6 min=4037 min_rank=3 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16364 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 3968, 3968, 4096, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 3968, 4224, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[492, 476, 525, 477, 519, 475, 545, 549, 547, 525, 521, 536, 507, 522, 532, 527, 527, 546, 506, 518, 531, 500, 519, 498, 523, 491, 517, 515, 520, 517, 549, 479, 511, 549, 510, 517, 510, 518, 455, 490, 518, 493, 508, 537, 536, 491, 484, 502, 499, 511, 510, 476, 522, 533, 527, 479, 507, 490, 512, 527, 499, 507, 525, 484] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[477, 480, 504, 535, 539, 514, 518, 503, 513, 471, 535, 529, 526, 493, 516, 509, 467, 512, 486, 516, 501, 511, 482, 500, 509, 511, 542, 529, 491, 496, 491, 495, 509, 524, 544, 557, 503, 497, 509, 543, 491, 502, 525, 537, 536, 538, 536, 536, 522, 508, 524, 525, 503, 498, 478, 524, 483, 529, 515, 522, 496, 519, 529, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[536, 528, 512, 485, 496, 515, 541, 530, 495, 529, 564, 460, 496, 491, 522, 539, 511, 503, 480, 536, 507, 520, 518, 478, 518, 535, 535, 495, 507, 526, 522, 535, 511, 519, 462, 581, 478, 540, 497, 515, 542, 481, 508, 469, 553, 527, 520, 514, 508, 504, 503, 522, 480, 535, 485, 514, 532, 484, 502, 502, 478, 514, 526, 497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[505, 496, 514, 500, 531, 528, 523, 483, 482, 532, 513, 508, 483, 490, 541, 523, 535, 506, 543, 480, 513, 476, 508, 567, 497, 514, 515, 520, 483, 504, 511, 552, 526, 516, 493, 524, 535, 545, 490, 498, 499, 527, 518, 534, 516, 513, 507, 475, 506, 516, 484, 506, 506, 567, 517, 486, 500, 487, 535, 544, 506, 504, 500, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[552, 504, 476, 570, 508, 472, 492, 491, 516, 491, 533, 501, 486, 517, 541, 506, 548, 518, 513, 505, 556, 513, 519, 518, 440, 501, 507, 506, 508, 559, 504, 519, 510, 544, 520, 519, 543, 555, 504, 527, 493, 488, 480, 524, 525, 520, 524, 488, 511, 510, 493, 488, 556, 509, 472, 472, 546, 506, 543, 489, 526, 504, 480, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[467, 514, 489, 536, 574, 512, 504, 505, 507, 542, 517, 495, 494, 517, 495, 521, 461, 493, 507, 538, 538, 481, 500, 536, 495, 502, 505, 499, 538, 525, 500, 519, 538, 511, 491, 524, 526, 479, 497, 500, 516, 515, 531, 490, 509, 512, 489, 537, 487, 518, 516, 469, 518, 532, 502, 544, 528, 536, 475, 539, 492, 527, 577, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[535, 504, 532, 515, 506, 498, 512, 489, 503, 490, 525, 484, 524, 518, 521, 485, 538, 550, 471, 502, 507, 519, 473, 496, 499, 532, 523, 540, 519, 509, 515, 558, 518, 524, 499, 515, 473, 498, 515, 521, 506, 522, 538, 514, 498, 486, 559, 529, 532, 523, 535, 507, 526, 517, 522, 450, 502, 502, 524, 489, 487, 523, 506, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[488, 514, 476, 514, 532, 526, 510, 510, 538, 517, 502, 573, 456, 566, 520, 533, 498, 529, 511, 560, 530, 529, 506, 503, 510, 506, 518, 502, 523, 530, 540, 500, 517, 503, 530, 508, 473, 503, 498, 533, 467, 489, 483, 517, 522, 492, 514, 495, 521, 494, 504, 517, 496, 498, 542, 521, 512, 522, 517, 501, 508, 498, 485, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[477, 480, 504, 535, 539, 514, 518, 503, 11, 0, 0, 0, 0, 0, 0, 0, 513, 471, 517, 529, 526, 493, 516, 509, 0, 0, 0, 0, 0, 0, 0, 0, 467, 512, 486, 516, 501, 511, 482, 500, 0, 0, 0, 0, 0, 0, 0, 0, 509, 511, 542, 529, 491, 482, 491, 495, 0, 0, 0, 0, 0, 0, 0, 0, 509, 524, 544, 542, 503, 497, 509, 543, 0, 0, 0, 0, 0, 0, 0, 0, 491, 502, 525, 537, 536, 538, 536, 536, 3, 0, 0, 0, 0, 0, 0, 0, 522, 508, 524, 525, 503, 498, 478, 524, 18, 5, 0, 0, 0, 0, 0, 0, 483, 529, 515, 522, 496, 519, 529, 505, 9, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4074, 3975, 4050, 4171, 4204, 4105, 4108] avg=4096.0000 max=4204 max_rank=5 min=3975 min_rank=2 imbalance_ratio=1.026367 max_load_reduction=28564 max_load_reduction_pct=0.871704 moved_to_echo=16588 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[492, 476, 525, 477, 519, 475, 545, 549, 10, 0, 0, 0, 0, 0, 0, 0, 547, 525, 504, 536, 507, 522, 532, 527, 0, 0, 0, 0, 0, 0, 0, 0, 527, 546, 506, 518, 531, 500, 519, 498, 0, 0, 0, 0, 0, 0, 0, 0, 523, 491, 517, 515, 520, 502, 549, 479, 0, 0, 0, 0, 0, 0, 0, 0, 511, 549, 510, 504, 510, 518, 455, 490, 0, 0, 0, 0, 0, 0, 0, 0, 518, 493, 508, 537, 536, 491, 484, 502, 2, 0, 0, 0, 0, 0, 0, 0, 499, 511, 510, 476, 522, 533, 527, 479, 17, 5, 0, 0, 0, 0, 0, 0, 507, 490, 512, 527, 499, 507, 525, 484, 10, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4200, 4145, 4096, 4047, 4071, 4079, 4062] avg=4096.0000 max=4200 max_rank=1 min=4047 min_rank=4 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16259 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[536, 528, 512, 485, 496, 515, 541, 530, 12, 0, 0, 0, 0, 0, 0, 0, 495, 529, 545, 460, 496, 491, 522, 539, 0, 0, 0, 0, 0, 0, 0, 0, 511, 503, 480, 536, 507, 520, 518, 478, 0, 0, 0, 0, 0, 0, 0, 0, 518, 535, 535, 495, 507, 511, 522, 535, 0, 0, 0, 0, 0, 0, 0, 0, 511, 519, 462, 565, 478, 540, 497, 515, 0, 0, 0, 0, 0, 0, 0, 0, 542, 481, 508, 469, 553, 527, 520, 514, 3, 0, 0, 0, 0, 0, 0, 0, 508, 504, 503, 522, 480, 535, 485, 514, 19, 5, 0, 0, 0, 0, 0, 0, 532, 484, 502, 502, 478, 514, 526, 497, 10, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4155, 4077, 4053, 4158, 4087, 4117, 4075, 4046] avg=4096.0000 max=4158 max_rank=3 min=4046 min_rank=7 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16325 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[467, 514, 489, 536, 574, 512, 504, 505, 11, 0, 0, 0, 0, 0, 0, 0, 507, 542, 500, 495, 494, 517, 495, 521, 0, 0, 0, 0, 0, 0, 0, 0, 461, 493, 507, 538, 538, 481, 500, 536, 0, 0, 0, 0, 0, 0, 0, 0, 495, 502, 505, 499, 538, 510, 500, 519, 0, 0, 0, 0, 0, 0, 0, 0, 538, 511, 491, 510, 526, 479, 497, 500, 0, 0, 0, 0, 0, 0, 0, 0, 516, 515, 531, 490, 509, 512, 489, 537, 2, 0, 0, 0, 0, 0, 0, 0, 487, 518, 516, 469, 518, 532, 502, 544, 17, 5, 0, 0, 0, 0, 0, 0, 528, 536, 475, 539, 492, 527, 577, 517, 10, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4071, 4054, 4068, 4052, 4101, 4108, 4202] avg=4096.0000 max=4202 max_rank=7 min=4052 min_rank=4 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16463 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[552, 504, 476, 570, 508, 472, 492, 491, 15, 0, 0, 0, 0, 0, 0, 0, 516, 491, 512, 501, 486, 517, 541, 506, 0, 0, 0, 0, 0, 0, 0, 0, 548, 518, 513, 505, 556, 513, 519, 518, 1, 0, 0, 0, 0, 0, 0, 0, 440, 501, 507, 506, 508, 535, 504, 519, 0, 0, 0, 0, 0, 0, 0, 0, 510, 544, 520, 493, 543, 555, 504, 527, 0, 0, 0, 0, 0, 0, 0, 0, 493, 488, 480, 524, 525, 520, 524, 488, 7, 0, 0, 0, 0, 0, 0, 0, 511, 510, 493, 488, 556, 509, 472, 472, 21, 10, 0, 0, 0, 0, 0, 0, 546, 506, 543, 489, 526, 504, 480, 509, 14, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4070, 4191, 4020, 4196, 4049, 4042, 4120] avg=4096.0000 max=4196 max_rank=4 min=4020 min_rank=3 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16407 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[505, 496, 514, 500, 531, 528, 523, 483, 11, 0, 0, 0, 0, 0, 0, 0, 482, 532, 496, 508, 483, 490, 541, 523, 0, 0, 0, 0, 0, 0, 0, 0, 535, 506, 543, 480, 513, 476, 508, 567, 0, 0, 0, 0, 0, 0, 0, 0, 497, 514, 515, 520, 483, 490, 511, 552, 0, 0, 0, 0, 0, 0, 0, 0, 526, 516, 493, 510, 535, 545, 490, 498, 0, 0, 0, 0, 0, 0, 0, 0, 499, 527, 518, 534, 516, 513, 507, 475, 2, 0, 0, 0, 0, 0, 0, 0, 506, 516, 484, 506, 506, 567, 517, 486, 17, 5, 0, 0, 0, 0, 0, 0, 500, 487, 535, 544, 506, 504, 500, 512, 9, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4055, 4128, 4082, 4113, 4091, 4110, 4098] avg=4096.0000 max=4128 max_rank=2 min=4055 min_rank=1 imbalance_ratio=1.007812 max_load_reduction=28640 max_load_reduction_pct=0.874023 moved_to_echo=16412 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[535, 504, 532, 515, 506, 498, 512, 489, 10, 0, 0, 0, 0, 0, 0, 0, 503, 490, 508, 484, 524, 518, 521, 485, 0, 0, 0, 0, 0, 0, 0, 0, 538, 550, 471, 502, 507, 519, 473, 496, 0, 0, 0, 0, 0, 0, 0, 0, 499, 532, 523, 540, 519, 494, 515, 558, 0, 0, 0, 0, 0, 0, 0, 0, 518, 524, 499, 502, 473, 498, 515, 521, 0, 0, 0, 0, 0, 0, 0, 0, 506, 522, 538, 514, 498, 486, 559, 529, 2, 0, 0, 0, 0, 0, 0, 0, 532, 523, 535, 507, 526, 517, 522, 450, 17, 5, 0, 0, 0, 0, 0, 0, 502, 502, 524, 489, 487, 523, 506, 516, 10, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4033, 4056, 4180, 4050, 4154, 4134, 4060] avg=4096.0000 max=4180 max_rank=3 min=4033 min_rank=1 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[488, 514, 476, 514, 532, 526, 510, 510, 10, 0, 0, 0, 0, 0, 0, 0, 538, 517, 485, 573, 456, 566, 520, 533, 0, 0, 0, 0, 0, 0, 0, 0, 498, 529, 511, 560, 530, 529, 506, 503, 0, 0, 0, 0, 0, 0, 0, 0, 510, 506, 518, 502, 523, 515, 540, 500, 0, 0, 0, 0, 0, 0, 0, 0, 517, 503, 530, 495, 473, 503, 498, 533, 0, 0, 0, 0, 0, 0, 0, 0, 467, 489, 483, 517, 522, 492, 514, 495, 2, 0, 0, 0, 0, 0, 0, 0, 521, 494, 504, 517, 496, 498, 542, 521, 17, 5, 0, 0, 0, 0, 0, 0, 512, 522, 517, 501, 508, 498, 485, 518, 10, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4188, 4166, 4114, 4052, 3981, 4115, 4072] avg=4096.0000 max=4188 max_rank=1 min=3981 min_rank=5 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16220 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[505, 537, 459, 516, 463, 502, 480, 517, 531, 489, 510, 532, 523, 535, 504, 554, 505, 531, 497, 490, 502, 486, 499, 496, 496, 491, 511, 515, 537, 522, 520, 511, 516, 513, 529, 538, 512, 507, 548, 517, 499, 507, 537, 534, 469, 496, 493, 505, 517, 530, 543, 525, 500, 517, 515, 531, 493, 495, 541, 516, 496, 516, 517, 530] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[518, 490, 504, 511, 505, 513, 515, 515, 492, 486, 559, 508, 516, 528, 529, 507, 500, 532, 511, 521, 524, 532, 515, 534, 500, 497, 490, 526, 522, 491, 518, 507, 523, 557, 538, 499, 514, 530, 501, 531, 498, 468, 511, 498, 514, 525, 513, 520, 513, 525, 504, 482, 523, 533, 516, 471, 489, 484, 532, 505, 490, 533, 517, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[524, 529, 525, 511, 557, 537, 566, 520, 542, 525, 513, 462, 530, 519, 547, 483, 506, 523, 508, 525, 477, 505, 530, 514, 521, 507, 518, 523, 473, 516, 540, 503, 468, 539, 501, 509, 520, 517, 509, 514, 499, 491, 517, 514, 518, 530, 475, 470, 534, 481, 569, 486, 487, 507, 508, 491, 481, 504, 483, 511, 512, 527, 513, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[501, 489, 550, 511, 537, 507, 495, 507, 492, 529, 543, 512, 523, 501, 515, 525, 504, 502, 539, 512, 522, 530, 517, 515, 525, 489, 493, 487, 518, 473, 533, 540, 515, 544, 488, 494, 521, 475, 511, 514, 521, 535, 497, 503, 503, 539, 500, 563, 525, 530, 481, 514, 505, 502, 515, 505, 520, 493, 494, 504, 500, 508, 488, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[496, 509, 496, 501, 517, 496, 498, 503, 489, 506, 529, 525, 528, 528, 533, 503, 545, 564, 510, 504, 474, 516, 477, 487, 488, 501, 568, 490, 510, 504, 489, 500, 520, 542, 466, 494, 525, 531, 534, 523, 519, 528, 489, 518, 516, 495, 498, 529, 515, 544, 502, 501, 532, 516, 509, 521, 538, 527, 484, 531, 527, 494, 499, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[561, 515, 501, 504, 490, 487, 513, 492, 531, 527, 525, 505, 492, 529, 507, 559, 515, 460, 528, 516, 515, 490, 513, 508, 532, 507, 521, 486, 522, 517, 532, 537, 452, 496, 525, 511, 517, 495, 497, 516, 482, 502, 476, 543, 523, 503, 526, 534, 538, 496, 493, 529, 505, 490, 511, 516, 530, 519, 533, 520, 519, 515, 473, 546] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[514, 540, 545, 506, 553, 498, 466, 519, 510, 523, 495, 492, 530, 512, 547, 472, 543, 496, 504, 522, 501, 513, 518, 576, 513, 511, 519, 491, 513, 498, 478, 515, 495, 539, 537, 516, 489, 488, 467, 496, 497, 517, 533, 506, 505, 517, 529, 487, 505, 540, 509, 498, 498, 523, 537, 527, 443, 505, 519, 508, 513, 526, 535, 531] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[524, 483, 509, 498, 514, 522, 497, 518, 520, 507, 532, 473, 538, 527, 531, 528, 494, 521, 492, 513, 500, 502, 493, 518, 531, 519, 537, 530, 496, 481, 537, 544, 519, 508, 519, 525, 466, 490, 542, 512, 476, 519, 502, 529, 483, 510, 548, 500, 507, 537, 492, 501, 524, 491, 517, 511, 521, 523, 479, 506, 518, 495, 547, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[505, 537, 459, 516, 463, 502, 480, 517, 8, 0, 0, 0, 0, 0, 0, 0, 531, 489, 510, 532, 523, 535, 464, 554, 0, 0, 0, 0, 0, 0, 0, 0, 505, 531, 497, 490, 502, 486, 499, 496, 4, 0, 0, 0, 0, 0, 0, 0, 496, 491, 511, 515, 537, 522, 520, 511, 3, 1, 0, 0, 0, 0, 0, 0, 516, 513, 529, 538, 512, 507, 548, 517, 0, 0, 0, 0, 0, 0, 0, 0, 499, 507, 537, 534, 469, 496, 493, 505, 19, 0, 0, 0, 0, 0, 0, 0, 517, 525, 543, 525, 500, 517, 515, 531, 0, 0, 0, 0, 0, 0, 0, 0, 493, 495, 541, 516, 496, 516, 517, 530, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3987, 4138, 4010, 4107, 4180, 4059, 4173, 4114] avg=4096.0000 max=4180 max_rank=4 min=3987 min_rank=0 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16526 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[524, 529, 525, 511, 557, 537, 566, 520, 8, 0, 0, 0, 0, 0, 0, 0, 542, 525, 513, 462, 530, 519, 504, 483, 0, 0, 0, 0, 0, 0, 0, 0, 506, 523, 508, 525, 477, 505, 530, 514, 4, 0, 0, 0, 0, 0, 0, 0, 521, 507, 518, 523, 473, 516, 540, 503, 3, 1, 0, 0, 0, 0, 0, 0, 468, 539, 501, 509, 520, 517, 509, 514, 0, 0, 0, 0, 0, 0, 0, 0, 499, 491, 517, 514, 518, 530, 475, 470, 21, 0, 0, 0, 0, 0, 0, 0, 534, 476, 569, 486, 487, 507, 508, 491, 0, 0, 0, 0, 0, 0, 0, 0, 481, 504, 483, 511, 512, 527, 513, 504, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4277, 4078, 4092, 4105, 4077, 4035, 4058, 4046] avg=4096.0000 max=4277 max_rank=0 min=4035 min_rank=5 imbalance_ratio=1.044189 max_load_reduction=28491 max_load_reduction_pct=0.869476 moved_to_echo=16216 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[518, 490, 504, 511, 505, 513, 515, 515, 8, 0, 0, 0, 0, 0, 0, 0, 492, 486, 559, 508, 516, 528, 487, 507, 0, 0, 0, 0, 0, 0, 0, 0, 500, 532, 511, 521, 524, 532, 515, 534, 4, 0, 0, 0, 0, 0, 0, 0, 500, 497, 490, 526, 522, 491, 518, 507, 3, 1, 0, 0, 0, 0, 0, 0, 523, 557, 538, 499, 514, 530, 501, 531, 0, 0, 0, 0, 0, 0, 0, 0, 498, 468, 511, 498, 514, 525, 513, 520, 20, 0, 0, 0, 0, 0, 0, 0, 513, 520, 504, 482, 523, 533, 516, 471, 0, 0, 0, 0, 0, 0, 0, 0, 489, 484, 532, 505, 490, 533, 517, 495, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4083, 4173, 4055, 4193, 4067, 4062, 4056] avg=4096.0000 max=4193 max_rank=4 min=4055 min_rank=3 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[496, 509, 496, 501, 517, 496, 498, 503, 8, 0, 0, 0, 0, 0, 0, 0, 489, 506, 529, 525, 528, 528, 490, 503, 0, 0, 0, 0, 0, 0, 0, 0, 545, 564, 510, 504, 474, 516, 477, 487, 4, 0, 0, 0, 0, 0, 0, 0, 488, 501, 568, 490, 510, 504, 489, 500, 3, 1, 0, 0, 0, 0, 0, 0, 520, 542, 466, 494, 525, 531, 534, 523, 0, 0, 0, 0, 0, 0, 0, 0, 519, 528, 489, 518, 516, 495, 498, 529, 21, 0, 0, 0, 0, 0, 0, 0, 515, 539, 502, 501, 532, 516, 509, 521, 0, 0, 0, 0, 0, 0, 0, 0, 538, 527, 484, 531, 527, 494, 499, 517, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4024, 4098, 4081, 4054, 4135, 4113, 4135, 4128] avg=4096.0000 max=4135 max_rank=4 min=4024 min_rank=0 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16511 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[561, 515, 501, 504, 490, 487, 513, 492, 11, 0, 0, 0, 0, 0, 0, 0, 531, 527, 525, 505, 492, 529, 453, 559, 0, 0, 0, 0, 0, 0, 0, 0, 515, 460, 528, 516, 515, 490, 513, 508, 8, 0, 0, 0, 0, 0, 0, 0, 532, 507, 521, 486, 522, 517, 532, 537, 5, 5, 0, 0, 0, 0, 0, 0, 452, 496, 525, 511, 517, 495, 497, 516, 4, 0, 0, 0, 0, 0, 0, 0, 482, 502, 476, 543, 523, 503, 526, 534, 24, 0, 0, 0, 0, 0, 0, 0, 538, 479, 493, 529, 505, 490, 511, 516, 0, 0, 0, 0, 0, 0, 0, 0, 530, 519, 533, 520, 519, 515, 473, 546, 14, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4121, 4053, 4164, 4013, 4113, 4061, 4169] avg=4096.0000 max=4169 max_rank=7 min=4013 min_rank=4 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16356 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[501, 489, 550, 511, 537, 507, 495, 507, 8, 0, 0, 0, 0, 0, 0, 0, 492, 529, 543, 512, 523, 501, 474, 525, 0, 0, 0, 0, 0, 0, 0, 0, 504, 502, 539, 512, 522, 530, 517, 515, 4, 0, 0, 0, 0, 0, 0, 0, 525, 489, 493, 487, 518, 473, 533, 540, 3, 1, 0, 0, 0, 0, 0, 0, 515, 544, 488, 494, 521, 475, 511, 514, 0, 0, 0, 0, 0, 0, 0, 0, 521, 535, 497, 503, 503, 539, 500, 563, 20, 0, 0, 0, 0, 0, 0, 0, 525, 525, 481, 514, 505, 502, 515, 505, 0, 0, 0, 0, 0, 0, 0, 0, 520, 493, 494, 504, 500, 508, 488, 525, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4099, 4145, 4062, 4062, 4181, 4072, 4042] avg=4096.0000 max=4181 max_rank=5 min=4042 min_rank=7 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16357 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[514, 540, 545, 506, 553, 498, 466, 519, 8, 0, 0, 0, 0, 0, 0, 0, 510, 523, 495, 492, 530, 512, 504, 472, 0, 0, 0, 0, 0, 0, 0, 0, 543, 496, 504, 522, 501, 513, 518, 576, 4, 0, 0, 0, 0, 0, 0, 0, 513, 511, 519, 491, 513, 498, 478, 515, 3, 1, 0, 0, 0, 0, 0, 0, 495, 539, 537, 516, 489, 488, 467, 496, 0, 0, 0, 0, 0, 0, 0, 0, 497, 517, 533, 506, 505, 517, 529, 487, 21, 0, 0, 0, 0, 0, 0, 0, 505, 535, 509, 498, 498, 523, 537, 527, 0, 0, 0, 0, 0, 0, 0, 0, 443, 505, 519, 508, 513, 526, 535, 531, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4149, 4038, 4177, 4042, 4027, 4112, 4132, 4091] avg=4096.0000 max=4177 max_rank=2 min=4027 min_rank=4 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16362 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[524, 483, 509, 498, 514, 522, 497, 518, 8, 0, 0, 0, 0, 0, 0, 0, 520, 507, 532, 473, 538, 527, 488, 528, 0, 0, 0, 0, 0, 0, 0, 0, 494, 521, 492, 513, 500, 502, 493, 518, 4, 0, 0, 0, 0, 0, 0, 0, 531, 519, 537, 530, 496, 481, 537, 544, 3, 1, 0, 0, 0, 0, 0, 0, 519, 508, 519, 525, 466, 490, 542, 512, 0, 0, 0, 0, 0, 0, 0, 0, 476, 519, 502, 529, 483, 510, 548, 500, 21, 0, 0, 0, 0, 0, 0, 0, 507, 532, 492, 501, 524, 491, 517, 511, 0, 0, 0, 0, 0, 0, 0, 0, 521, 523, 479, 506, 518, 495, 547, 522, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4113, 4037, 4179, 4081, 4088, 4075, 4122] avg=4096.0000 max=4179 max_rank=3 min=4037 min_rank=2 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4352, 4224, 4096, 4096, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[531, 533, 561, 506, 520, 520, 519, 489, 528, 523, 519, 477, 502, 517, 501, 500, 508, 500, 525, 514, 502, 530, 497, 536, 518, 559, 511, 466, 510, 533, 508, 479, 518, 494, 469, 518, 536, 503, 510, 506, 516, 505, 520, 516, 551, 502, 537, 503, 515, 543, 508, 525, 473, 504, 480, 533, 519, 474, 517, 497, 485, 522, 498, 529] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[493, 531, 509, 527, 525, 519, 516, 524, 479, 526, 512, 515, 519, 535, 513, 504, 527, 493, 485, 521, 502, 530, 523, 518, 528, 492, 484, 507, 499, 511, 531, 481, 512, 518, 494, 503, 525, 525, 505, 546, 522, 511, 505, 526, 535, 502, 520, 495, 512, 519, 515, 483, 490, 535, 511, 508, 502, 521, 543, 493, 513, 508, 476, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[508, 517, 495, 504, 479, 524, 506, 531, 504, 509, 507, 509, 539, 512, 494, 537, 525, 495, 511, 465, 451, 514, 540, 555, 543, 538, 517, 510, 511, 484, 499, 527, 515, 524, 500, 506, 490, 527, 502, 506, 492, 506, 493, 540, 520, 513, 506, 545, 494, 508, 520, 504, 532, 510, 475, 505, 508, 506, 528, 505, 505, 536, 531, 556] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[521, 513, 478, 564, 511, 500, 509, 560, 512, 513, 538, 498, 488, 547, 530, 524, 554, 525, 525, 514, 506, 515, 515, 528, 512, 479, 531, 527, 492, 540, 521, 496, 498, 493, 485, 522, 508, 482, 483, 502, 532, 507, 543, 477, 519, 496, 473, 498, 508, 488, 498, 543, 513, 517, 536, 511, 545, 520, 486, 487, 501, 491, 508, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[466, 517, 508, 479, 501, 497, 520, 510, 545, 509, 499, 503, 501, 517, 501, 515, 526, 502, 483, 486, 516, 497, 513, 515, 513, 528, 541, 512, 520, 512, 550, 501, 450, 535, 555, 503, 538, 514, 516, 528, 510, 509, 460, 561, 487, 465, 510, 518, 494, 536, 533, 496, 538, 512, 552, 496, 506, 528, 545, 501, 538, 516, 495, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[519, 517, 541, 501, 537, 470, 497, 524, 475, 523, 483, 545, 490, 548, 504, 521, 495, 487, 509, 535, 545, 494, 520, 528, 528, 508, 559, 490, 496, 557, 543, 518, 517, 492, 467, 500, 513, 524, 531, 512, 510, 539, 478, 488, 507, 486, 523, 503, 485, 514, 504, 507, 515, 540, 496, 504, 514, 531, 530, 497, 521, 504, 499, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[510, 450, 510, 485, 510, 505, 505, 523, 538, 544, 543, 510, 523, 508, 490, 519, 490, 512, 517, 517, 491, 514, 502, 546, 499, 518, 505, 541, 522, 498, 522, 522, 484, 516, 551, 496, 491, 491, 533, 490, 501, 507, 524, 520, 505, 488, 516, 500, 494, 535, 512, 494, 564, 484, 541, 501, 507, 538, 524, 530, 537, 524, 506, 475] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[476, 527, 523, 539, 520, 532, 510, 535, 493, 502, 508, 513, 505, 563, 498, 523, 507, 500, 501, 538, 507, 534, 520, 528, 507, 515, 535, 538, 509, 485, 542, 555, 502, 517, 517, 515, 495, 494, 519, 506, 520, 504, 497, 514, 514, 505, 492, 519, 503, 506, 529, 499, 498, 480, 471, 498, 524, 489, 504, 549, 479, 492, 489, 540] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[531, 533, 561, 506, 520, 520, 519, 485, 0, 0, 0, 0, 0, 0, 0, 0, 528, 523, 519, 477, 502, 503, 501, 500, 0, 0, 0, 0, 0, 0, 0, 0, 508, 500, 525, 514, 502, 530, 497, 530, 0, 0, 0, 0, 0, 0, 0, 0, 518, 559, 511, 466, 510, 533, 477, 479, 0, 0, 0, 0, 0, 0, 0, 0, 518, 494, 469, 518, 536, 503, 510, 506, 27, 0, 0, 0, 0, 0, 0, 0, 516, 505, 520, 516, 551, 502, 537, 503, 14, 4, 0, 0, 0, 0, 0, 0, 515, 543, 508, 525, 473, 504, 480, 533, 6, 4, 3, 0, 0, 0, 0, 0, 519, 474, 514, 497, 485, 522, 498, 529, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4175, 4053, 4106, 4053, 4081, 4168, 4094, 4038] avg=4096.0000 max=4175 max_rank=0 min=4038 min_rank=7 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[508, 517, 495, 504, 479, 524, 506, 527, 0, 0, 0, 0, 0, 0, 0, 0, 504, 509, 507, 509, 539, 498, 494, 537, 0, 0, 0, 0, 0, 0, 0, 0, 525, 495, 511, 465, 451, 514, 540, 549, 0, 0, 0, 0, 0, 0, 0, 0, 543, 538, 517, 510, 511, 484, 468, 527, 0, 0, 0, 0, 0, 0, 0, 0, 515, 524, 500, 506, 490, 527, 502, 506, 27, 0, 0, 0, 0, 0, 0, 0, 492, 506, 493, 540, 520, 513, 506, 545, 14, 4, 0, 0, 0, 0, 0, 0, 494, 508, 520, 504, 532, 510, 475, 505, 6, 4, 4, 0, 0, 0, 0, 0, 508, 506, 524, 505, 505, 536, 531, 556, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4097, 4050, 4098, 4097, 4133, 4062, 4171] avg=4096.0000 max=4171 max_rank=7 min=4050 min_rank=2 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16463 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[493, 531, 509, 527, 525, 519, 516, 520, 0, 0, 0, 0, 0, 0, 0, 0, 479, 526, 512, 515, 519, 520, 513, 504, 0, 0, 0, 0, 0, 0, 0, 0, 527, 493, 485, 521, 502, 530, 523, 512, 0, 0, 0, 0, 0, 0, 0, 0, 528, 492, 484, 507, 499, 511, 498, 481, 0, 0, 0, 0, 0, 0, 0, 0, 512, 518, 494, 503, 525, 525, 505, 546, 29, 0, 0, 0, 0, 0, 0, 0, 522, 511, 505, 526, 535, 502, 520, 495, 15, 4, 0, 0, 0, 0, 0, 0, 512, 519, 515, 483, 490, 535, 511, 508, 6, 4, 4, 0, 0, 0, 0, 0, 502, 521, 539, 493, 513, 508, 476, 516, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4140, 4088, 4093, 4000, 4157, 4135, 4087, 4068] avg=4096.0000 max=4157 max_rank=4 min=4000 min_rank=3 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16447 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[466, 517, 508, 479, 501, 497, 520, 500, 0, 0, 0, 0, 0, 0, 0, 0, 545, 509, 499, 503, 501, 498, 501, 515, 0, 0, 0, 0, 0, 0, 0, 0, 526, 502, 483, 486, 516, 497, 513, 501, 0, 0, 0, 0, 0, 0, 0, 0, 513, 528, 541, 512, 520, 512, 510, 501, 0, 0, 0, 0, 0, 0, 0, 0, 450, 535, 555, 503, 538, 514, 516, 528, 34, 0, 0, 0, 0, 0, 0, 0, 510, 509, 460, 561, 487, 465, 510, 518, 19, 6, 6, 0, 0, 0, 0, 0, 494, 536, 533, 496, 538, 512, 552, 496, 8, 10, 7, 0, 0, 0, 0, 0, 506, 528, 538, 501, 538, 516, 495, 520, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3988, 4071, 4024, 4137, 4173, 4051, 4182, 4142] avg=4096.0000 max=4182 max_rank=6 min=3988 min_rank=0 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16548 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[521, 513, 478, 564, 511, 500, 509, 555, 0, 0, 0, 0, 0, 0, 0, 0, 512, 513, 538, 498, 488, 532, 530, 524, 0, 0, 0, 0, 0, 0, 0, 0, 554, 525, 525, 514, 506, 515, 515, 522, 0, 0, 0, 0, 0, 0, 0, 0, 512, 479, 531, 527, 492, 540, 489, 496, 0, 0, 0, 0, 0, 0, 0, 0, 498, 493, 485, 522, 508, 482, 483, 502, 28, 0, 0, 0, 0, 0, 0, 0, 532, 507, 543, 477, 519, 496, 473, 498, 15, 4, 0, 0, 0, 0, 0, 0, 508, 488, 498, 543, 513, 517, 536, 511, 6, 5, 3, 0, 0, 0, 0, 0, 545, 520, 483, 487, 501, 491, 508, 512, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4151, 4135, 4176, 4066, 4001, 4064, 4128, 4047] avg=4096.0000 max=4176 max_rank=2 min=4001 min_rank=4 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16240 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[519, 517, 541, 501, 537, 470, 497, 520, 0, 0, 0, 0, 0, 0, 0, 0, 475, 523, 483, 545, 490, 533, 504, 521, 0, 0, 0, 0, 0, 0, 0, 0, 495, 487, 509, 535, 545, 494, 520, 522, 0, 0, 0, 0, 0, 0, 0, 0, 528, 508, 559, 490, 496, 557, 510, 518, 0, 0, 0, 0, 0, 0, 0, 0, 517, 492, 467, 500, 513, 524, 531, 512, 29, 0, 0, 0, 0, 0, 0, 0, 510, 539, 478, 488, 507, 486, 523, 503, 15, 4, 0, 0, 0, 0, 0, 0, 485, 514, 504, 507, 515, 540, 496, 504, 6, 4, 4, 0, 0, 0, 0, 0, 514, 531, 526, 497, 521, 504, 499, 510, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4074, 4107, 4166, 4085, 4053, 4079, 4102] avg=4096.0000 max=4166 max_rank=3 min=4053 min_rank=5 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16319 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[510, 450, 510, 485, 510, 505, 505, 519, 0, 0, 0, 0, 0, 0, 0, 0, 538, 544, 543, 510, 523, 494, 490, 519, 0, 0, 0, 0, 0, 0, 0, 0, 490, 512, 517, 517, 491, 514, 502, 540, 0, 0, 0, 0, 0, 0, 0, 0, 499, 518, 505, 541, 522, 498, 490, 522, 0, 0, 0, 0, 0, 0, 0, 0, 484, 516, 551, 496, 491, 491, 533, 490, 28, 0, 0, 0, 0, 0, 0, 0, 501, 507, 524, 520, 505, 488, 516, 500, 14, 4, 0, 0, 0, 0, 0, 0, 494, 535, 512, 494, 564, 484, 541, 501, 6, 4, 4, 0, 0, 0, 0, 0, 507, 538, 520, 530, 537, 524, 506, 475, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3994, 4161, 4083, 4095, 4080, 4079, 4139, 4137] avg=4096.0000 max=4161 max_rank=1 min=3994 min_rank=0 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16435 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[476, 527, 523, 539, 520, 532, 510, 531, 0, 0, 0, 0, 0, 0, 0, 0, 493, 502, 508, 513, 505, 547, 498, 523, 0, 0, 0, 0, 0, 0, 0, 0, 507, 500, 501, 538, 507, 534, 520, 522, 0, 0, 0, 0, 0, 0, 0, 0, 507, 515, 535, 538, 509, 485, 509, 555, 0, 0, 0, 0, 0, 0, 0, 0, 502, 517, 517, 515, 495, 494, 519, 506, 29, 0, 0, 0, 0, 0, 0, 0, 520, 504, 497, 514, 514, 505, 492, 519, 16, 4, 0, 0, 0, 0, 0, 0, 503, 506, 529, 499, 498, 480, 471, 498, 6, 4, 3, 0, 0, 0, 0, 0, 524, 489, 501, 549, 479, 492, 489, 540, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4158, 4089, 4129, 4153, 4094, 4085, 3997, 4063] avg=4096.0000 max=4158 max_rank=0 min=3997 min_rank=6 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16239 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 3968, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 3968, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[521, 543, 489, 544, 479, 493, 514, 501, 509, 525, 523, 521, 507, 529, 547, 553, 487, 504, 521, 497, 525, 526, 472, 499, 497, 517, 546, 502, 480, 481, 529, 514, 516, 505, 526, 531, 487, 554, 538, 467, 491, 505, 509, 470, 486, 521, 549, 531, 510, 503, 534, 518, 504, 511, 497, 465, 557, 483, 506, 487, 553, 508, 549, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[537, 518, 539, 526, 499, 522, 499, 502, 492, 512, 493, 491, 524, 479, 529, 501, 521, 497, 525, 528, 531, 501, 494, 450, 498, 479, 483, 509, 485, 519, 505, 484, 529, 533, 556, 500, 533, 494, 585, 537, 499, 520, 515, 519, 504, 514, 524, 507, 511, 508, 531, 490, 521, 532, 528, 513, 520, 491, 524, 519, 518, 501, 528, 492] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[497, 515, 548, 566, 486, 491, 502, 521, 478, 515, 491, 519, 522, 498, 524, 552, 514, 536, 506, 534, 525, 466, 528, 536, 537, 529, 504, 505, 522, 539, 503, 485, 506, 501, 511, 536, 492, 459, 526, 491, 534, 511, 485, 500, 532, 539, 514, 510, 464, 505, 525, 497, 511, 491, 532, 536, 516, 512, 500, 512, 505, 485, 515, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[494, 507, 514, 539, 499, 523, 538, 536, 502, 537, 498, 472, 507, 518, 525, 504, 510, 487, 507, 523, 524, 514, 525, 494, 507, 533, 499, 510, 520, 485, 495, 504, 512, 506, 520, 565, 484, 511, 520, 500, 499, 480, 522, 525, 525, 508, 533, 529, 512, 534, 551, 502, 518, 485, 551, 468, 521, 462, 522, 513, 531, 492, 515, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[485, 523, 456, 565, 531, 522, 532, 511, 488, 536, 516, 504, 505, 498, 489, 499, 517, 529, 510, 543, 543, 517, 520, 494, 523, 539, 511, 527, 507, 520, 534, 507, 499, 504, 511, 517, 503, 517, 487, 536, 501, 488, 510, 503, 522, 498, 504, 522, 522, 474, 495, 508, 498, 536, 504, 498, 504, 493, 519, 518, 488, 517, 549, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[527, 500, 507, 512, 507, 512, 503, 472, 507, 489, 494, 540, 518, 518, 572, 525, 514, 533, 500, 514, 471, 531, 507, 523, 521, 535, 510, 510, 526, 512, 541, 511, 505, 485, 491, 530, 505, 461, 488, 548, 499, 505, 518, 526, 495, 513, 500, 514, 517, 551, 533, 525, 510, 493, 520, 488, 510, 515, 546, 495, 492, 507, 518, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[520, 462, 482, 502, 546, 505, 537, 531, 533, 489, 495, 490, 498, 515, 541, 534, 517, 511, 506, 510, 535, 522, 533, 523, 496, 550, 500, 481, 497, 499, 496, 462, 512, 506, 531, 546, 522, 520, 542, 503, 514, 487, 520, 471, 512, 507, 532, 479, 526, 517, 515, 466, 506, 539, 524, 532, 533, 483, 479, 538, 518, 518, 530, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[486, 468, 533, 514, 522, 523, 490, 504, 495, 506, 544, 505, 512, 490, 489, 492, 549, 533, 476, 494, 522, 514, 511, 489, 538, 538, 529, 499, 485, 472, 520, 521, 494, 530, 537, 488, 500, 507, 494, 525, 512, 578, 507, 473, 531, 530, 496, 507, 512, 533, 533, 539, 487, 498, 493, 524, 481, 530, 529, 525, 533, 519, 541, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[537, 518, 539, 524, 499, 522, 499, 502, 0, 0, 0, 0, 0, 0, 0, 0, 492, 512, 493, 491, 524, 479, 529, 501, 2, 1, 0, 0, 0, 0, 0, 0, 521, 497, 525, 528, 526, 501, 494, 450, 0, 0, 0, 0, 0, 0, 0, 0, 498, 479, 483, 509, 485, 519, 505, 484, 12, 4, 0, 0, 0, 0, 0, 0, 529, 533, 556, 488, 533, 494, 585, 537, 0, 0, 0, 0, 0, 0, 0, 0, 499, 520, 515, 519, 504, 514, 524, 507, 6, 4, 0, 0, 0, 0, 0, 0, 511, 508, 531, 490, 521, 532, 528, 513, 0, 0, 0, 0, 0, 0, 0, 0, 520, 491, 524, 519, 518, 501, 518, 492, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4140, 4024, 4042, 3978, 4255, 4112, 4134, 4083] avg=4096.0000 max=4255 max_rank=4 min=3978 min_rank=3 imbalance_ratio=1.038818 max_load_reduction=28513 max_load_reduction_pct=0.870148 moved_to_echo=16584 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[494, 507, 514, 536, 499, 523, 538, 536, 0, 0, 0, 0, 0, 0, 0, 0, 502, 537, 498, 472, 507, 518, 525, 504, 3, 1, 0, 0, 0, 0, 0, 0, 510, 487, 507, 523, 519, 514, 525, 494, 0, 0, 0, 0, 0, 0, 0, 0, 507, 533, 499, 510, 520, 485, 495, 504, 14, 4, 0, 0, 0, 0, 0, 0, 512, 506, 520, 551, 484, 511, 520, 500, 0, 0, 0, 0, 0, 0, 0, 0, 499, 480, 522, 525, 525, 508, 533, 529, 5, 4, 0, 0, 0, 0, 0, 0, 512, 534, 551, 502, 518, 485, 551, 468, 0, 0, 0, 0, 0, 0, 0, 0, 521, 462, 522, 513, 531, 492, 506, 502, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4147, 4067, 4079, 4071, 4104, 4130, 4121, 4049] avg=4096.0000 max=4147 max_rank=0 min=4049 min_rank=7 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16404 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[497, 515, 548, 563, 486, 491, 502, 521, 0, 0, 0, 0, 0, 0, 0, 0, 478, 515, 491, 519, 522, 498, 524, 552, 3, 1, 0, 0, 0, 0, 0, 0, 514, 536, 506, 534, 520, 466, 528, 536, 0, 0, 0, 0, 0, 0, 0, 0, 537, 529, 504, 505, 522, 539, 503, 485, 13, 4, 0, 0, 0, 0, 0, 0, 506, 501, 511, 523, 492, 459, 526, 491, 0, 0, 0, 0, 0, 0, 0, 0, 534, 511, 485, 500, 532, 539, 514, 510, 5, 4, 0, 0, 0, 0, 0, 0, 464, 505, 525, 497, 511, 491, 532, 536, 0, 0, 0, 0, 0, 0, 0, 0, 516, 512, 500, 512, 505, 485, 506, 521, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4123, 4103, 4140, 4141, 4009, 4134, 4061, 4057] avg=4096.0000 max=4141 max_rank=3 min=4009 min_rank=4 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16261 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[521, 543, 489, 541, 479, 493, 514, 501, 0, 0, 0, 0, 0, 0, 0, 0, 509, 525, 523, 521, 507, 529, 547, 553, 3, 1, 0, 0, 0, 0, 0, 0, 487, 504, 521, 497, 520, 526, 472, 499, 0, 0, 0, 0, 0, 0, 0, 0, 497, 517, 546, 502, 480, 481, 529, 514, 13, 5, 0, 0, 0, 0, 0, 0, 516, 505, 526, 518, 487, 554, 538, 467, 0, 0, 0, 0, 0, 0, 0, 0, 491, 505, 509, 470, 486, 521, 549, 531, 6, 4, 0, 0, 0, 0, 0, 0, 510, 503, 534, 518, 504, 511, 497, 465, 0, 0, 0, 0, 0, 0, 0, 0, 557, 483, 506, 487, 553, 508, 538, 502, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4218, 4026, 4084, 4111, 4072, 4042, 4134] avg=4096.0000 max=4218 max_rank=1 min=4026 min_rank=2 imbalance_ratio=1.029785 max_load_reduction=28550 max_load_reduction_pct=0.871277 moved_to_echo=16359 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[485, 523, 456, 562, 531, 522, 532, 511, 0, 0, 0, 0, 0, 0, 0, 0, 488, 536, 516, 504, 505, 498, 489, 499, 3, 1, 0, 0, 0, 0, 0, 0, 517, 529, 510, 543, 538, 517, 520, 494, 0, 0, 0, 0, 0, 0, 0, 0, 523, 539, 511, 527, 507, 520, 534, 507, 13, 5, 0, 0, 0, 0, 0, 0, 499, 504, 511, 504, 503, 517, 487, 536, 0, 0, 0, 0, 0, 0, 0, 0, 501, 488, 510, 503, 522, 498, 504, 522, 6, 4, 0, 0, 0, 0, 0, 0, 522, 474, 495, 508, 498, 536, 504, 498, 0, 0, 0, 0, 0, 0, 0, 0, 504, 493, 519, 518, 488, 517, 538, 522, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4122, 4039, 4168, 4186, 4061, 4058, 4035, 4099] avg=4096.0000 max=4186 max_rank=3 min=4035 min_rank=6 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16253 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[527, 500, 507, 506, 507, 512, 503, 472, 0, 0, 0, 0, 0, 0, 0, 0, 507, 489, 494, 540, 518, 518, 572, 525, 6, 8, 6, 0, 0, 0, 0, 0, 514, 533, 500, 514, 456, 531, 507, 523, 0, 0, 0, 0, 0, 0, 0, 0, 521, 535, 510, 510, 526, 512, 541, 511, 17, 9, 0, 0, 0, 0, 0, 0, 505, 485, 491, 513, 505, 461, 488, 548, 0, 0, 0, 0, 0, 0, 0, 0, 499, 505, 518, 526, 495, 513, 500, 514, 9, 7, 0, 0, 0, 0, 0, 0, 517, 551, 527, 525, 510, 493, 520, 488, 0, 0, 0, 0, 0, 0, 0, 0, 510, 515, 546, 495, 492, 507, 500, 503, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4034, 4183, 4078, 4192, 3996, 4086, 4131, 4068] avg=4096.0000 max=4192 max_rank=3 min=3996 min_rank=4 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16281 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[520, 462, 482, 500, 546, 505, 537, 531, 0, 0, 0, 0, 0, 0, 0, 0, 533, 489, 495, 490, 498, 515, 541, 534, 2, 1, 0, 0, 0, 0, 0, 0, 517, 511, 506, 510, 530, 522, 533, 523, 0, 0, 0, 0, 0, 0, 0, 0, 496, 550, 500, 481, 497, 499, 496, 462, 13, 4, 0, 0, 0, 0, 0, 0, 512, 506, 531, 533, 522, 520, 542, 503, 0, 0, 0, 0, 0, 0, 0, 0, 514, 487, 520, 471, 512, 507, 532, 479, 6, 4, 0, 0, 0, 0, 0, 0, 526, 517, 515, 466, 506, 539, 524, 532, 0, 0, 0, 0, 0, 0, 0, 0, 533, 483, 479, 538, 518, 518, 520, 522, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4098, 4152, 3998, 4169, 4032, 4125, 4111] avg=4096.0000 max=4169 max_rank=4 min=3998 min_rank=3 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16437 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[486, 468, 533, 512, 522, 523, 490, 504, 0, 0, 0, 0, 0, 0, 0, 0, 495, 506, 544, 505, 512, 490, 489, 492, 2, 1, 0, 0, 0, 0, 0, 0, 549, 533, 476, 494, 517, 514, 511, 489, 0, 0, 0, 0, 0, 0, 0, 0, 538, 538, 529, 499, 485, 472, 520, 521, 12, 4, 0, 0, 0, 0, 0, 0, 494, 530, 537, 476, 500, 507, 494, 525, 0, 0, 0, 0, 0, 0, 0, 0, 512, 578, 507, 473, 531, 530, 496, 507, 6, 4, 0, 0, 0, 0, 0, 0, 512, 533, 533, 539, 487, 498, 493, 524, 0, 0, 0, 0, 0, 0, 0, 0, 481, 530, 529, 525, 533, 519, 531, 519, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4038, 4036, 4083, 4118, 4063, 4144, 4119, 4167] avg=4096.0000 max=4167 max_rank=7 min=4036 min_rank=1 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16493 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4352, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[505, 491, 548, 510, 489, 518, 519, 505, 515, 504, 499, 528, 505, 506, 506, 496, 492, 536, 506, 541, 493, 502, 499, 513, 544, 480, 506, 527, 483, 530, 483, 512, 500, 528, 527, 512, 524, 497, 510, 516, 510, 481, 547, 497, 514, 517, 504, 523, 529, 517, 497, 486, 576, 508, 526, 513, 480, 531, 563, 480, 538, 484, 531, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[519, 527, 488, 532, 500, 490, 512, 507, 487, 511, 526, 491, 533, 522, 530, 504, 556, 545, 501, 547, 549, 497, 510, 524, 505, 487, 491, 480, 511, 522, 510, 512, 526, 541, 526, 491, 520, 528, 541, 538, 509, 493, 509, 496, 528, 530, 512, 520, 511, 484, 490, 497, 522, 539, 489, 500, 489, 486, 520, 482, 507, 498, 509, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[511, 524, 468, 512, 488, 495, 514, 447, 522, 504, 508, 507, 542, 530, 549, 531, 531, 519, 524, 540, 520, 535, 483, 486, 506, 532, 514, 510, 495, 540, 522, 537, 527, 487, 526, 496, 515, 483, 487, 521, 475, 508, 488, 491, 520, 521, 536, 507, 527, 509, 531, 506, 495, 496, 517, 486, 535, 511, 507, 565, 508, 498, 509, 534] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[570, 542, 497, 498, 547, 493, 532, 530, 552, 488, 507, 511, 508, 486, 506, 532, 541, 477, 514, 550, 520, 572, 500, 499, 543, 492, 496, 477, 509, 503, 537, 549, 480, 532, 505, 498, 499, 487, 481, 518, 506, 513, 505, 498, 477, 540, 491, 493, 543, 521, 482, 528, 492, 520, 480, 494, 530, 543, 523, 561, 508, 457, 499, 486] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[514, 518, 508, 538, 519, 505, 505, 492, 518, 538, 515, 505, 506, 484, 518, 526, 515, 540, 489, 504, 497, 549, 506, 538, 490, 513, 507, 498, 488, 489, 542, 480, 543, 561, 528, 507, 511, 510, 515, 514, 503, 513, 518, 502, 512, 503, 488, 520, 496, 521, 502, 549, 497, 532, 512, 514, 532, 470, 509, 492, 501, 531, 487, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[539, 539, 469, 500, 506, 499, 527, 497, 477, 509, 491, 492, 525, 559, 521, 503, 507, 486, 546, 516, 555, 476, 462, 509, 525, 521, 521, 516, 509, 474, 491, 484, 466, 513, 511, 486, 517, 510, 507, 543, 497, 525, 479, 533, 533, 547, 524, 526, 493, 483, 531, 534, 541, 489, 559, 522, 519, 531, 493, 498, 538, 523, 526, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[499, 497, 505, 502, 501, 529, 499, 528, 504, 484, 498, 529, 491, 501, 495, 541, 483, 511, 561, 529, 491, 478, 488, 552, 531, 507, 509, 522, 510, 563, 508, 480, 519, 508, 480, 547, 484, 472, 464, 550, 533, 489, 520, 547, 479, 503, 521, 529, 566, 527, 478, 490, 547, 529, 525, 479, 516, 554, 522, 486, 517, 508, 530, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[492, 523, 515, 518, 541, 495, 533, 532, 502, 545, 539, 513, 515, 514, 507, 493, 513, 535, 520, 484, 501, 492, 525, 530, 485, 509, 536, 520, 477, 564, 516, 492, 468, 542, 513, 495, 529, 504, 524, 504, 527, 507, 492, 519, 494, 485, 496, 486, 542, 518, 528, 497, 483, 553, 530, 526, 518, 495, 497, 503, 519, 493, 482, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[505, 491, 548, 510, 489, 518, 519, 505, 7, 0, 0, 0, 0, 0, 0, 0, 515, 504, 499, 528, 505, 506, 502, 496, 0, 0, 0, 0, 0, 0, 0, 0, 492, 536, 506, 511, 493, 502, 499, 513, 0, 0, 0, 0, 0, 0, 0, 0, 544, 480, 506, 527, 483, 530, 483, 512, 10, 7, 0, 0, 0, 0, 0, 0, 500, 528, 527, 512, 524, 497, 510, 516, 4, 2, 0, 0, 0, 0, 0, 0, 510, 481, 547, 497, 514, 517, 504, 523, 20, 0, 0, 0, 0, 0, 0, 0, 513, 517, 497, 486, 576, 508, 526, 513, 0, 0, 0, 0, 0, 0, 0, 0, 480, 531, 563, 480, 538, 484, 531, 511, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4055, 4052, 4082, 4120, 4113, 4136, 4118] avg=4096.0000 max=4136 max_rank=6 min=4052 min_rank=2 imbalance_ratio=1.009766 max_load_reduction=28632 max_load_reduction_pct=0.873779 moved_to_echo=16487 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[511, 524, 468, 512, 488, 495, 514, 447, 7, 0, 0, 0, 0, 0, 0, 0, 522, 504, 508, 507, 542, 530, 545, 531, 0, 0, 0, 0, 0, 0, 0, 0, 531, 519, 524, 510, 520, 535, 483, 486, 0, 0, 0, 0, 0, 0, 0, 0, 506, 532, 514, 510, 495, 540, 522, 537, 10, 7, 0, 0, 0, 0, 0, 0, 527, 487, 526, 496, 515, 483, 487, 521, 4, 2, 0, 0, 0, 0, 0, 0, 475, 508, 488, 491, 520, 521, 536, 507, 20, 0, 0, 0, 0, 0, 0, 0, 511, 509, 531, 506, 495, 496, 517, 486, 0, 0, 0, 0, 0, 0, 0, 0, 535, 511, 507, 565, 508, 498, 509, 534, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3966, 4189, 4108, 4173, 4048, 4066, 4051, 4167] avg=4096.0000 max=4189 max_rank=1 min=3966 min_rank=0 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16332 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[519, 527, 488, 532, 500, 490, 512, 507, 6, 0, 0, 0, 0, 0, 0, 0, 487, 511, 526, 491, 533, 522, 526, 504, 0, 0, 0, 0, 0, 0, 0, 0, 556, 545, 501, 517, 549, 497, 510, 524, 0, 0, 0, 0, 0, 0, 0, 0, 505, 487, 491, 480, 511, 522, 510, 512, 10, 7, 0, 0, 0, 0, 0, 0, 526, 541, 526, 491, 520, 528, 541, 538, 4, 2, 0, 0, 0, 0, 0, 0, 509, 493, 509, 496, 528, 530, 512, 520, 20, 0, 0, 0, 0, 0, 0, 0, 496, 484, 490, 497, 522, 539, 489, 500, 0, 0, 0, 0, 0, 0, 0, 0, 489, 486, 520, 482, 507, 498, 509, 511, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4100, 4199, 4035, 4217, 4117, 4017, 4002] avg=4096.0000 max=4217 max_rank=4 min=4002 min_rank=7 imbalance_ratio=1.029541 max_load_reduction=28551 max_load_reduction_pct=0.871307 moved_to_echo=16353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[570, 542, 497, 498, 547, 493, 532, 530, 10, 0, 0, 0, 0, 0, 0, 0, 552, 488, 507, 511, 508, 486, 498, 532, 0, 0, 0, 0, 0, 0, 0, 0, 541, 477, 514, 511, 520, 572, 500, 499, 0, 0, 0, 0, 0, 0, 0, 0, 543, 492, 496, 477, 509, 503, 537, 549, 15, 12, 0, 0, 0, 0, 0, 0, 480, 532, 505, 498, 499, 487, 481, 518, 8, 4, 3, 0, 0, 0, 0, 0, 506, 513, 505, 498, 477, 540, 491, 493, 24, 0, 0, 0, 0, 0, 0, 0, 517, 521, 482, 528, 492, 520, 480, 494, 0, 0, 0, 0, 0, 0, 0, 0, 530, 543, 523, 561, 505, 457, 499, 486, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4219, 4082, 4134, 4133, 4015, 4047, 4034, 4104] avg=4096.0000 max=4219 max_rank=0 min=4015 min_rank=4 imbalance_ratio=1.030029 max_load_reduction=28549 max_load_reduction_pct=0.871246 moved_to_echo=16200 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[539, 539, 469, 500, 506, 499, 527, 497, 6, 0, 0, 0, 0, 0, 0, 0, 477, 509, 491, 492, 525, 559, 517, 503, 0, 0, 0, 0, 0, 0, 0, 0, 507, 486, 546, 487, 555, 476, 462, 509, 0, 0, 0, 0, 0, 0, 0, 0, 525, 521, 521, 516, 509, 474, 491, 484, 10, 7, 0, 0, 0, 0, 0, 0, 466, 513, 511, 486, 517, 510, 507, 543, 4, 1, 0, 0, 0, 0, 0, 0, 497, 525, 479, 533, 533, 547, 524, 526, 19, 0, 0, 0, 0, 0, 0, 0, 479, 483, 531, 534, 541, 489, 559, 522, 0, 0, 0, 0, 0, 0, 0, 0, 519, 531, 493, 498, 538, 523, 526, 520, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4073, 4028, 4058, 4058, 4183, 4138, 4148] avg=4096.0000 max=4183 max_rank=5 min=4028 min_rank=2 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16527 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[514, 518, 508, 538, 519, 505, 505, 492, 6, 0, 0, 0, 0, 0, 0, 0, 518, 538, 515, 505, 506, 484, 514, 526, 0, 0, 0, 0, 0, 0, 0, 0, 515, 540, 489, 476, 497, 549, 506, 538, 0, 0, 0, 0, 0, 0, 0, 0, 490, 513, 507, 498, 488, 489, 542, 480, 9, 7, 0, 0, 0, 0, 0, 0, 543, 561, 528, 507, 511, 510, 515, 514, 4, 2, 0, 0, 0, 0, 0, 0, 503, 513, 518, 502, 512, 503, 488, 520, 19, 0, 0, 0, 0, 0, 0, 0, 481, 521, 502, 549, 497, 532, 512, 514, 0, 0, 0, 0, 0, 0, 0, 0, 532, 470, 509, 492, 501, 531, 487, 521, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4106, 4110, 4023, 4195, 4078, 4108, 4043] avg=4096.0000 max=4195 max_rank=4 min=4023 min_rank=3 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16424 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[499, 497, 505, 502, 501, 529, 499, 528, 7, 0, 0, 0, 0, 0, 0, 0, 504, 484, 498, 529, 491, 501, 491, 541, 0, 0, 0, 0, 0, 0, 0, 0, 483, 511, 561, 500, 491, 478, 488, 552, 0, 0, 0, 0, 0, 0, 0, 0, 531, 507, 509, 522, 510, 563, 508, 480, 10, 8, 0, 0, 0, 0, 0, 0, 519, 508, 480, 547, 484, 472, 464, 550, 4, 2, 0, 0, 0, 0, 0, 0, 533, 489, 520, 547, 479, 503, 521, 529, 19, 0, 0, 0, 0, 0, 0, 0, 549, 527, 478, 490, 547, 529, 525, 479, 0, 0, 0, 0, 0, 0, 0, 0, 516, 554, 522, 486, 517, 508, 530, 523, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4039, 4064, 4148, 4030, 4140, 4124, 4156] avg=4096.0000 max=4156 max_rank=7 min=4030 min_rank=4 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16450 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[492, 523, 515, 518, 541, 495, 533, 532, 7, 0, 0, 0, 0, 0, 0, 0, 502, 545, 539, 513, 515, 514, 503, 493, 0, 0, 0, 0, 0, 0, 0, 0, 513, 535, 520, 457, 501, 492, 525, 530, 0, 0, 0, 0, 0, 0, 0, 0, 485, 509, 536, 520, 477, 564, 516, 492, 9, 8, 0, 0, 0, 0, 0, 0, 468, 542, 513, 495, 529, 504, 524, 504, 4, 2, 0, 0, 0, 0, 0, 0, 527, 507, 492, 519, 494, 485, 496, 486, 18, 0, 0, 0, 0, 0, 0, 0, 525, 518, 528, 497, 483, 553, 530, 526, 0, 0, 0, 0, 0, 0, 0, 0, 518, 495, 497, 503, 519, 493, 482, 523, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4156, 4124, 4073, 4116, 4085, 4024, 4160, 4030] avg=4096.0000 max=4160 max_rank=6 min=4024 min_rank=5 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16299 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[494, 532, 507, 484, 514, 482, 514, 526, 509, 465, 543, 534, 538, 505, 529, 490, 479, 504, 534, 498, 506, 517, 502, 510, 508, 539, 526, 512, 494, 493, 540, 521, 535, 475, 473, 494, 527, 544, 528, 528, 512, 541, 482, 503, 539, 535, 507, 494, 520, 507, 517, 498, 544, 521, 553, 507, 483, 466, 487, 520, 539, 515, 508, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[485, 550, 491, 527, 479, 504, 500, 530, 494, 507, 490, 505, 524, 504, 503, 525, 493, 534, 519, 491, 525, 513, 511, 552, 496, 538, 510, 494, 543, 550, 510, 489, 518, 485, 515, 479, 493, 493, 508, 499, 473, 521, 531, 469, 499, 488, 514, 504, 528, 529, 510, 562, 535, 550, 493, 520, 536, 530, 551, 481, 490, 518, 537, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[523, 493, 534, 532, 513, 495, 506, 520, 484, 545, 511, 502, 546, 483, 469, 491, 535, 533, 494, 490, 553, 525, 506, 535, 494, 516, 488, 544, 527, 547, 479, 479, 482, 530, 505, 534, 493, 541, 515, 511, 525, 518, 516, 503, 542, 527, 507, 514, 502, 503, 495, 486, 507, 484, 529, 538, 524, 527, 466, 518, 523, 496, 522, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[521, 523, 546, 488, 482, 506, 521, 478, 545, 488, 538, 522, 493, 530, 533, 541, 504, 470, 483, 526, 583, 509, 475, 519, 510, 478, 491, 503, 500, 546, 537, 518, 524, 525, 534, 462, 516, 493, 504, 528, 517, 497, 542, 529, 528, 526, 545, 523, 474, 503, 508, 480, 519, 512, 519, 507, 525, 488, 480, 504, 504, 524, 508, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[493, 527, 510, 511, 523, 499, 523, 495, 517, 471, 545, 542, 507, 497, 522, 519, 523, 494, 512, 511, 502, 529, 486, 546, 541, 497, 488, 550, 509, 495, 518, 535, 551, 517, 519, 481, 518, 518, 520, 471, 495, 495, 481, 490, 527, 514, 517, 513, 512, 467, 545, 501, 492, 573, 523, 502, 481, 502, 544, 524, 528, 457, 552, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[514, 522, 531, 530, 514, 505, 512, 546, 487, 484, 513, 489, 560, 494, 529, 480, 505, 499, 493, 534, 523, 530, 524, 533, 523, 510, 530, 530, 522, 514, 483, 498, 535, 514, 517, 484, 512, 523, 480, 458, 495, 512, 536, 478, 530, 521, 518, 506, 498, 518, 499, 519, 555, 484, 493, 516, 514, 519, 529, 508, 496, 506, 519, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[490, 521, 532, 499, 515, 521, 481, 522, 554, 513, 502, 511, 504, 522, 537, 552, 520, 499, 555, 528, 503, 540, 505, 569, 508, 520, 538, 501, 543, 521, 496, 523, 464, 505, 513, 478, 501, 515, 481, 514, 509, 508, 496, 547, 503, 481, 497, 512, 508, 527, 485, 539, 499, 549, 487, 502, 535, 518, 476, 510, 490, 482, 498, 494] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[478, 488, 486, 533, 500, 533, 544, 500, 534, 558, 515, 518, 545, 514, 525, 494, 454, 491, 483, 556, 485, 492, 518, 511, 507, 571, 544, 507, 499, 507, 493, 494, 538, 479, 484, 510, 498, 522, 513, 516, 506, 476, 537, 523, 493, 498, 514, 519, 502, 522, 506, 533, 548, 536, 530, 547, 508, 481, 509, 525, 496, 510, 506, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[494, 532, 507, 484, 514, 482, 514, 526, 7, 0, 0, 0, 0, 0, 0, 0, 509, 465, 543, 534, 517, 505, 529, 490, 0, 0, 0, 0, 0, 0, 0, 0, 479, 504, 534, 498, 506, 517, 502, 498, 0, 0, 0, 0, 0, 0, 0, 0, 508, 539, 526, 512, 494, 475, 540, 521, 0, 0, 0, 0, 0, 0, 0, 0, 535, 475, 473, 494, 527, 544, 528, 528, 25, 15, 0, 0, 0, 0, 0, 0, 512, 541, 482, 503, 539, 535, 507, 494, 5, 0, 0, 0, 0, 0, 0, 0, 520, 507, 517, 498, 544, 496, 553, 507, 0, 0, 0, 0, 0, 0, 0, 0, 483, 466, 487, 520, 539, 515, 508, 517, 18, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4092, 4038, 4115, 4144, 4118, 4142, 4059] avg=4096.0000 max=4144 max_rank=4 min=4038 min_rank=2 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16463 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[485, 550, 491, 527, 479, 504, 500, 530, 8, 0, 0, 0, 0, 0, 0, 0, 494, 507, 490, 505, 504, 504, 503, 525, 0, 0, 0, 0, 0, 0, 0, 0, 493, 534, 519, 491, 525, 513, 511, 538, 0, 0, 0, 0, 0, 0, 0, 0, 496, 538, 510, 494, 543, 529, 510, 489, 0, 0, 0, 0, 0, 0, 0, 0, 518, 485, 515, 479, 493, 493, 508, 499, 27, 14, 0, 0, 0, 0, 0, 0, 473, 521, 531, 469, 499, 488, 514, 504, 6, 0, 0, 0, 0, 0, 0, 0, 528, 529, 510, 562, 535, 523, 493, 520, 0, 0, 0, 0, 0, 0, 0, 0, 536, 530, 551, 481, 490, 518, 537, 523, 21, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4032, 4124, 4109, 4031, 4005, 4200, 4193] avg=4096.0000 max=4200 max_rank=6 min=4005 min_rank=5 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[493, 527, 510, 511, 523, 499, 523, 495, 8, 0, 0, 0, 0, 0, 0, 0, 517, 471, 545, 542, 487, 497, 522, 519, 0, 0, 0, 0, 0, 0, 0, 0, 523, 494, 512, 511, 502, 529, 486, 532, 0, 0, 0, 0, 0, 0, 0, 0, 541, 497, 488, 550, 509, 477, 518, 535, 0, 0, 0, 0, 0, 0, 0, 0, 551, 517, 519, 481, 518, 518, 520, 471, 28, 14, 0, 0, 0, 0, 0, 0, 495, 495, 481, 490, 527, 514, 517, 513, 6, 0, 0, 0, 0, 0, 0, 0, 512, 467, 545, 501, 492, 545, 523, 502, 0, 0, 0, 0, 0, 0, 0, 0, 481, 502, 544, 524, 528, 457, 552, 501, 18, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4100, 4089, 4115, 4137, 4038, 4087, 4113] avg=4096.0000 max=4137 max_rank=4 min=4038 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[521, 523, 546, 488, 482, 506, 521, 478, 7, 0, 0, 0, 0, 0, 0, 0, 545, 488, 538, 522, 474, 530, 533, 541, 0, 0, 0, 0, 0, 0, 0, 0, 504, 470, 483, 526, 583, 509, 475, 506, 0, 0, 0, 0, 0, 0, 0, 0, 510, 478, 491, 503, 500, 526, 537, 518, 0, 0, 0, 0, 0, 0, 0, 0, 524, 525, 534, 462, 516, 493, 504, 528, 25, 13, 0, 0, 0, 0, 0, 0, 517, 497, 542, 529, 528, 526, 545, 523, 6, 0, 0, 0, 0, 0, 0, 0, 474, 503, 508, 480, 519, 487, 519, 507, 0, 0, 0, 0, 0, 0, 0, 0, 525, 488, 480, 504, 504, 524, 508, 513, 20, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4171, 4056, 4063, 4124, 4213, 3997, 4072] avg=4096.0000 max=4213 max_rank=5 min=3997 min_rank=6 imbalance_ratio=1.028564 max_load_reduction=28555 max_load_reduction_pct=0.871429 moved_to_echo=16406 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[523, 493, 534, 532, 513, 495, 506, 520, 7, 0, 0, 0, 0, 0, 0, 0, 484, 545, 511, 502, 525, 483, 469, 491, 0, 0, 0, 0, 0, 0, 0, 0, 535, 533, 494, 490, 553, 525, 506, 522, 0, 0, 0, 0, 0, 0, 0, 0, 494, 516, 488, 544, 527, 527, 479, 479, 0, 0, 0, 0, 0, 0, 0, 0, 482, 530, 505, 534, 493, 541, 515, 511, 24, 15, 0, 0, 0, 0, 0, 0, 525, 518, 516, 503, 542, 527, 507, 514, 6, 0, 0, 0, 0, 0, 0, 0, 502, 503, 495, 486, 507, 460, 529, 538, 0, 0, 0, 0, 0, 0, 0, 0, 524, 527, 466, 518, 523, 496, 522, 493, 20, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4123, 4010, 4158, 4054, 4150, 4158, 4020, 4095] avg=4096.0000 max=4158 max_rank=2 min=4010 min_rank=1 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16423 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[514, 522, 531, 530, 514, 505, 512, 546, 11, 7, 0, 0, 0, 0, 0, 0, 487, 484, 513, 489, 530, 494, 529, 480, 0, 0, 0, 0, 0, 0, 0, 0, 505, 499, 493, 534, 523, 530, 524, 512, 0, 0, 0, 0, 0, 0, 0, 0, 523, 510, 530, 530, 522, 482, 483, 498, 0, 0, 0, 0, 0, 0, 0, 0, 535, 514, 517, 484, 512, 523, 480, 458, 27, 19, 0, 0, 0, 0, 0, 0, 495, 512, 536, 478, 530, 521, 518, 506, 10, 0, 0, 0, 0, 0, 0, 0, 498, 518, 499, 519, 555, 457, 493, 516, 0, 0, 0, 0, 0, 0, 0, 0, 514, 519, 529, 508, 496, 506, 519, 515, 25, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4192, 4006, 4120, 4078, 4069, 4106, 4055, 4142] avg=4096.0000 max=4192 max_rank=0 min=4006 min_rank=1 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[490, 521, 532, 499, 515, 521, 481, 522, 8, 0, 0, 0, 0, 0, 0, 0, 554, 513, 502, 511, 484, 522, 537, 552, 0, 0, 0, 0, 0, 0, 0, 0, 520, 499, 555, 528, 503, 540, 505, 555, 0, 0, 0, 0, 0, 0, 0, 0, 508, 520, 538, 501, 543, 502, 496, 523, 0, 0, 0, 0, 0, 0, 0, 0, 464, 505, 513, 478, 501, 515, 481, 514, 27, 14, 0, 0, 0, 0, 0, 0, 509, 508, 496, 547, 503, 481, 497, 512, 6, 0, 0, 0, 0, 0, 0, 0, 508, 527, 485, 539, 499, 522, 487, 502, 0, 0, 0, 0, 0, 0, 0, 0, 535, 518, 476, 510, 490, 482, 498, 494, 19, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4175, 4205, 4131, 4012, 4059, 4069, 4028] avg=4096.0000 max=4205 max_rank=2 min=4012 min_rank=4 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16168 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[478, 488, 486, 533, 500, 533, 544, 500, 7, 0, 0, 0, 0, 0, 0, 0, 534, 558, 515, 518, 524, 514, 525, 494, 0, 0, 0, 0, 0, 0, 0, 0, 454, 491, 483, 556, 485, 492, 518, 499, 0, 0, 0, 0, 0, 0, 0, 0, 507, 571, 544, 507, 499, 488, 493, 494, 0, 0, 0, 0, 0, 0, 0, 0, 538, 479, 484, 510, 498, 522, 513, 516, 26, 15, 0, 0, 0, 0, 0, 0, 506, 476, 537, 523, 493, 498, 514, 519, 5, 0, 0, 0, 0, 0, 0, 0, 502, 522, 506, 533, 548, 510, 530, 547, 0, 0, 0, 0, 0, 0, 0, 0, 508, 481, 509, 525, 496, 510, 506, 506, 19, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4182, 3978, 4103, 4101, 4071, 4198, 4066] avg=4096.0000 max=4198 max_rank=6 min=3978 min_rank=2 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16436 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 3968, 4096, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[529, 523, 498, 498, 526, 503, 524, 500, 516, 490, 549, 564, 561, 515, 481, 515, 506, 506, 521, 512, 513, 537, 516, 525, 501, 499, 483, 540, 470, 510, 496, 516, 543, 524, 485, 549, 490, 523, 483, 516, 499, 526, 530, 525, 539, 505, 511, 479, 477, 480, 548, 504, 508, 516, 510, 501, 507, 515, 511, 493, 509, 498, 509, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[521, 532, 527, 518, 554, 529, 536, 477, 530, 501, 499, 515, 504, 512, 521, 492, 546, 537, 509, 508, 512, 474, 502, 528, 483, 530, 509, 520, 524, 509, 518, 513, 529, 512, 503, 521, 498, 498, 498, 529, 511, 496, 556, 501, 496, 536, 478, 513, 559, 499, 517, 512, 495, 479, 507, 527, 502, 475, 496, 469, 515, 514, 551, 486] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[468, 524, 551, 516, 492, 513, 538, 532, 531, 509, 521, 524, 517, 541, 525, 534, 495, 516, 534, 495, 471, 498, 521, 472, 545, 497, 520, 495, 507, 481, 517, 545, 513, 495, 513, 507, 506, 571, 519, 493, 484, 519, 515, 516, 534, 500, 506, 555, 513, 505, 521, 490, 553, 481, 496, 523, 488, 523, 460, 530, 478, 503, 511, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[514, 493, 513, 528, 491, 514, 457, 506, 516, 514, 498, 525, 536, 464, 541, 511, 507, 515, 502, 491, 514, 537, 556, 525, 542, 513, 543, 496, 456, 513, 505, 517, 490, 503, 465, 492, 494, 528, 509, 510, 520, 471, 522, 506, 519, 527, 512, 529, 486, 541, 507, 506, 512, 545, 554, 477, 504, 518, 521, 528, 497, 540, 506, 576] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[533, 536, 488, 507, 537, 500, 506, 539, 485, 470, 511, 474, 510, 509, 551, 529, 493, 479, 489, 497, 496, 460, 563, 482, 553, 531, 581, 540, 523, 527, 517, 522, 497, 504, 522, 491, 545, 506, 479, 511, 508, 541, 519, 485, 540, 495, 500, 484, 547, 547, 494, 503, 523, 528, 497, 487, 521, 552, 504, 511, 517, 481, 480, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[522, 521, 506, 555, 504, 493, 506, 476, 503, 518, 469, 493, 486, 516, 475, 520, 511, 530, 505, 523, 508, 502, 510, 509, 478, 477, 527, 501, 502, 513, 513, 467, 495, 503, 498, 509, 525, 499, 512, 529, 522, 536, 511, 537, 518, 535, 499, 540, 528, 526, 499, 559, 530, 501, 477, 595, 524, 491, 521, 541, 510, 495, 524, 540] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[457, 526, 521, 486, 503, 483, 511, 520, 522, 475, 510, 501, 485, 500, 537, 491, 492, 515, 494, 487, 500, 508, 481, 551, 557, 510, 500, 511, 535, 530, 546, 513, 541, 536, 489, 526, 486, 524, 507, 484, 512, 536, 500, 518, 547, 501, 534, 500, 511, 459, 502, 514, 499, 517, 539, 529, 515, 536, 538, 545, 488, 546, 495, 536] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[533, 511, 515, 490, 510, 528, 508, 535, 523, 526, 524, 506, 512, 514, 519, 490, 551, 498, 487, 491, 482, 525, 513, 540, 491, 503, 491, 503, 501, 505, 482, 518, 500, 538, 516, 526, 485, 489, 491, 512, 484, 514, 516, 506, 515, 516, 516, 502, 530, 528, 495, 526, 530, 484, 504, 532, 497, 519, 510, 519, 547, 510, 538, 548] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[529, 519, 498, 498, 526, 503, 524, 500, 0, 0, 0, 0, 0, 0, 0, 0, 516, 490, 549, 564, 561, 515, 481, 515, 3, 1, 0, 0, 0, 0, 0, 0, 506, 506, 521, 512, 513, 537, 516, 525, 19, 4, 0, 0, 0, 0, 0, 0, 501, 499, 482, 540, 470, 510, 496, 516, 0, 0, 0, 0, 0, 0, 0, 0, 543, 524, 485, 549, 490, 523, 483, 516, 10, 9, 1, 0, 0, 0, 0, 0, 499, 526, 530, 525, 520, 505, 511, 479, 0, 0, 0, 0, 0, 0, 0, 0, 477, 480, 548, 504, 508, 516, 510, 488, 0, 0, 0, 0, 0, 0, 0, 0, 507, 515, 511, 493, 509, 498, 509, 502, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4195, 4159, 4014, 4133, 4095, 4031, 4044] avg=4096.0000 max=4195 max_rank=1 min=4014 min_rank=3 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16303 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[521, 528, 527, 518, 554, 529, 536, 477, 0, 0, 0, 0, 0, 0, 0, 0, 530, 501, 499, 515, 504, 512, 521, 492, 3, 1, 0, 0, 0, 0, 0, 0, 546, 537, 509, 508, 512, 474, 502, 528, 18, 5, 0, 0, 0, 0, 0, 0, 483, 530, 508, 520, 524, 509, 518, 513, 0, 0, 0, 0, 0, 0, 0, 0, 529, 512, 503, 521, 498, 498, 498, 529, 10, 10, 1, 0, 0, 0, 0, 0, 511, 496, 556, 501, 478, 536, 478, 513, 0, 0, 0, 0, 0, 0, 0, 0, 559, 499, 517, 512, 495, 479, 507, 512, 0, 0, 0, 0, 0, 0, 0, 0, 502, 475, 496, 469, 515, 514, 551, 476, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4190, 4078, 4139, 4105, 4109, 4069, 4080, 3998] avg=4096.0000 max=4190 max_rank=0 min=3998 min_rank=7 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16256 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[468, 520, 551, 516, 492, 513, 538, 532, 0, 0, 0, 0, 0, 0, 0, 0, 531, 509, 521, 524, 517, 541, 525, 534, 3, 1, 0, 0, 0, 0, 0, 0, 495, 516, 534, 495, 471, 498, 521, 472, 19, 5, 0, 0, 0, 0, 0, 0, 545, 497, 519, 495, 507, 481, 517, 545, 0, 0, 0, 0, 0, 0, 0, 0, 513, 495, 513, 507, 506, 571, 519, 493, 10, 10, 1, 0, 0, 0, 0, 0, 484, 519, 515, 516, 515, 500, 506, 555, 0, 0, 0, 0, 0, 0, 0, 0, 513, 505, 521, 490, 553, 481, 496, 508, 0, 0, 0, 0, 0, 0, 0, 0, 488, 523, 460, 530, 478, 503, 511, 492, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4130, 4206, 4026, 4106, 4138, 4110, 4067, 3985] avg=4096.0000 max=4206 max_rank=1 min=3985 min_rank=7 imbalance_ratio=1.026855 max_load_reduction=28562 max_load_reduction_pct=0.871643 moved_to_echo=16300 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[533, 532, 488, 507, 537, 500, 506, 539, 0, 0, 0, 0, 0, 0, 0, 0, 485, 470, 511, 474, 510, 509, 551, 529, 3, 1, 0, 0, 0, 0, 0, 0, 493, 479, 489, 497, 496, 460, 563, 482, 19, 4, 0, 0, 0, 0, 0, 0, 553, 531, 580, 540, 523, 527, 517, 522, 0, 0, 0, 0, 0, 0, 0, 0, 497, 504, 522, 491, 545, 506, 479, 511, 10, 9, 1, 0, 0, 0, 0, 0, 508, 541, 519, 485, 521, 495, 500, 484, 0, 0, 0, 0, 0, 0, 0, 0, 547, 547, 494, 503, 523, 528, 497, 474, 0, 0, 0, 0, 0, 0, 0, 0, 521, 552, 504, 511, 517, 481, 480, 501, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4142, 4043, 3982, 4293, 4075, 4053, 4113, 4067] avg=4096.0000 max=4293 max_rank=3 min=3982 min_rank=2 imbalance_ratio=1.048096 max_load_reduction=28475 max_load_reduction_pct=0.868988 moved_to_echo=16308 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[514, 489, 513, 528, 491, 514, 457, 506, 0, 0, 0, 0, 0, 0, 0, 0, 516, 514, 498, 525, 536, 464, 541, 511, 3, 1, 0, 0, 0, 0, 0, 0, 507, 515, 502, 491, 514, 537, 556, 525, 19, 4, 0, 0, 0, 0, 0, 0, 542, 513, 542, 496, 456, 513, 505, 517, 0, 0, 0, 0, 0, 0, 0, 0, 490, 503, 465, 492, 494, 528, 509, 510, 12, 9, 1, 0, 0, 0, 0, 0, 520, 471, 522, 506, 500, 527, 512, 529, 0, 0, 0, 0, 0, 0, 0, 0, 486, 541, 507, 506, 512, 545, 554, 464, 0, 0, 0, 0, 0, 0, 0, 0, 504, 518, 521, 528, 497, 540, 506, 564, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4012, 4109, 4170, 4084, 4013, 4087, 4115, 4178] avg=4096.0000 max=4178 max_rank=7 min=4012 min_rank=0 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[457, 511, 521, 486, 503, 483, 511, 520, 0, 0, 0, 0, 0, 0, 0, 0, 522, 475, 510, 501, 485, 500, 537, 491, 8, 6, 0, 0, 0, 0, 0, 0, 492, 515, 494, 487, 500, 508, 481, 551, 24, 8, 0, 0, 0, 0, 0, 0, 557, 510, 494, 511, 535, 530, 546, 513, 0, 0, 0, 0, 0, 0, 0, 0, 541, 536, 489, 526, 486, 524, 507, 484, 15, 13, 7, 0, 0, 0, 0, 0, 512, 536, 500, 518, 523, 501, 534, 500, 0, 0, 0, 0, 0, 0, 0, 0, 511, 459, 502, 514, 499, 517, 539, 508, 0, 0, 0, 0, 0, 0, 0, 0, 515, 536, 538, 545, 488, 546, 495, 521, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3992, 4035, 4060, 4196, 4128, 4124, 4049, 4184] avg=4096.0000 max=4196 max_rank=3 min=3992 min_rank=0 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16485 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[522, 517, 506, 555, 504, 493, 506, 476, 0, 0, 0, 0, 0, 0, 0, 0, 503, 518, 469, 493, 486, 516, 475, 520, 3, 1, 0, 0, 0, 0, 0, 0, 511, 530, 505, 523, 508, 502, 510, 509, 19, 5, 0, 0, 0, 0, 0, 0, 478, 477, 526, 501, 502, 513, 513, 467, 0, 0, 0, 0, 0, 0, 0, 0, 495, 503, 498, 509, 525, 499, 512, 529, 11, 11, 1, 0, 0, 0, 0, 0, 522, 536, 511, 537, 499, 535, 499, 540, 0, 0, 0, 0, 0, 0, 0, 0, 528, 526, 499, 559, 530, 501, 477, 579, 0, 0, 0, 0, 0, 0, 0, 0, 524, 491, 521, 541, 510, 495, 524, 529, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 3984, 4122, 3977, 4093, 4179, 4199, 4135] avg=4096.0000 max=4199 max_rank=6 min=3977 min_rank=3 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16606 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[533, 507, 515, 490, 510, 528, 508, 535, 0, 0, 0, 0, 0, 0, 0, 0, 523, 526, 524, 506, 512, 514, 519, 490, 3, 1, 0, 0, 0, 0, 0, 0, 551, 498, 487, 491, 482, 525, 513, 540, 18, 5, 0, 0, 0, 0, 0, 0, 491, 503, 490, 503, 501, 505, 482, 518, 0, 0, 0, 0, 0, 0, 0, 0, 500, 538, 516, 526, 485, 489, 491, 512, 11, 10, 1, 0, 0, 0, 0, 0, 484, 514, 516, 506, 497, 516, 516, 502, 0, 0, 0, 0, 0, 0, 0, 0, 530, 528, 495, 526, 530, 484, 504, 517, 0, 0, 0, 0, 0, 0, 0, 0, 497, 519, 510, 519, 547, 510, 538, 537, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4118, 4110, 3993, 4079, 4051, 4114, 4177] avg=4096.0000 max=4177 max_rank=7 min=3993 min_rank=3 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16421 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[493, 492, 500, 511, 536, 487, 479, 480, 510, 511, 476, 522, 514, 494, 512, 475, 529, 493, 563, 499, 511, 497, 508, 519, 537, 529, 522, 498, 528, 534, 558, 545, 487, 509, 485, 514, 520, 534, 536, 524, 497, 541, 493, 512, 483, 546, 537, 546, 552, 507, 540, 481, 522, 487, 525, 494, 510, 530, 504, 508, 496, 513, 486, 487] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[504, 553, 495, 551, 518, 522, 499, 525, 518, 463, 498, 502, 518, 508, 524, 501, 501, 525, 491, 530, 558, 517, 516, 465, 502, 538, 473, 494, 547, 541, 552, 519, 506, 493, 498, 561, 508, 509, 520, 529, 495, 501, 505, 534, 523, 492, 517, 528, 511, 514, 481, 500, 515, 515, 534, 521, 512, 491, 499, 468, 491, 493, 520, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[452, 515, 548, 534, 484, 528, 521, 500, 526, 526, 519, 507, 546, 515, 478, 490, 526, 544, 496, 509, 511, 520, 503, 516, 479, 510, 553, 472, 537, 517, 480, 504, 528, 506, 539, 500, 532, 488, 471, 506, 522, 509, 477, 532, 525, 535, 481, 541, 487, 493, 503, 484, 475, 532, 502, 511, 523, 498, 508, 544, 553, 538, 539, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[525, 500, 530, 528, 530, 518, 505, 525, 532, 519, 518, 502, 531, 485, 546, 503, 502, 528, 521, 505, 485, 484, 511, 495, 491, 496, 500, 499, 502, 505, 526, 526, 504, 507, 543, 451, 546, 481, 504, 524, 526, 515, 534, 504, 514, 503, 474, 509, 525, 482, 470, 520, 508, 552, 534, 531, 526, 515, 532, 502, 503, 490, 524, 542] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[497, 530, 495, 521, 512, 532, 525, 541, 493, 510, 498, 479, 495, 504, 493, 483, 552, 506, 541, 489, 505, 473, 480, 560, 454, 540, 502, 498, 516, 522, 534, 515, 520, 515, 502, 525, 517, 489, 517, 494, 527, 483, 512, 496, 525, 530, 531, 529, 521, 516, 516, 515, 546, 521, 485, 491, 523, 507, 511, 563, 511, 507, 490, 538] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[526, 529, 494, 531, 502, 496, 501, 480, 535, 500, 530, 501, 489, 523, 477, 498, 498, 499, 536, 541, 501, 473, 525, 494, 511, 547, 479, 524, 521, 515, 479, 481, 524, 532, 551, 508, 517, 525, 498, 553, 491, 542, 520, 517, 490, 514, 530, 504, 556, 506, 516, 487, 503, 486, 512, 510, 524, 489, 544, 494, 525, 510, 521, 533] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[531, 510, 559, 538, 487, 509, 508, 514, 504, 490, 499, 500, 506, 512, 490, 505, 502, 512, 537, 528, 499, 520, 524, 536, 503, 497, 505, 491, 475, 517, 496, 477, 521, 515, 505, 532, 525, 526, 517, 553, 539, 529, 481, 521, 523, 492, 488, 511, 521, 495, 540, 514, 539, 530, 486, 524, 511, 514, 540, 482, 522, 515, 483, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[553, 511, 466, 545, 491, 510, 553, 488, 529, 492, 511, 493, 505, 475, 536, 494, 493, 521, 502, 526, 543, 491, 489, 537, 481, 506, 512, 526, 520, 521, 536, 504, 527, 563, 481, 512, 505, 469, 508, 526, 542, 495, 548, 535, 490, 529, 519, 523, 520, 522, 502, 477, 524, 515, 519, 526, 536, 462, 530, 517, 498, 498, 489, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[493, 492, 500, 500, 536, 487, 479, 480, 0, 0, 0, 0, 0, 0, 0, 0, 510, 511, 476, 522, 514, 494, 512, 475, 24, 20, 8, 0, 0, 0, 0, 0, 529, 493, 559, 499, 511, 497, 508, 519, 0, 0, 0, 0, 0, 0, 0, 0, 537, 529, 522, 498, 528, 534, 558, 545, 3, 2, 0, 0, 0, 0, 0, 0, 487, 509, 485, 514, 520, 534, 536, 504, 0, 0, 0, 0, 0, 0, 0, 0, 497, 541, 493, 512, 483, 546, 537, 522, 0, 0, 0, 0, 0, 0, 0, 0, 552, 507, 540, 481, 522, 487, 525, 494, 2, 0, 0, 0, 0, 0, 0, 0, 510, 530, 504, 508, 496, 513, 486, 487, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3967, 4066, 4115, 4256, 4089, 4131, 4110, 4034] avg=4096.0000 max=4256 max_rank=3 min=3967 min_rank=0 imbalance_ratio=1.039062 max_load_reduction=28512 max_load_reduction_pct=0.870117 moved_to_echo=16364 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[525, 500, 530, 516, 530, 518, 505, 525, 0, 0, 0, 0, 0, 0, 0, 0, 532, 519, 518, 502, 531, 485, 546, 503, 22, 20, 9, 0, 0, 0, 0, 0, 502, 528, 517, 505, 485, 484, 511, 495, 0, 0, 0, 0, 0, 0, 0, 0, 491, 496, 500, 499, 502, 505, 526, 526, 3, 2, 0, 0, 0, 0, 0, 0, 504, 507, 543, 451, 546, 481, 504, 504, 0, 0, 0, 0, 0, 0, 0, 0, 526, 515, 534, 504, 514, 503, 474, 487, 0, 0, 0, 0, 0, 0, 0, 0, 525, 482, 470, 520, 508, 552, 534, 531, 2, 0, 0, 0, 0, 0, 0, 0, 526, 515, 532, 502, 503, 490, 524, 542, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4149, 4187, 4027, 4050, 4040, 4057, 4124, 4134] avg=4096.0000 max=4187 max_rank=1 min=4027 min_rank=2 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16355 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[504, 553, 495, 538, 518, 522, 499, 525, 0, 0, 0, 0, 0, 0, 0, 0, 518, 463, 498, 502, 518, 508, 524, 501, 23, 20, 9, 0, 0, 0, 0, 0, 501, 525, 487, 530, 558, 517, 516, 465, 0, 0, 0, 0, 0, 0, 0, 0, 502, 538, 473, 494, 547, 541, 552, 519, 4, 2, 0, 0, 0, 0, 0, 0, 506, 493, 498, 561, 508, 509, 520, 509, 0, 0, 0, 0, 0, 0, 0, 0, 495, 501, 505, 534, 523, 492, 517, 505, 0, 0, 0, 0, 0, 0, 0, 0, 511, 514, 481, 500, 515, 515, 534, 521, 2, 0, 0, 0, 0, 0, 0, 0, 512, 491, 499, 468, 491, 493, 520, 516, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4154, 4084, 4099, 4172, 4104, 4072, 4093, 3990] avg=4096.0000 max=4172 max_rank=3 min=3990 min_rank=7 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16259 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[526, 529, 494, 519, 502, 496, 501, 480, 0, 0, 0, 0, 0, 0, 0, 0, 535, 500, 530, 501, 489, 523, 477, 498, 22, 21, 9, 0, 0, 0, 0, 0, 498, 499, 532, 541, 501, 473, 525, 494, 0, 0, 0, 0, 0, 0, 0, 0, 511, 547, 479, 524, 521, 515, 479, 481, 3, 2, 0, 0, 0, 0, 0, 0, 524, 532, 551, 508, 517, 525, 498, 532, 0, 0, 0, 0, 0, 0, 0, 0, 491, 542, 520, 517, 490, 514, 530, 482, 0, 0, 0, 0, 0, 0, 0, 0, 556, 506, 516, 487, 503, 486, 512, 510, 2, 0, 0, 0, 0, 0, 0, 0, 524, 489, 544, 494, 525, 510, 521, 533, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4047, 4105, 4063, 4062, 4187, 4086, 4078, 4140] avg=4096.0000 max=4187 max_rank=4 min=4047 min_rank=0 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16491 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[497, 530, 495, 500, 512, 532, 525, 541, 0, 0, 0, 0, 0, 0, 0, 0, 493, 510, 498, 479, 495, 504, 493, 483, 28, 25, 12, 0, 0, 0, 0, 0, 552, 506, 526, 489, 505, 473, 480, 560, 0, 0, 0, 0, 0, 0, 0, 0, 454, 540, 502, 498, 516, 522, 534, 515, 9, 4, 0, 0, 0, 0, 0, 0, 520, 515, 502, 525, 517, 489, 517, 469, 0, 0, 0, 0, 0, 0, 0, 0, 527, 483, 512, 496, 525, 530, 531, 501, 0, 0, 0, 0, 0, 0, 0, 0, 521, 516, 516, 515, 546, 521, 485, 491, 5, 0, 0, 0, 0, 0, 0, 0, 523, 507, 511, 563, 511, 507, 490, 538, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4132, 4020, 4091, 4094, 4054, 4105, 4116, 4156] avg=4096.0000 max=4156 max_rank=7 min=4020 min_rank=1 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[452, 515, 548, 522, 484, 528, 521, 500, 0, 0, 0, 0, 0, 0, 0, 0, 526, 526, 519, 507, 546, 515, 478, 490, 24, 20, 9, 0, 0, 0, 0, 0, 526, 544, 492, 509, 511, 520, 503, 516, 0, 0, 0, 0, 0, 0, 0, 0, 479, 510, 553, 472, 537, 517, 480, 504, 3, 2, 0, 0, 0, 0, 0, 0, 528, 506, 539, 500, 532, 488, 471, 486, 0, 0, 0, 0, 0, 0, 0, 0, 522, 509, 477, 532, 525, 535, 481, 517, 0, 0, 0, 0, 0, 0, 0, 0, 487, 493, 503, 484, 475, 532, 502, 511, 2, 0, 0, 0, 0, 0, 0, 0, 523, 498, 508, 544, 553, 538, 539, 520, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4070, 4160, 4121, 4057, 4050, 4098, 3989, 4223] avg=4096.0000 max=4223 max_rank=7 min=3989 min_rank=6 imbalance_ratio=1.031006 max_load_reduction=28545 max_load_reduction_pct=0.871124 moved_to_echo=16360 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[531, 510, 559, 526, 487, 509, 508, 514, 0, 0, 0, 0, 0, 0, 0, 0, 504, 490, 499, 500, 506, 512, 490, 505, 23, 21, 9, 0, 0, 0, 0, 0, 502, 512, 533, 528, 499, 520, 524, 536, 0, 0, 0, 0, 0, 0, 0, 0, 503, 497, 505, 491, 475, 517, 496, 477, 3, 2, 0, 0, 0, 0, 0, 0, 521, 515, 505, 532, 525, 526, 517, 532, 0, 0, 0, 0, 0, 0, 0, 0, 539, 529, 481, 521, 523, 492, 488, 488, 0, 0, 0, 0, 0, 0, 0, 0, 521, 495, 540, 514, 539, 530, 486, 524, 2, 0, 0, 0, 0, 0, 0, 0, 511, 514, 540, 482, 522, 515, 483, 493, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4144, 4059, 4154, 3966, 4173, 4061, 4151, 4060] avg=4096.0000 max=4173 max_rank=4 min=3966 min_rank=3 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16445 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[553, 511, 466, 533, 491, 510, 553, 488, 0, 0, 0, 0, 0, 0, 0, 0, 529, 492, 511, 493, 505, 475, 536, 494, 23, 20, 9, 0, 0, 0, 0, 0, 493, 521, 498, 526, 543, 491, 489, 537, 0, 0, 0, 0, 0, 0, 0, 0, 481, 506, 512, 526, 520, 521, 536, 504, 3, 2, 0, 0, 0, 0, 0, 0, 527, 563, 481, 512, 505, 469, 508, 506, 0, 0, 0, 0, 0, 0, 0, 0, 542, 495, 548, 535, 490, 529, 519, 500, 0, 0, 0, 0, 0, 0, 0, 0, 520, 522, 502, 477, 524, 515, 519, 526, 2, 0, 0, 0, 0, 0, 0, 0, 536, 462, 530, 517, 498, 498, 489, 501, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4087, 4098, 4111, 4071, 4158, 4107, 4031] avg=4096.0000 max=4158 max_rank=5 min=4031 min_rank=7 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4096, 3968, 256, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3968, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True + [2026-09-21 08:26:48] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 262.1 | throughput per GPU (TFLOP/s/GPU): 122.9 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.026161E+01 | loss scale: 1.0 | grad norm: 28.805 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[525, 517, 503, 530, 524, 517, 480, 503, 469, 522, 487, 491, 511, 479, 562, 528, 543, 476, 502, 485, 508, 502, 492, 536, 489, 533, 546, 507, 520, 515, 522, 486, 493, 531, 496, 507, 522, 540, 475, 516, 508, 497, 516, 464, 546, 528, 516, 536, 487, 518, 518, 521, 511, 526, 513, 481, 488, 518, 540, 584, 500, 493, 543, 526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[518, 519, 514, 463, 521, 494, 519, 522, 540, 512, 478, 539, 499, 508, 487, 547, 506, 521, 502, 500, 505, 495, 530, 507, 507, 498, 512, 520, 520, 547, 487, 515, 537, 521, 529, 526, 480, 502, 505, 492, 481, 523, 529, 483, 513, 520, 526, 509, 512, 463, 558, 511, 537, 498, 532, 513, 522, 513, 503, 518, 561, 520, 487, 492] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[492, 524, 510, 490, 502, 526, 505, 524, 488, 473, 521, 537, 487, 503, 480, 463, 517, 508, 542, 509, 524, 514, 526, 561, 546, 500, 474, 498, 517, 538, 496, 535, 544, 550, 503, 522, 481, 537, 499, 516, 483, 509, 479, 497, 531, 522, 512, 530, 474, 491, 497, 491, 527, 541, 501, 529, 571, 488, 500, 523, 536, 521, 505, 528] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[469, 471, 534, 503, 499, 516, 539, 512, 540, 491, 517, 537, 519, 525, 504, 528, 512, 509, 515, 505, 530, 499, 505, 509, 538, 522, 527, 498, 488, 498, 546, 492, 507, 507, 517, 534, 503, 503, 495, 510, 527, 539, 494, 523, 533, 473, 545, 520, 489, 493, 499, 510, 502, 532, 485, 564, 520, 488, 530, 492, 509, 502, 564, 462] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[508, 502, 481, 538, 522, 501, 523, 521, 487, 534, 514, 484, 525, 539, 540, 478, 545, 524, 544, 508, 494, 526, 527, 546, 532, 526, 502, 486, 537, 499, 532, 476, 512, 511, 509, 506, 518, 493, 488, 500, 509, 529, 531, 492, 492, 539, 513, 473, 498, 511, 502, 539, 507, 537, 514, 525, 537, 478, 516, 491, 486, 495, 506, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[485, 497, 515, 541, 491, 448, 532, 556, 521, 530, 531, 478, 495, 508, 513, 508, 562, 555, 472, 530, 497, 505, 530, 523, 488, 493, 519, 471, 512, 522, 511, 517, 532, 503, 509, 510, 515, 481, 537, 532, 501, 504, 534, 541, 499, 514, 515, 477, 462, 533, 493, 543, 500, 518, 525, 525, 486, 500, 491, 567, 492, 545, 524, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[515, 502, 506, 546, 516, 497, 518, 529, 492, 494, 514, 557, 513, 506, 500, 500, 510, 571, 498, 528, 520, 524, 501, 505, 481, 490, 509, 534, 481, 518, 503, 485, 508, 491, 531, 500, 540, 495, 505, 497, 521, 555, 463, 475, 536, 538, 498, 524, 494, 515, 490, 493, 519, 552, 520, 495, 487, 519, 514, 526, 520, 509, 535, 540] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[495, 524, 474, 529, 545, 490, 517, 533, 479, 512, 518, 560, 478, 524, 524, 515, 502, 582, 553, 540, 507, 498, 486, 509, 508, 499, 488, 507, 509, 487, 510, 523, 505, 512, 494, 522, 528, 545, 500, 537, 505, 527, 499, 503, 505, 485, 521, 503, 507, 490, 479, 476, 515, 521, 529, 546, 500, 518, 491, 515, 510, 528, 532, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[518, 519, 514, 463, 521, 494, 519, 522, 5, 4, 0, 0, 0, 0, 0, 0, 540, 512, 478, 539, 499, 508, 487, 547, 15, 0, 0, 0, 0, 0, 0, 0, 506, 480, 502, 500, 505, 495, 530, 507, 0, 0, 0, 0, 0, 0, 0, 0, 507, 498, 512, 520, 520, 547, 487, 515, 21, 0, 0, 0, 0, 0, 0, 0, 537, 521, 529, 526, 480, 502, 505, 492, 0, 0, 0, 0, 0, 0, 0, 0, 481, 523, 529, 483, 513, 520, 526, 509, 4, 0, 0, 0, 0, 0, 0, 0, 512, 463, 558, 511, 537, 498, 532, 513, 8, 0, 0, 0, 0, 0, 0, 0, 522, 513, 503, 502, 561, 520, 487, 492, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4125, 4025, 4127, 4092, 4088, 4132, 4100] avg=4096.0000 max=4132 max_rank=6 min=4025 min_rank=2 imbalance_ratio=1.008789 max_load_reduction=28636 max_load_reduction_pct=0.873901 moved_to_echo=16412 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[492, 524, 510, 490, 502, 526, 505, 524, 5, 4, 0, 0, 0, 0, 0, 0, 488, 473, 521, 537, 487, 503, 480, 463, 14, 0, 0, 0, 0, 0, 0, 0, 517, 468, 542, 509, 524, 514, 526, 561, 0, 0, 0, 0, 0, 0, 0, 0, 546, 500, 474, 498, 517, 538, 496, 535, 21, 0, 0, 0, 0, 0, 0, 0, 544, 550, 503, 522, 481, 537, 499, 516, 0, 0, 0, 0, 0, 0, 0, 0, 483, 509, 479, 497, 531, 522, 512, 530, 4, 0, 0, 0, 0, 0, 0, 0, 474, 491, 497, 491, 527, 541, 501, 529, 8, 0, 0, 0, 0, 0, 0, 0, 571, 488, 500, 507, 536, 521, 505, 528, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 3966, 4161, 4125, 4152, 4067, 4059, 4156] avg=4096.0000 max=4161 max_rank=2 min=3966 min_rank=1 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16434 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[525, 517, 503, 530, 524, 517, 480, 503, 5, 5, 0, 0, 0, 0, 0, 0, 469, 522, 487, 491, 511, 479, 562, 528, 14, 0, 0, 0, 0, 0, 0, 0, 543, 438, 502, 485, 508, 502, 492, 536, 0, 0, 0, 0, 0, 0, 0, 0, 489, 533, 546, 507, 520, 515, 522, 486, 19, 0, 0, 0, 0, 0, 0, 0, 493, 531, 496, 507, 522, 540, 475, 516, 0, 0, 0, 0, 0, 0, 0, 0, 508, 497, 516, 464, 546, 528, 516, 536, 4, 0, 0, 0, 0, 0, 0, 0, 487, 518, 518, 521, 511, 526, 513, 481, 9, 0, 0, 0, 0, 0, 0, 0, 488, 518, 540, 566, 500, 493, 543, 526, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4063, 4006, 4137, 4080, 4115, 4084, 4174] avg=4096.0000 max=4174 max_rank=7 min=4006 min_rank=2 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16453 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[508, 502, 481, 538, 522, 501, 523, 521, 5, 4, 0, 0, 0, 0, 0, 0, 487, 534, 514, 484, 525, 539, 540, 478, 15, 0, 0, 0, 0, 0, 0, 0, 545, 483, 544, 508, 494, 526, 527, 546, 0, 0, 0, 0, 0, 0, 0, 0, 532, 526, 502, 486, 537, 499, 532, 476, 21, 0, 0, 0, 0, 0, 0, 0, 512, 511, 509, 506, 518, 493, 488, 500, 0, 0, 0, 0, 0, 0, 0, 0, 509, 529, 531, 492, 492, 539, 513, 473, 4, 0, 0, 0, 0, 0, 0, 0, 498, 511, 502, 539, 507, 537, 514, 525, 8, 0, 0, 0, 0, 0, 0, 0, 537, 478, 516, 475, 486, 495, 506, 510, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4116, 4173, 4111, 4037, 4082, 4141, 4003] avg=4096.0000 max=4173 max_rank=2 min=4003 min_rank=7 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16263 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[469, 471, 534, 503, 499, 516, 539, 512, 5, 4, 0, 0, 0, 0, 0, 0, 540, 491, 517, 537, 519, 525, 504, 528, 14, 0, 0, 0, 0, 0, 0, 0, 512, 469, 515, 505, 530, 499, 505, 509, 0, 0, 0, 0, 0, 0, 0, 0, 538, 522, 527, 498, 488, 498, 546, 492, 21, 0, 0, 0, 0, 0, 0, 0, 507, 507, 517, 534, 503, 503, 495, 510, 0, 0, 0, 0, 0, 0, 0, 0, 527, 539, 494, 523, 533, 473, 545, 520, 4, 0, 0, 0, 0, 0, 0, 0, 489, 493, 499, 510, 502, 532, 485, 564, 8, 0, 0, 0, 0, 0, 0, 0, 520, 488, 530, 476, 509, 502, 564, 462, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4052, 4175, 4044, 4130, 4076, 4158, 4082, 4051] avg=4096.0000 max=4175 max_rank=1 min=4044 min_rank=2 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[515, 502, 506, 546, 516, 497, 518, 529, 11, 8, 0, 0, 0, 0, 0, 0, 492, 494, 514, 557, 513, 506, 500, 500, 20, 0, 0, 0, 0, 0, 0, 0, 510, 514, 498, 528, 520, 524, 501, 505, 0, 0, 0, 0, 0, 0, 0, 0, 481, 490, 509, 534, 481, 518, 503, 485, 26, 0, 0, 0, 0, 0, 0, 0, 508, 491, 531, 500, 540, 495, 505, 497, 2, 0, 0, 0, 0, 0, 0, 0, 521, 555, 463, 475, 536, 538, 498, 524, 7, 0, 0, 0, 0, 0, 0, 0, 494, 515, 490, 493, 519, 552, 520, 495, 13, 0, 0, 0, 0, 0, 0, 0, 487, 519, 514, 496, 520, 509, 535, 540, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4148, 4096, 4100, 4027, 4069, 4117, 4091, 4120] avg=4096.0000 max=4148 max_rank=0 min=4027 min_rank=3 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[485, 497, 515, 541, 491, 448, 532, 556, 6, 5, 0, 0, 0, 0, 0, 0, 521, 530, 531, 478, 495, 508, 513, 508, 16, 0, 0, 0, 0, 0, 0, 0, 562, 510, 472, 530, 497, 505, 530, 523, 0, 0, 0, 0, 0, 0, 0, 0, 488, 493, 519, 471, 512, 522, 511, 517, 23, 0, 0, 0, 0, 0, 0, 0, 532, 503, 509, 510, 515, 481, 537, 532, 0, 0, 0, 0, 0, 0, 0, 0, 501, 504, 534, 541, 499, 514, 515, 477, 4, 0, 0, 0, 0, 0, 0, 0, 462, 533, 493, 543, 500, 518, 525, 525, 9, 0, 0, 0, 0, 0, 0, 0, 486, 500, 491, 549, 492, 545, 524, 504, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4100, 4129, 4056, 4119, 4089, 4108, 4091] avg=4096.0000 max=4129 max_rank=2 min=4056 min_rank=3 imbalance_ratio=1.008057 max_load_reduction=28639 max_load_reduction_pct=0.873993 moved_to_echo=16407 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[495, 524, 474, 529, 545, 490, 517, 533, 6, 4, 0, 0, 0, 0, 0, 0, 479, 512, 518, 560, 478, 524, 524, 515, 17, 0, 0, 0, 0, 0, 0, 0, 502, 535, 553, 540, 507, 498, 486, 509, 0, 0, 0, 0, 0, 0, 0, 0, 508, 499, 488, 507, 509, 487, 510, 523, 24, 0, 0, 0, 0, 0, 0, 0, 505, 512, 494, 522, 528, 545, 500, 537, 0, 0, 0, 0, 0, 0, 0, 0, 505, 527, 499, 503, 505, 485, 521, 503, 4, 0, 0, 0, 0, 0, 0, 0, 507, 490, 479, 476, 515, 521, 529, 546, 8, 0, 0, 0, 0, 0, 0, 0, 500, 518, 491, 499, 510, 528, 532, 495, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4117, 4127, 4130, 4055, 4143, 4052, 4071, 4073] avg=4096.0000 max=4143 max_rank=4 min=4052 min_rank=5 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16339 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3968, 4096, 4096, 4096, 4224, 4352, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 3968, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[485, 524, 533, 531, 514, 513, 501, 507, 525, 535, 522, 493, 472, 489, 533, 549, 511, 504, 501, 517, 515, 520, 517, 505, 487, 497, 503, 541, 532, 502, 527, 459, 553, 511, 474, 545, 502, 519, 530, 504, 547, 513, 481, 485, 523, 527, 526, 495, 511, 531, 507, 506, 513, 528, 506, 538, 459, 529, 488, 562, 504, 511, 466, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[538, 508, 487, 547, 517, 516, 501, 538, 507, 526, 521, 521, 489, 488, 524, 527, 500, 492, 528, 457, 537, 473, 535, 508, 513, 535, 524, 512, 486, 512, 515, 498, 507, 505, 518, 495, 510, 494, 528, 534, 485, 530, 542, 511, 530, 516, 518, 500, 526, 493, 492, 526, 502, 518, 516, 534, 512, 510, 489, 486, 523, 524, 509, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[526, 532, 538, 489, 526, 505, 510, 519, 509, 527, 540, 492, 515, 491, 552, 497, 534, 464, 534, 494, 528, 497, 486, 535, 510, 510, 490, 515, 510, 514, 529, 545, 512, 487, 489, 476, 528, 530, 485, 498, 499, 458, 551, 479, 522, 503, 486, 510, 499, 512, 534, 517, 550, 519, 496, 506, 502, 532, 580, 500, 482, 484, 549, 530] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[485, 518, 515, 522, 514, 509, 477, 512, 521, 526, 506, 539, 508, 518, 524, 501, 545, 492, 519, 509, 515, 496, 470, 503, 467, 526, 479, 482, 562, 498, 519, 540, 516, 504, 502, 505, 516, 512, 506, 491, 531, 532, 531, 517, 566, 500, 485, 548, 545, 495, 496, 528, 494, 476, 512, 504, 525, 479, 522, 520, 498, 534, 518, 543] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[573, 489, 549, 481, 522, 526, 541, 532, 494, 506, 506, 482, 493, 505, 531, 482, 475, 508, 514, 481, 532, 511, 515, 541, 564, 529, 510, 504, 536, 480, 456, 522, 530, 514, 517, 525, 515, 492, 549, 504, 493, 488, 492, 511, 505, 496, 516, 482, 486, 498, 511, 538, 507, 498, 518, 540, 521, 475, 513, 536, 524, 523, 513, 548] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[508, 526, 497, 545, 499, 482, 492, 478, 518, 524, 496, 539, 515, 523, 490, 536, 531, 509, 548, 512, 476, 507, 507, 484, 532, 530, 540, 501, 535, 490, 487, 515, 536, 496, 480, 551, 515, 520, 555, 527, 511, 479, 514, 495, 504, 535, 481, 512, 503, 527, 483, 518, 499, 504, 532, 505, 474, 546, 498, 544, 528, 502, 507, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[511, 527, 515, 529, 514, 509, 499, 507, 496, 538, 511, 511, 532, 529, 510, 487, 493, 520, 515, 515, 483, 483, 516, 506, 485, 514, 513, 582, 440, 507, 528, 525, 531, 545, 435, 522, 508, 518, 539, 515, 531, 459, 512, 510, 525, 482, 491, 528, 521, 485, 521, 498, 507, 509, 548, 513, 469, 541, 549, 522, 518, 532, 530, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[514, 503, 469, 542, 501, 487, 529, 489, 546, 511, 507, 498, 514, 512, 503, 508, 496, 522, 497, 499, 501, 511, 504, 506, 475, 522, 525, 514, 538, 525, 542, 510, 532, 508, 515, 522, 516, 512, 470, 510, 513, 500, 502, 543, 525, 533, 519, 489, 498, 529, 525, 486, 516, 542, 507, 443, 532, 531, 529, 501, 538, 561, 498, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[573, 489, 549, 474, 522, 526, 541, 532, 0, 0, 0, 0, 0, 0, 0, 0, 494, 498, 506, 482, 493, 505, 531, 482, 0, 0, 0, 0, 0, 0, 0, 0, 475, 508, 514, 481, 532, 511, 515, 541, 30, 7, 0, 0, 0, 0, 0, 0, 564, 524, 510, 504, 536, 480, 456, 522, 0, 0, 0, 0, 0, 0, 0, 0, 526, 514, 517, 525, 515, 492, 549, 504, 0, 0, 0, 0, 0, 0, 0, 0, 493, 488, 492, 511, 505, 496, 516, 482, 8, 5, 2, 0, 0, 0, 0, 0, 486, 498, 511, 538, 507, 498, 518, 540, 2, 0, 0, 0, 0, 0, 0, 0, 521, 475, 513, 536, 524, 493, 513, 548, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4206, 3991, 4114, 4096, 4142, 3998, 4098, 4123] avg=4096.0000 max=4206 max_rank=0 min=3991 min_rank=1 imbalance_ratio=1.026855 max_load_reduction=28562 max_load_reduction_pct=0.871643 moved_to_echo=16361 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[526, 532, 538, 482, 526, 505, 510, 519, 0, 0, 0, 0, 0, 0, 0, 0, 509, 518, 540, 492, 515, 491, 552, 497, 0, 0, 0, 0, 0, 0, 0, 0, 534, 464, 534, 494, 528, 497, 486, 535, 28, 7, 0, 0, 0, 0, 0, 0, 510, 505, 490, 515, 510, 514, 529, 545, 0, 0, 0, 0, 0, 0, 0, 0, 508, 487, 489, 476, 528, 530, 485, 498, 0, 0, 0, 0, 0, 0, 0, 0, 499, 458, 551, 479, 522, 503, 486, 510, 9, 5, 2, 0, 0, 0, 0, 0, 499, 512, 534, 517, 550, 519, 496, 506, 2, 0, 0, 0, 0, 0, 0, 0, 502, 532, 580, 500, 482, 456, 549, 530, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4138, 4114, 4107, 4118, 4001, 4024, 4135, 4131] avg=4096.0000 max=4138 max_rank=0 min=4001 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16291 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[511, 527, 515, 521, 514, 509, 499, 507, 0, 0, 0, 0, 0, 0, 0, 0, 496, 529, 511, 511, 532, 529, 510, 487, 0, 0, 0, 0, 0, 0, 0, 0, 493, 520, 515, 515, 483, 483, 516, 506, 30, 8, 0, 0, 0, 0, 0, 0, 485, 509, 513, 582, 440, 507, 528, 525, 0, 0, 0, 0, 0, 0, 0, 0, 527, 545, 435, 522, 508, 518, 539, 515, 0, 0, 0, 0, 0, 0, 0, 0, 531, 459, 512, 510, 525, 482, 491, 528, 9, 5, 2, 0, 0, 0, 0, 0, 521, 485, 521, 498, 507, 509, 548, 513, 2, 0, 0, 0, 0, 0, 0, 0, 469, 541, 549, 522, 518, 502, 530, 504, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4105, 4069, 4089, 4109, 4054, 4104, 4135] avg=4096.0000 max=4135 max_rank=7 min=4054 min_rank=5 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16402 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[508, 526, 497, 537, 499, 482, 492, 478, 0, 0, 0, 0, 0, 0, 0, 0, 518, 516, 496, 539, 515, 523, 490, 536, 0, 0, 0, 0, 0, 0, 0, 0, 531, 509, 548, 512, 476, 507, 507, 484, 29, 8, 0, 0, 0, 0, 0, 0, 532, 525, 540, 501, 535, 490, 487, 515, 0, 0, 0, 0, 0, 0, 0, 0, 532, 496, 480, 551, 515, 520, 555, 527, 0, 0, 0, 0, 0, 0, 0, 0, 511, 479, 514, 495, 504, 535, 481, 512, 8, 5, 2, 0, 0, 0, 0, 0, 503, 527, 483, 518, 499, 504, 532, 505, 2, 0, 0, 0, 0, 0, 0, 0, 474, 546, 498, 544, 528, 473, 507, 515, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4019, 4133, 4111, 4125, 4176, 4046, 4073, 4085] avg=4096.0000 max=4176 max_rank=4 min=4019 min_rank=0 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[514, 503, 469, 534, 501, 487, 529, 489, 0, 0, 0, 0, 0, 0, 0, 0, 546, 503, 507, 498, 514, 512, 503, 508, 0, 0, 0, 0, 0, 0, 0, 0, 496, 522, 497, 499, 501, 511, 504, 506, 32, 8, 0, 0, 0, 0, 0, 0, 475, 517, 525, 514, 538, 525, 542, 510, 0, 0, 0, 0, 0, 0, 0, 0, 528, 508, 515, 522, 516, 512, 470, 510, 0, 0, 0, 0, 0, 0, 0, 0, 513, 500, 502, 543, 525, 533, 519, 489, 8, 5, 2, 0, 0, 0, 0, 0, 498, 529, 525, 486, 516, 542, 507, 443, 2, 0, 0, 0, 0, 0, 0, 0, 532, 531, 529, 501, 538, 529, 498, 503, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4026, 4091, 4076, 4146, 4081, 4139, 4048, 4161] avg=4096.0000 max=4161 max_rank=7 min=4026 min_rank=0 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[538, 508, 487, 539, 517, 516, 501, 538, 0, 0, 0, 0, 0, 0, 0, 0, 507, 517, 521, 521, 489, 488, 524, 527, 0, 0, 0, 0, 0, 0, 0, 0, 500, 492, 528, 457, 537, 473, 535, 508, 30, 8, 0, 0, 0, 0, 0, 0, 513, 529, 524, 512, 486, 512, 515, 498, 0, 0, 0, 0, 0, 0, 0, 0, 503, 505, 518, 495, 510, 494, 528, 534, 0, 0, 0, 0, 0, 0, 0, 0, 485, 530, 542, 511, 530, 516, 518, 500, 9, 6, 2, 0, 0, 0, 0, 0, 526, 493, 492, 526, 502, 518, 516, 534, 2, 0, 0, 0, 0, 0, 0, 0, 512, 510, 489, 486, 523, 494, 509, 505, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4144, 4094, 4068, 4089, 4087, 4149, 4109, 4028] avg=4096.0000 max=4149 max_rank=5 min=4028 min_rank=7 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[485, 518, 515, 502, 514, 509, 477, 512, 0, 0, 0, 0, 0, 0, 0, 0, 521, 514, 506, 539, 508, 518, 524, 501, 0, 0, 0, 0, 0, 0, 0, 0, 545, 492, 519, 509, 515, 496, 470, 503, 34, 13, 0, 0, 0, 0, 0, 0, 467, 515, 479, 482, 562, 498, 519, 540, 0, 0, 0, 0, 0, 0, 0, 0, 503, 504, 502, 505, 516, 512, 506, 491, 0, 0, 0, 0, 0, 0, 0, 0, 531, 532, 531, 517, 566, 500, 485, 548, 12, 11, 5, 7, 0, 0, 0, 0, 545, 495, 496, 528, 494, 476, 512, 504, 8, 0, 0, 0, 0, 0, 0, 0, 525, 479, 522, 520, 498, 500, 518, 543, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4032, 4131, 4096, 4062, 4039, 4245, 4058, 4105] avg=4096.0000 max=4245 max_rank=5 min=4032 min_rank=0 imbalance_ratio=1.036377 max_load_reduction=28523 max_load_reduction_pct=0.870453 moved_to_echo=16447 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[485, 524, 533, 523, 514, 513, 501, 507, 0, 0, 0, 0, 0, 0, 0, 0, 525, 526, 522, 493, 472, 489, 533, 549, 0, 0, 0, 0, 0, 0, 0, 0, 511, 504, 501, 517, 515, 520, 517, 505, 29, 8, 0, 0, 0, 0, 0, 0, 487, 492, 503, 541, 532, 502, 527, 459, 0, 0, 0, 0, 0, 0, 0, 0, 548, 511, 474, 545, 502, 519, 530, 504, 0, 0, 0, 0, 0, 0, 0, 0, 547, 513, 481, 485, 523, 527, 526, 495, 9, 5, 2, 0, 0, 0, 0, 0, 511, 531, 507, 506, 513, 528, 506, 538, 3, 0, 0, 0, 0, 0, 0, 0, 459, 529, 488, 562, 504, 482, 466, 510, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4109, 4127, 4043, 4133, 4113, 4143, 4000] avg=4096.0000 max=4143 max_rank=6 min=4000 min_rank=7 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 3968, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 3968, 4224, 4096, 4224, 4096, 4096, 4096, 128, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 3968, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[507, 534, 512, 509, 483, 496, 505, 482, 543, 522, 491, 504, 488, 539, 487, 528, 485, 507, 491, 537, 492, 528, 507, 546, 541, 484, 516, 543, 486, 499, 523, 490, 521, 525, 525, 519, 517, 509, 518, 532, 495, 549, 524, 470, 489, 482, 538, 494, 528, 554, 517, 516, 500, 528, 527, 503, 520, 484, 508, 499, 482, 517, 531, 542] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[493, 482, 517, 461, 466, 501, 513, 503, 522, 490, 509, 505, 509, 536, 528, 513, 533, 504, 552, 492, 522, 496, 517, 521, 492, 504, 516, 502, 529, 511, 488, 515, 501, 513, 513, 521, 482, 530, 493, 504, 515, 509, 551, 508, 515, 540, 499, 544, 550, 501, 550, 513, 513, 495, 511, 516, 513, 526, 504, 558, 522, 529, 495, 492] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[523, 507, 560, 477, 526, 493, 522, 521, 540, 523, 538, 494, 497, 544, 532, 500, 511, 493, 503, 492, 522, 526, 482, 486, 510, 497, 512, 500, 517, 492, 480, 509, 544, 482, 488, 517, 537, 527, 552, 512, 509, 508, 473, 499, 519, 511, 541, 546, 512, 523, 515, 493, 524, 494, 545, 540, 480, 515, 494, 516, 506, 496, 498, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[471, 487, 529, 526, 495, 530, 513, 519, 527, 512, 515, 530, 510, 509, 477, 480, 525, 535, 519, 502, 509, 524, 478, 494, 523, 510, 487, 511, 518, 498, 530, 542, 480, 531, 505, 515, 512, 519, 487, 513, 515, 505, 511, 502, 544, 489, 498, 557, 515, 489, 521, 531, 524, 522, 512, 535, 527, 502, 507, 511, 528, 505, 505, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[508, 530, 529, 533, 489, 482, 517, 508, 530, 507, 498, 520, 522, 507, 556, 500, 497, 508, 514, 535, 520, 523, 506, 489, 543, 505, 521, 495, 512, 513, 504, 518, 513, 483, 518, 478, 501, 514, 508, 510, 513, 486, 500, 518, 490, 495, 522, 550, 506, 523, 541, 487, 493, 499, 530, 540, 526, 506, 506, 541, 493, 505, 524, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[523, 512, 525, 536, 508, 503, 519, 502, 480, 506, 475, 478, 489, 542, 524, 516, 513, 502, 484, 500, 537, 524, 531, 523, 493, 535, 530, 536, 508, 521, 482, 496, 542, 498, 492, 486, 491, 507, 505, 490, 468, 497, 538, 516, 491, 553, 516, 511, 523, 514, 473, 496, 528, 512, 524, 526, 522, 541, 514, 528, 529, 540, 533, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[509, 520, 499, 543, 503, 525, 500, 499, 536, 496, 549, 517, 508, 503, 488, 487, 486, 519, 536, 516, 476, 515, 530, 481, 489, 536, 487, 497, 528, 506, 537, 478, 506, 536, 500, 535, 523, 493, 499, 532, 513, 544, 511, 529, 505, 523, 478, 525, 522, 486, 539, 493, 489, 521, 541, 491, 522, 513, 500, 518, 515, 507, 533, 527] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[487, 483, 488, 513, 508, 531, 563, 480, 528, 518, 507, 534, 500, 485, 501, 512, 519, 496, 500, 522, 505, 531, 546, 502, 502, 489, 517, 508, 561, 495, 533, 512, 478, 532, 498, 488, 509, 494, 526, 547, 554, 484, 541, 512, 523, 502, 495, 555, 540, 539, 500, 535, 505, 520, 528, 513, 502, 520, 492, 516, 498, 516, 445, 485] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[507, 534, 512, 509, 483, 496, 505, 482, 28, 0, 0, 0, 0, 0, 0, 0, 543, 522, 491, 504, 488, 539, 487, 528, 0, 0, 0, 0, 0, 0, 0, 0, 485, 507, 491, 537, 492, 528, 507, 546, 4, 1, 0, 0, 0, 0, 0, 0, 541, 484, 516, 543, 486, 499, 523, 490, 11, 1, 0, 0, 0, 0, 0, 0, 521, 525, 525, 519, 517, 509, 518, 532, 10, 0, 0, 0, 0, 0, 0, 0, 495, 549, 524, 470, 489, 482, 538, 479, 0, 0, 0, 0, 0, 0, 0, 0, 528, 554, 517, 516, 500, 528, 488, 503, 0, 0, 0, 0, 0, 0, 0, 0, 520, 484, 508, 498, 482, 517, 531, 542, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4056, 4102, 4098, 4094, 4176, 4026, 4134, 4082] avg=4096.0000 max=4176 max_rank=4 min=4026 min_rank=5 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16418 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[493, 482, 517, 461, 466, 501, 513, 503, 27, 0, 0, 0, 0, 0, 0, 0, 522, 490, 509, 505, 509, 536, 528, 513, 0, 0, 0, 0, 0, 0, 0, 0, 533, 504, 552, 492, 522, 496, 517, 521, 4, 1, 0, 0, 0, 0, 0, 0, 492, 504, 516, 502, 529, 511, 488, 515, 11, 1, 0, 0, 0, 0, 0, 0, 501, 513, 513, 521, 482, 530, 493, 504, 11, 0, 0, 0, 0, 0, 0, 0, 515, 509, 551, 508, 515, 540, 499, 528, 0, 0, 0, 0, 0, 0, 0, 0, 550, 501, 550, 513, 513, 495, 473, 516, 0, 0, 0, 0, 0, 0, 0, 0, 513, 526, 504, 557, 522, 529, 495, 492, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3963, 4112, 4142, 4069, 4068, 4165, 4111, 4138] avg=4096.0000 max=4165 max_rank=5 min=3963 min_rank=0 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16482 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[471, 487, 529, 526, 495, 530, 513, 519, 27, 0, 0, 0, 0, 0, 0, 0, 527, 512, 515, 530, 510, 509, 477, 480, 0, 0, 0, 0, 0, 0, 0, 0, 525, 535, 519, 502, 509, 524, 478, 494, 4, 1, 0, 0, 0, 0, 0, 0, 523, 510, 487, 511, 518, 498, 530, 542, 11, 1, 0, 0, 0, 0, 0, 0, 480, 531, 505, 515, 512, 519, 487, 513, 11, 0, 0, 0, 0, 0, 0, 0, 515, 505, 511, 502, 544, 489, 498, 541, 0, 0, 0, 0, 0, 0, 0, 0, 515, 489, 521, 531, 524, 522, 474, 535, 0, 0, 0, 0, 0, 0, 0, 0, 527, 502, 507, 510, 528, 505, 505, 516, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4060, 4091, 4131, 4073, 4105, 4111, 4100] avg=4096.0000 max=4131 max_rank=3 min=4060 min_rank=1 imbalance_ratio=1.008545 max_load_reduction=28637 max_load_reduction_pct=0.873932 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[523, 507, 560, 477, 526, 493, 522, 521, 29, 0, 0, 0, 0, 0, 0, 0, 540, 523, 538, 494, 497, 544, 532, 500, 0, 0, 0, 0, 0, 0, 0, 0, 511, 493, 503, 492, 522, 526, 482, 486, 4, 1, 0, 0, 0, 0, 0, 0, 510, 497, 512, 500, 517, 492, 480, 509, 12, 1, 0, 0, 0, 0, 0, 0, 544, 482, 488, 517, 537, 527, 552, 512, 11, 0, 0, 0, 0, 0, 0, 0, 509, 508, 473, 499, 519, 511, 541, 530, 0, 0, 0, 0, 0, 0, 0, 0, 512, 523, 515, 493, 524, 494, 504, 540, 0, 0, 0, 0, 0, 0, 0, 0, 480, 515, 494, 515, 506, 496, 498, 523, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4158, 4168, 4020, 4030, 4170, 4090, 4105, 4027] avg=4096.0000 max=4170 max_rank=4 min=4020 min_rank=2 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[508, 530, 529, 533, 489, 482, 517, 508, 34, 0, 0, 0, 0, 0, 0, 0, 530, 507, 498, 520, 522, 507, 556, 500, 7, 0, 0, 0, 0, 0, 0, 0, 497, 508, 514, 535, 520, 523, 506, 489, 9, 7, 0, 0, 0, 0, 0, 0, 543, 505, 521, 495, 512, 513, 504, 518, 16, 3, 0, 0, 0, 0, 0, 0, 513, 483, 518, 478, 501, 514, 508, 510, 17, 0, 0, 0, 0, 0, 0, 0, 513, 486, 500, 518, 490, 495, 522, 521, 0, 0, 0, 0, 0, 0, 0, 0, 506, 523, 541, 487, 493, 499, 480, 540, 0, 0, 0, 0, 0, 0, 0, 0, 526, 506, 506, 527, 493, 505, 524, 510, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4130, 4147, 4108, 4130, 4042, 4045, 4069, 4097] avg=4096.0000 max=4147 max_rank=1 min=4042 min_rank=4 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16253 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[523, 512, 525, 536, 508, 503, 519, 502, 28, 0, 0, 0, 0, 0, 0, 0, 480, 506, 475, 478, 489, 542, 524, 516, 0, 0, 0, 0, 0, 0, 0, 0, 513, 502, 484, 500, 537, 524, 531, 523, 4, 1, 0, 0, 0, 0, 0, 0, 493, 535, 530, 536, 508, 521, 482, 496, 11, 1, 0, 0, 0, 0, 0, 0, 542, 498, 492, 486, 491, 507, 505, 490, 10, 0, 0, 0, 0, 0, 0, 0, 468, 497, 538, 516, 491, 553, 516, 496, 0, 0, 0, 0, 0, 0, 0, 0, 523, 514, 473, 496, 528, 512, 485, 526, 0, 0, 0, 0, 0, 0, 0, 0, 522, 541, 514, 527, 529, 540, 533, 511, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4156, 4010, 4119, 4113, 4021, 4075, 4057, 4217] avg=4096.0000 max=4217 max_rank=7 min=4010 min_rank=1 imbalance_ratio=1.029541 max_load_reduction=28551 max_load_reduction_pct=0.871307 moved_to_echo=16370 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[487, 483, 488, 513, 508, 531, 563, 480, 28, 0, 0, 0, 0, 0, 0, 0, 528, 518, 507, 534, 500, 485, 501, 512, 0, 0, 0, 0, 0, 0, 0, 0, 519, 496, 500, 522, 505, 531, 546, 502, 4, 1, 0, 0, 0, 0, 0, 0, 502, 489, 517, 508, 561, 495, 533, 512, 12, 1, 0, 0, 0, 0, 0, 0, 478, 532, 498, 488, 509, 494, 526, 547, 11, 0, 0, 0, 0, 0, 0, 0, 554, 484, 541, 512, 523, 502, 495, 539, 0, 0, 0, 0, 0, 0, 0, 0, 540, 539, 500, 535, 505, 520, 488, 513, 0, 0, 0, 0, 0, 0, 0, 0, 502, 520, 492, 515, 498, 516, 445, 485, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4085, 4126, 4130, 4083, 4150, 4140, 3973] avg=4096.0000 max=4150 max_rank=5 min=3973 min_rank=7 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16346 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[509, 520, 499, 543, 503, 525, 500, 499, 29, 0, 0, 0, 0, 0, 0, 0, 536, 496, 549, 517, 508, 503, 488, 487, 0, 0, 0, 0, 0, 0, 0, 0, 486, 519, 536, 516, 476, 515, 530, 481, 4, 1, 0, 0, 0, 0, 0, 0, 489, 536, 487, 497, 528, 506, 537, 478, 12, 1, 0, 0, 0, 0, 0, 0, 506, 536, 500, 535, 523, 493, 499, 532, 11, 0, 0, 0, 0, 0, 0, 0, 513, 544, 511, 529, 505, 523, 478, 509, 0, 0, 0, 0, 0, 0, 0, 0, 522, 486, 539, 493, 489, 521, 500, 491, 0, 0, 0, 0, 0, 0, 0, 0, 522, 513, 500, 517, 515, 507, 533, 527, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4127, 4084, 4064, 4071, 4135, 4112, 4041, 4134] avg=4096.0000 max=4135 max_rank=4 min=4041 min_rank=6 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16422 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[527, 531, 509, 502, 518, 535, 486, 504, 496, 540, 498, 481, 518, 514, 494, 520, 521, 558, 486, 500, 497, 511, 506, 546, 523, 481, 495, 520, 517, 505, 515, 506, 490, 531, 522, 537, 495, 471, 510, 515, 552, 537, 479, 500, 526, 526, 535, 539, 479, 536, 498, 530, 511, 500, 509, 527, 483, 522, 527, 527, 502, 492, 507, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[513, 477, 516, 533, 456, 526, 531, 473, 513, 505, 517, 501, 507, 534, 530, 532, 462, 533, 525, 515, 486, 499, 511, 532, 482, 535, 497, 526, 507, 506, 518, 501, 492, 481, 510, 510, 511, 534, 503, 480, 507, 524, 526, 506, 466, 517, 490, 531, 525, 509, 510, 522, 549, 548, 524, 500, 533, 544, 532, 488, 510, 518, 538, 531] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[475, 512, 503, 482, 526, 526, 526, 518, 545, 517, 497, 508, 515, 488, 519, 552, 520, 513, 498, 502, 513, 547, 528, 476, 496, 519, 538, 530, 502, 504, 533, 506, 464, 542, 516, 558, 544, 488, 509, 499, 497, 533, 555, 517, 514, 526, 521, 513, 492, 525, 459, 526, 520, 481, 496, 484, 539, 501, 501, 518, 495, 489, 522, 490] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[491, 508, 534, 537, 520, 526, 526, 520, 485, 500, 514, 537, 475, 521, 515, 528, 522, 499, 504, 488, 519, 505, 550, 491, 489, 523, 544, 523, 499, 537, 531, 511, 523, 510, 505, 522, 528, 524, 505, 504, 539, 496, 489, 470, 539, 505, 518, 531, 523, 516, 506, 523, 521, 498, 486, 495, 514, 515, 517, 513, 472, 469, 496, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[514, 501, 486, 533, 522, 509, 490, 468, 508, 516, 456, 469, 531, 528, 546, 504, 474, 468, 540, 510, 532, 533, 500, 546, 526, 512, 505, 544, 532, 510, 512, 551, 505, 535, 509, 509, 535, 508, 515, 508, 536, 500, 510, 450, 516, 534, 521, 499, 509, 486, 509, 471, 480, 473, 546, 509, 531, 528, 519, 513, 570, 518, 528, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[520, 496, 537, 500, 517, 529, 488, 543, 492, 498, 496, 522, 474, 515, 538, 543, 534, 478, 476, 522, 501, 517, 502, 546, 525, 475, 556, 526, 531, 534, 514, 496, 517, 522, 499, 481, 502, 504, 508, 486, 531, 477, 543, 520, 519, 552, 526, 520, 497, 527, 503, 501, 520, 486, 547, 509, 460, 531, 509, 502, 478, 487, 510, 553] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[471, 510, 526, 512, 517, 510, 492, 534, 537, 507, 501, 505, 546, 559, 509, 505, 504, 483, 505, 517, 493, 496, 500, 508, 495, 520, 497, 500, 543, 495, 511, 511, 517, 488, 513, 536, 483, 516, 496, 478, 469, 528, 505, 531, 569, 527, 552, 499, 512, 528, 504, 527, 529, 491, 548, 526, 526, 512, 522, 523, 518, 480, 475, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[546, 500, 511, 504, 496, 497, 528, 509, 506, 537, 518, 513, 539, 524, 498, 501, 543, 492, 544, 537, 531, 500, 502, 508, 503, 535, 503, 517, 499, 500, 519, 547, 539, 506, 507, 499, 519, 535, 520, 502, 536, 497, 516, 528, 511, 466, 554, 504, 469, 487, 502, 499, 503, 515, 456, 490, 522, 540, 514, 501, 495, 514, 511, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[527, 531, 509, 502, 518, 535, 486, 504, 9, 0, 0, 0, 0, 0, 0, 0, 496, 540, 498, 481, 518, 514, 494, 511, 0, 0, 0, 0, 0, 0, 0, 0, 521, 558, 486, 500, 497, 511, 506, 546, 7, 0, 0, 0, 0, 0, 0, 0, 523, 481, 495, 497, 517, 505, 515, 506, 0, 0, 0, 0, 0, 0, 0, 0, 490, 531, 522, 537, 495, 471, 510, 515, 14, 2, 0, 0, 0, 0, 0, 0, 552, 537, 479, 500, 526, 526, 498, 539, 0, 0, 0, 0, 0, 0, 0, 0, 479, 536, 498, 530, 511, 500, 509, 527, 35, 0, 0, 0, 0, 0, 0, 0, 483, 522, 527, 527, 502, 492, 507, 493, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4052, 4132, 4039, 4087, 4157, 4125, 4055] avg=4096.0000 max=4157 max_rank=5 min=4039 min_rank=3 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16424 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[513, 477, 516, 533, 456, 526, 531, 473, 9, 0, 0, 0, 0, 0, 0, 0, 513, 505, 517, 501, 507, 534, 530, 522, 0, 0, 0, 0, 0, 0, 0, 0, 462, 533, 525, 515, 486, 499, 511, 532, 8, 0, 0, 0, 0, 0, 0, 0, 482, 535, 497, 503, 507, 506, 518, 501, 0, 0, 0, 0, 0, 0, 0, 0, 492, 481, 510, 510, 511, 534, 503, 480, 14, 2, 0, 0, 0, 0, 0, 0, 507, 524, 526, 506, 466, 517, 456, 531, 0, 0, 0, 0, 0, 0, 0, 0, 525, 509, 510, 522, 549, 548, 524, 500, 32, 0, 0, 0, 0, 0, 0, 0, 533, 544, 532, 488, 510, 518, 538, 531, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4034, 4129, 4071, 4049, 4037, 4033, 4219, 4196] avg=4096.0000 max=4219 max_rank=6 min=4033 min_rank=5 imbalance_ratio=1.030029 max_load_reduction=28549 max_load_reduction_pct=0.871246 moved_to_echo=16485 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[491, 508, 534, 537, 520, 526, 526, 520, 9, 0, 0, 0, 0, 0, 0, 0, 485, 500, 514, 537, 475, 521, 515, 519, 0, 0, 0, 0, 0, 0, 0, 0, 522, 499, 504, 488, 519, 505, 550, 491, 7, 0, 0, 0, 0, 0, 0, 0, 489, 523, 544, 500, 499, 537, 531, 511, 0, 0, 0, 0, 0, 0, 0, 0, 523, 510, 505, 522, 528, 524, 505, 504, 14, 2, 0, 0, 0, 0, 0, 0, 539, 496, 489, 470, 539, 505, 482, 531, 0, 0, 0, 0, 0, 0, 0, 0, 523, 516, 506, 523, 521, 498, 486, 495, 34, 0, 0, 0, 0, 0, 0, 0, 514, 515, 517, 513, 472, 469, 496, 524, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4171, 4066, 4085, 4134, 4137, 4051, 4102, 4022] avg=4096.0000 max=4171 max_rank=0 min=4022 min_rank=7 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16312 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[475, 512, 503, 482, 526, 526, 526, 518, 9, 0, 0, 0, 0, 0, 0, 0, 545, 517, 497, 508, 515, 488, 519, 542, 0, 0, 0, 0, 0, 0, 0, 0, 520, 513, 498, 502, 513, 547, 528, 476, 8, 0, 0, 0, 0, 0, 0, 0, 496, 519, 538, 507, 502, 504, 533, 506, 0, 0, 0, 0, 0, 0, 0, 0, 464, 542, 516, 558, 544, 488, 509, 499, 14, 2, 0, 0, 0, 0, 0, 0, 497, 533, 555, 517, 514, 526, 485, 513, 0, 0, 0, 0, 0, 0, 0, 0, 492, 525, 459, 526, 520, 481, 496, 484, 34, 0, 0, 0, 0, 0, 0, 0, 539, 501, 501, 518, 495, 489, 522, 490, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4131, 4105, 4105, 4136, 4140, 4017, 4057] avg=4096.0000 max=4140 max_rank=5 min=4017 min_rank=6 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16350 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[514, 501, 486, 533, 522, 509, 490, 468, 10, 0, 0, 0, 0, 0, 0, 0, 508, 516, 456, 469, 531, 528, 546, 495, 0, 0, 0, 0, 0, 0, 0, 0, 474, 468, 540, 510, 532, 533, 500, 546, 7, 0, 0, 0, 0, 0, 0, 0, 526, 512, 505, 519, 532, 510, 512, 551, 0, 0, 0, 0, 0, 0, 0, 0, 505, 535, 509, 509, 535, 508, 515, 508, 15, 2, 0, 0, 0, 0, 0, 0, 536, 500, 510, 450, 516, 534, 485, 499, 0, 0, 0, 0, 0, 0, 0, 0, 509, 486, 509, 471, 480, 473, 546, 509, 34, 0, 0, 0, 0, 0, 0, 0, 531, 528, 519, 513, 570, 518, 528, 512, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4033, 4049, 4110, 4167, 4141, 4030, 4017, 4221] avg=4096.0000 max=4221 max_rank=7 min=4017 min_rank=6 imbalance_ratio=1.030518 max_load_reduction=28547 max_load_reduction_pct=0.871185 moved_to_echo=16409 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[520, 496, 537, 500, 517, 529, 488, 543, 14, 7, 0, 0, 0, 0, 0, 0, 492, 498, 496, 522, 474, 515, 538, 519, 0, 0, 0, 0, 0, 0, 0, 0, 534, 478, 476, 522, 501, 517, 502, 546, 12, 0, 0, 0, 0, 0, 0, 0, 525, 475, 556, 493, 531, 534, 514, 496, 0, 0, 0, 0, 0, 0, 0, 0, 517, 522, 499, 481, 502, 504, 508, 486, 19, 7, 0, 0, 0, 0, 0, 0, 531, 477, 543, 520, 519, 552, 479, 520, 0, 0, 0, 0, 0, 0, 0, 0, 497, 527, 503, 501, 520, 486, 547, 509, 40, 0, 0, 0, 0, 0, 0, 0, 460, 531, 509, 502, 478, 487, 510, 553, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4151, 4054, 4088, 4124, 4045, 4141, 4130, 4035] avg=4096.0000 max=4151 max_rank=0 min=4035 min_rank=7 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16351 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[471, 510, 526, 512, 517, 510, 492, 534, 9, 0, 0, 0, 0, 0, 0, 0, 537, 507, 501, 505, 546, 559, 509, 496, 0, 0, 0, 0, 0, 0, 0, 0, 504, 483, 505, 517, 493, 496, 500, 508, 7, 0, 0, 0, 0, 0, 0, 0, 495, 520, 497, 478, 543, 495, 511, 511, 0, 0, 0, 0, 0, 0, 0, 0, 517, 488, 513, 536, 483, 516, 496, 478, 13, 2, 0, 0, 0, 0, 0, 0, 469, 528, 505, 531, 569, 527, 514, 499, 0, 0, 0, 0, 0, 0, 0, 0, 512, 528, 504, 527, 529, 491, 548, 526, 36, 0, 0, 0, 0, 0, 0, 0, 526, 512, 522, 523, 518, 480, 475, 521, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4160, 4013, 4050, 4042, 4142, 4201, 4079] avg=4096.0000 max=4201 max_rank=6 min=4013 min_rank=2 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16464 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[546, 500, 511, 504, 496, 497, 528, 509, 9, 0, 0, 0, 0, 0, 0, 0, 506, 537, 518, 513, 539, 524, 498, 492, 0, 0, 0, 0, 0, 0, 0, 0, 543, 492, 544, 537, 531, 500, 502, 508, 7, 0, 0, 0, 0, 0, 0, 0, 503, 535, 503, 494, 499, 500, 519, 547, 0, 0, 0, 0, 0, 0, 0, 0, 539, 506, 507, 499, 519, 535, 520, 502, 14, 2, 0, 0, 0, 0, 0, 0, 536, 497, 516, 528, 511, 466, 516, 504, 0, 0, 0, 0, 0, 0, 0, 0, 469, 487, 502, 499, 503, 515, 456, 490, 36, 0, 0, 0, 0, 0, 0, 0, 522, 540, 514, 501, 495, 514, 511, 504, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4127, 4164, 4100, 4143, 4074, 3957, 4103] avg=4096.0000 max=4164 max_rank=2 min=3957 min_rank=6 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16277 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 3968, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[484, 485, 500, 522, 499, 505, 491, 512, 530, 459, 513, 486, 469, 500, 535, 501, 516, 510, 539, 511, 542, 483, 504, 533, 535, 547, 510, 534, 486, 541, 522, 522, 473, 546, 511, 542, 543, 493, 514, 515, 492, 522, 519, 485, 514, 524, 495, 480, 556, 495, 508, 499, 508, 521, 513, 531, 531, 490, 509, 512, 519, 546, 523, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[483, 522, 513, 498, 538, 502, 544, 459, 531, 496, 540, 498, 522, 516, 520, 507, 504, 488, 549, 510, 520, 531, 500, 455, 496, 524, 518, 499, 492, 514, 522, 518, 496, 529, 517, 541, 497, 488, 476, 521, 539, 541, 530, 489, 505, 520, 497, 520, 472, 504, 509, 532, 538, 543, 491, 542, 536, 511, 496, 500, 525, 513, 536, 485] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[507, 521, 503, 512, 508, 533, 511, 509, 546, 507, 534, 524, 522, 488, 515, 512, 544, 504, 518, 508, 518, 541, 510, 502, 508, 508, 499, 513, 487, 493, 527, 505, 480, 483, 531, 517, 517, 501, 555, 482, 496, 507, 545, 531, 511, 499, 518, 507, 528, 489, 519, 442, 530, 506, 536, 515, 509, 509, 527, 497, 503, 536, 493, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[544, 517, 501, 510, 508, 490, 544, 519, 500, 556, 529, 498, 509, 513, 504, 487, 526, 504, 499, 514, 486, 492, 510, 539, 498, 497, 522, 505, 529, 520, 525, 542, 479, 488, 481, 509, 499, 523, 576, 500, 499, 503, 519, 470, 515, 501, 547, 496, 506, 532, 562, 509, 560, 487, 506, 512, 513, 513, 505, 491, 500, 533, 514, 483] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[484, 522, 496, 488, 516, 525, 503, 552, 500, 515, 499, 521, 483, 505, 511, 522, 496, 501, 542, 526, 539, 528, 503, 498, 502, 494, 476, 501, 523, 516, 516, 527, 485, 483, 510, 509, 538, 472, 511, 521, 561, 513, 513, 523, 459, 532, 505, 503, 529, 541, 542, 534, 529, 506, 534, 505, 519, 539, 461, 494, 496, 504, 521, 546] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[513, 513, 528, 498, 525, 536, 531, 512, 525, 544, 502, 545, 483, 535, 515, 493, 500, 512, 495, 531, 522, 479, 504, 504, 539, 510, 507, 483, 515, 483, 502, 508, 498, 511, 526, 518, 527, 545, 469, 524, 507, 534, 501, 511, 496, 512, 523, 489, 542, 512, 519, 542, 502, 488, 508, 543, 486, 524, 517, 508, 516, 496, 511, 471] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[498, 536, 491, 514, 495, 493, 490, 531, 506, 489, 502, 525, 519, 518, 547, 507, 542, 522, 502, 541, 501, 508, 503, 528, 495, 453, 500, 509, 516, 528, 523, 502, 504, 521, 533, 548, 499, 518, 502, 516, 528, 527, 529, 515, 541, 528, 506, 517, 492, 514, 514, 468, 533, 558, 491, 511, 507, 508, 508, 481, 523, 483, 498, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[540, 474, 482, 518, 492, 510, 537, 521, 545, 504, 509, 505, 532, 518, 511, 488, 531, 510, 493, 520, 519, 523, 493, 503, 524, 519, 511, 541, 511, 530, 470, 561, 501, 533, 502, 502, 469, 540, 526, 533, 492, 511, 519, 501, 504, 508, 532, 514, 509, 513, 500, 481, 528, 487, 507, 518, 499, 484, 531, 535, 486, 514, 532, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[484, 485, 500, 522, 499, 505, 491, 512, 8, 4, 0, 0, 0, 0, 0, 0, 528, 459, 513, 486, 469, 500, 535, 501, 0, 0, 0, 0, 0, 0, 0, 0, 516, 510, 539, 505, 542, 483, 504, 533, 0, 0, 0, 0, 0, 0, 0, 0, 535, 547, 510, 534, 486, 541, 522, 522, 1, 0, 0, 0, 0, 0, 0, 0, 473, 546, 511, 542, 543, 493, 514, 515, 2, 2, 0, 0, 0, 0, 0, 0, 492, 522, 518, 485, 514, 524, 495, 480, 0, 0, 0, 0, 0, 0, 0, 0, 556, 495, 508, 499, 481, 521, 513, 531, 0, 0, 0, 0, 0, 0, 0, 0, 531, 490, 509, 512, 519, 546, 523, 513, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4010, 3991, 4132, 4198, 4141, 4030, 4104, 4162] avg=4096.0000 max=4198 max_rank=3 min=3991 min_rank=1 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16437 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[507, 521, 503, 512, 508, 533, 511, 509, 8, 4, 0, 0, 0, 0, 0, 0, 544, 507, 534, 524, 522, 488, 515, 512, 0, 0, 0, 0, 0, 0, 0, 0, 544, 504, 518, 502, 518, 541, 510, 502, 0, 0, 0, 0, 0, 0, 0, 0, 508, 508, 499, 513, 487, 493, 527, 505, 1, 0, 0, 0, 0, 0, 0, 0, 480, 483, 531, 517, 517, 501, 555, 482, 2, 2, 0, 0, 0, 0, 0, 0, 496, 507, 544, 531, 511, 499, 518, 507, 0, 0, 0, 0, 0, 0, 0, 0, 528, 489, 519, 442, 502, 506, 536, 515, 0, 0, 0, 0, 0, 0, 0, 0, 509, 509, 527, 497, 503, 536, 493, 512, 20, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4146, 4139, 4041, 4070, 4113, 4037, 4106] avg=4096.0000 max=4146 max_rank=1 min=4037 min_rank=6 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16326 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[544, 517, 501, 510, 508, 490, 544, 519, 9, 4, 0, 0, 0, 0, 0, 0, 498, 556, 529, 498, 509, 513, 504, 487, 0, 0, 0, 0, 0, 0, 0, 0, 526, 504, 499, 508, 486, 492, 510, 539, 0, 0, 0, 0, 0, 0, 0, 0, 498, 497, 522, 505, 529, 520, 525, 542, 1, 0, 0, 0, 0, 0, 0, 0, 479, 488, 481, 509, 499, 523, 576, 500, 2, 2, 0, 0, 0, 0, 0, 0, 499, 503, 518, 470, 515, 501, 547, 496, 0, 0, 0, 0, 0, 0, 0, 0, 506, 532, 562, 509, 530, 487, 506, 512, 0, 0, 0, 0, 0, 0, 0, 0, 513, 513, 505, 491, 500, 533, 514, 483, 21, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4146, 4094, 4064, 4139, 4059, 4049, 4144, 4073] avg=4096.0000 max=4146 max_rank=0 min=4049 min_rank=5 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16325 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[498, 536, 491, 514, 495, 493, 490, 531, 8, 5, 0, 0, 0, 0, 0, 0, 504, 489, 502, 525, 519, 518, 547, 507, 0, 0, 0, 0, 0, 0, 0, 0, 542, 522, 502, 534, 501, 508, 503, 528, 0, 0, 0, 0, 0, 0, 0, 0, 495, 453, 500, 509, 516, 528, 523, 502, 1, 0, 0, 0, 0, 0, 0, 0, 504, 521, 533, 548, 499, 518, 502, 516, 2, 2, 0, 0, 0, 0, 0, 0, 528, 527, 528, 515, 541, 528, 506, 517, 0, 0, 0, 0, 0, 0, 0, 0, 492, 514, 514, 468, 505, 558, 491, 511, 0, 0, 0, 0, 0, 0, 0, 0, 507, 508, 508, 481, 523, 483, 498, 513, 20, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4111, 4140, 4027, 4145, 4190, 4053, 4041] avg=4096.0000 max=4190 max_rank=5 min=4027 min_rank=3 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[483, 522, 513, 498, 538, 502, 544, 459, 9, 4, 0, 0, 0, 0, 0, 0, 529, 496, 540, 498, 522, 516, 520, 507, 0, 0, 0, 0, 0, 0, 0, 0, 504, 488, 549, 504, 520, 531, 500, 455, 0, 0, 0, 0, 0, 0, 0, 0, 496, 524, 518, 499, 492, 514, 522, 518, 1, 0, 0, 0, 0, 0, 0, 0, 496, 529, 517, 541, 497, 488, 476, 521, 2, 2, 0, 0, 0, 0, 0, 0, 539, 541, 529, 489, 505, 520, 497, 520, 0, 0, 0, 0, 0, 0, 0, 0, 472, 504, 509, 532, 509, 543, 491, 542, 0, 0, 0, 0, 0, 0, 0, 0, 536, 511, 496, 500, 525, 513, 536, 485, 20, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4128, 4051, 4084, 4069, 4140, 4102, 4122] avg=4096.0000 max=4140 max_rank=5 min=4051 min_rank=2 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16433 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[513, 513, 528, 498, 525, 536, 531, 512, 13, 10, 0, 0, 0, 0, 0, 0, 517, 544, 502, 545, 483, 535, 515, 493, 0, 0, 0, 0, 0, 0, 0, 0, 500, 512, 495, 513, 522, 479, 504, 504, 0, 0, 0, 0, 0, 0, 0, 0, 539, 510, 507, 483, 515, 483, 502, 508, 8, 0, 0, 0, 0, 0, 0, 0, 498, 511, 526, 518, 527, 545, 469, 524, 8, 8, 7, 0, 0, 0, 0, 0, 507, 534, 486, 511, 496, 512, 523, 489, 0, 0, 0, 0, 0, 0, 0, 0, 542, 512, 519, 542, 467, 488, 508, 543, 0, 0, 0, 0, 0, 0, 0, 0, 486, 524, 517, 508, 516, 496, 511, 471, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4179, 4134, 4029, 4055, 4141, 4058, 4121, 4051] avg=4096.0000 max=4179 max_rank=0 min=4029 min_rank=2 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16371 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[484, 522, 496, 488, 516, 525, 503, 552, 8, 4, 0, 0, 0, 0, 0, 0, 498, 515, 499, 521, 483, 505, 511, 522, 0, 0, 0, 0, 0, 0, 0, 0, 496, 501, 542, 520, 539, 528, 503, 498, 0, 0, 0, 0, 0, 0, 0, 0, 502, 494, 476, 501, 523, 516, 516, 527, 1, 0, 0, 0, 0, 0, 0, 0, 485, 483, 510, 509, 538, 472, 511, 521, 2, 2, 0, 0, 0, 0, 0, 0, 561, 513, 512, 523, 459, 532, 505, 503, 0, 0, 0, 0, 0, 0, 0, 0, 529, 541, 542, 534, 501, 506, 534, 505, 0, 0, 0, 0, 0, 0, 0, 0, 519, 539, 461, 494, 496, 504, 521, 546, 20, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4054, 4127, 4056, 4033, 4108, 4192, 4100] avg=4096.0000 max=4192 max_rank=6 min=4033 min_rank=4 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16433 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[540, 474, 482, 518, 492, 510, 537, 521, 8, 4, 0, 0, 0, 0, 0, 0, 543, 504, 509, 505, 532, 518, 511, 488, 0, 0, 0, 0, 0, 0, 0, 0, 531, 510, 493, 514, 519, 523, 493, 503, 0, 0, 0, 0, 0, 0, 0, 0, 524, 519, 511, 541, 511, 530, 470, 561, 1, 0, 0, 0, 0, 0, 0, 0, 501, 533, 502, 502, 469, 540, 526, 533, 2, 2, 0, 0, 0, 0, 0, 0, 492, 511, 518, 501, 504, 508, 532, 514, 0, 0, 0, 0, 0, 0, 0, 0, 509, 513, 500, 481, 500, 487, 507, 518, 0, 0, 0, 0, 0, 0, 0, 0, 499, 484, 531, 535, 486, 514, 532, 512, 20, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4110, 4086, 4168, 4110, 4080, 4015, 4113] avg=4096.0000 max=4168 max_rank=3 min=4015 min_rank=6 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16318 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[3968, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[498, 520, 532, 506, 531, 493, 495, 515, 505, 509, 517, 532, 500, 523, 496, 511, 471, 512, 515, 521, 512, 526, 516, 511, 506, 497, 554, 526, 568, 544, 510, 496, 476, 478, 527, 518, 511, 505, 556, 469, 536, 545, 518, 560, 485, 475, 474, 494, 492, 544, 520, 548, 500, 512, 484, 502, 479, 506, 526, 548, 480, 482, 529, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[510, 526, 526, 499, 524, 517, 539, 525, 501, 527, 507, 503, 493, 503, 540, 484, 542, 524, 479, 510, 498, 506, 515, 507, 507, 522, 500, 524, 513, 498, 468, 528, 523, 499, 521, 530, 477, 534, 476, 533, 484, 510, 523, 456, 500, 526, 533, 524, 534, 495, 534, 484, 517, 493, 533, 531, 520, 512, 536, 500, 512, 528, 505, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[532, 548, 514, 468, 514, 497, 512, 500, 509, 488, 532, 520, 501, 497, 497, 519, 499, 513, 490, 494, 520, 561, 502, 492, 525, 513, 509, 507, 488, 522, 530, 512, 468, 523, 540, 556, 518, 557, 473, 468, 542, 509, 545, 515, 514, 485, 525, 517, 494, 515, 529, 566, 495, 530, 514, 487, 494, 510, 480, 507, 530, 505, 538, 494] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[486, 531, 523, 518, 520, 467, 528, 533, 497, 472, 559, 528, 506, 481, 526, 491, 521, 518, 532, 528, 494, 517, 527, 505, 505, 495, 525, 525, 489, 494, 488, 494, 504, 525, 525, 505, 521, 505, 516, 508, 563, 508, 473, 501, 531, 534, 538, 533, 502, 491, 513, 525, 511, 503, 478, 535, 473, 540, 480, 525, 506, 515, 505, 553] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[527, 521, 524, 508, 553, 493, 492, 559, 483, 522, 520, 555, 528, 501, 512, 508, 522, 498, 472, 484, 484, 487, 506, 508, 493, 544, 516, 515, 531, 493, 528, 509, 501, 505, 496, 504, 491, 503, 496, 530, 504, 494, 549, 515, 533, 531, 525, 535, 515, 510, 518, 498, 486, 485, 522, 535, 534, 504, 534, 509, 509, 502, 521, 478] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[484, 524, 510, 531, 508, 511, 522, 504, 494, 510, 551, 535, 475, 498, 527, 528, 533, 490, 526, 495, 525, 472, 526, 501, 492, 542, 505, 485, 468, 508, 505, 537, 484, 531, 518, 547, 547, 497, 501, 552, 521, 532, 526, 539, 513, 509, 506, 498, 517, 492, 495, 487, 513, 528, 493, 515, 522, 489, 512, 508, 523, 534, 509, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[526, 507, 493, 523, 509, 519, 527, 490, 501, 518, 496, 490, 506, 527, 505, 530, 518, 509, 521, 488, 500, 513, 514, 517, 485, 512, 525, 520, 527, 527, 540, 493, 512, 497, 507, 517, 555, 473, 500, 531, 468, 527, 502, 482, 509, 519, 531, 529, 475, 565, 542, 493, 508, 515, 522, 520, 526, 493, 498, 490, 526, 514, 507, 539] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[525, 526, 481, 587, 512, 510, 468, 486, 443, 490, 516, 532, 501, 495, 500, 516, 539, 515, 549, 501, 504, 513, 509, 494, 527, 476, 534, 508, 521, 514, 511, 507, 481, 560, 518, 512, 527, 508, 511, 521, 512, 521, 517, 516, 493, 514, 498, 497, 520, 529, 520, 553, 512, 515, 465, 543, 496, 478, 545, 490, 500, 513, 550, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[510, 509, 526, 499, 524, 517, 539, 525, 0, 0, 0, 0, 0, 0, 0, 0, 501, 527, 507, 503, 493, 503, 540, 484, 21, 1, 0, 0, 0, 0, 0, 0, 542, 524, 479, 510, 498, 506, 515, 507, 16, 3, 0, 0, 0, 0, 0, 0, 507, 522, 499, 524, 513, 498, 468, 528, 0, 0, 0, 0, 0, 0, 0, 0, 523, 499, 521, 529, 477, 534, 476, 533, 0, 0, 0, 0, 0, 0, 0, 0, 484, 510, 502, 456, 500, 526, 533, 524, 0, 0, 0, 0, 0, 0, 0, 0, 534, 495, 529, 484, 517, 493, 533, 531, 0, 0, 0, 0, 0, 0, 0, 0, 520, 512, 536, 500, 512, 528, 505, 520, 2, 1, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4149, 4080, 4100, 4059, 4092, 4035, 4116, 4137] avg=4096.0000 max=4149 max_rank=0 min=4035 min_rank=5 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[498, 504, 532, 506, 531, 493, 495, 515, 0, 0, 0, 0, 0, 0, 0, 0, 505, 509, 517, 532, 500, 523, 496, 511, 21, 0, 0, 0, 0, 0, 0, 0, 471, 512, 515, 521, 512, 526, 516, 511, 16, 3, 0, 0, 0, 0, 0, 0, 506, 497, 553, 526, 568, 544, 510, 496, 0, 0, 0, 0, 0, 0, 0, 0, 476, 478, 527, 517, 511, 505, 556, 469, 0, 0, 0, 0, 0, 0, 0, 0, 536, 545, 497, 560, 485, 475, 474, 494, 0, 0, 0, 0, 0, 0, 0, 0, 492, 544, 515, 548, 500, 512, 484, 502, 0, 0, 0, 0, 0, 0, 0, 0, 479, 506, 526, 548, 480, 482, 529, 521, 2, 1, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4114, 4103, 4200, 4039, 4066, 4097, 4075] avg=4096.0000 max=4200 max_rank=3 min=4039 min_rank=4 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16277 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[532, 530, 514, 468, 514, 497, 512, 500, 0, 0, 0, 0, 0, 0, 0, 0, 509, 488, 532, 520, 501, 497, 497, 519, 22, 1, 0, 0, 0, 0, 0, 0, 499, 513, 490, 494, 520, 561, 502, 492, 17, 3, 0, 0, 0, 0, 0, 0, 525, 513, 508, 507, 488, 522, 530, 512, 0, 0, 0, 0, 0, 0, 0, 0, 468, 523, 540, 555, 518, 557, 473, 468, 0, 0, 0, 0, 0, 0, 0, 0, 542, 509, 523, 515, 514, 485, 525, 517, 0, 0, 0, 0, 0, 0, 0, 0, 494, 515, 524, 566, 495, 530, 514, 487, 0, 0, 0, 0, 0, 0, 0, 0, 494, 510, 480, 507, 530, 505, 538, 494, 2, 1, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4086, 4091, 4105, 4102, 4130, 4125, 4062] avg=4096.0000 max=4130 max_rank=5 min=4062 min_rank=7 imbalance_ratio=1.008301 max_load_reduction=28638 max_load_reduction_pct=0.873962 moved_to_echo=16419 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[527, 505, 524, 508, 553, 493, 492, 559, 0, 0, 0, 0, 0, 0, 0, 0, 483, 522, 520, 555, 528, 501, 512, 508, 22, 0, 0, 0, 0, 0, 0, 0, 522, 498, 472, 484, 484, 487, 506, 508, 16, 3, 0, 0, 0, 0, 0, 0, 493, 544, 515, 515, 531, 493, 528, 509, 0, 0, 0, 0, 0, 0, 0, 0, 501, 505, 496, 503, 491, 503, 496, 530, 0, 0, 0, 0, 0, 0, 0, 0, 504, 494, 527, 515, 533, 531, 525, 535, 0, 0, 0, 0, 0, 0, 0, 0, 515, 510, 513, 498, 486, 485, 522, 535, 0, 0, 0, 0, 0, 0, 0, 0, 534, 504, 534, 509, 509, 502, 521, 478, 2, 1, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4161, 4151, 3980, 4128, 4025, 4164, 4064, 4095] avg=4096.0000 max=4164 max_rank=5 min=3980 min_rank=2 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16348 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[486, 514, 523, 518, 520, 467, 528, 533, 0, 0, 0, 0, 0, 0, 0, 0, 497, 472, 559, 528, 506, 481, 526, 491, 19, 1, 0, 0, 0, 0, 0, 0, 521, 518, 532, 528, 494, 517, 527, 505, 16, 3, 0, 0, 0, 0, 0, 0, 505, 495, 524, 525, 489, 494, 488, 494, 0, 0, 0, 0, 0, 0, 0, 0, 504, 525, 525, 504, 521, 505, 516, 508, 0, 0, 0, 0, 0, 0, 0, 0, 563, 508, 454, 501, 531, 534, 538, 533, 0, 0, 0, 0, 0, 0, 0, 0, 502, 491, 508, 525, 511, 503, 478, 535, 0, 0, 0, 0, 0, 0, 0, 0, 473, 540, 480, 525, 506, 515, 505, 553, 2, 1, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4080, 4161, 4014, 4108, 4162, 4053, 4101] avg=4096.0000 max=4162 max_rank=5 min=4014 min_rank=3 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16424 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[484, 501, 510, 531, 508, 511, 522, 504, 0, 0, 0, 0, 0, 0, 0, 0, 494, 510, 551, 535, 475, 498, 527, 528, 27, 4, 0, 0, 0, 0, 0, 0, 533, 490, 526, 495, 525, 472, 526, 501, 19, 5, 0, 0, 0, 0, 0, 0, 492, 542, 500, 485, 468, 508, 505, 537, 0, 0, 0, 0, 0, 0, 0, 0, 484, 531, 518, 544, 547, 497, 501, 552, 0, 0, 0, 0, 0, 0, 0, 0, 521, 532, 499, 539, 513, 509, 506, 498, 0, 0, 0, 0, 0, 0, 0, 0, 517, 492, 481, 487, 513, 528, 493, 515, 0, 0, 0, 0, 0, 0, 0, 0, 522, 489, 512, 508, 523, 534, 509, 488, 9, 5, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4071, 4149, 4092, 4037, 4174, 4117, 4026, 4102] avg=4096.0000 max=4174 max_rank=4 min=4026 min_rank=6 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16419 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[526, 492, 493, 523, 509, 519, 527, 490, 0, 0, 0, 0, 0, 0, 0, 0, 501, 518, 496, 490, 506, 527, 505, 530, 20, 0, 0, 0, 0, 0, 0, 0, 518, 509, 521, 488, 500, 513, 514, 517, 15, 3, 0, 0, 0, 0, 0, 0, 485, 512, 524, 520, 527, 527, 540, 493, 0, 0, 0, 0, 0, 0, 0, 0, 512, 497, 507, 516, 555, 473, 500, 531, 0, 0, 0, 0, 0, 0, 0, 0, 468, 527, 482, 482, 509, 519, 531, 529, 0, 0, 0, 0, 0, 0, 0, 0, 475, 565, 537, 493, 508, 515, 522, 520, 0, 0, 0, 0, 0, 0, 0, 0, 526, 493, 498, 490, 526, 514, 507, 539, 2, 1, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4093, 4098, 4128, 4091, 4047, 4135, 4097] avg=4096.0000 max=4135 max_rank=6 min=4047 min_rank=5 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16370 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[525, 509, 481, 587, 512, 510, 468, 486, 0, 0, 0, 0, 0, 0, 0, 0, 443, 490, 516, 532, 501, 495, 500, 516, 21, 1, 0, 0, 0, 0, 0, 0, 539, 515, 549, 501, 504, 513, 509, 494, 16, 3, 0, 0, 0, 0, 0, 0, 527, 476, 533, 508, 521, 514, 511, 507, 0, 0, 0, 0, 0, 0, 0, 0, 481, 560, 518, 511, 527, 508, 511, 521, 0, 0, 0, 0, 0, 0, 0, 0, 512, 521, 496, 516, 493, 514, 498, 497, 0, 0, 0, 0, 0, 0, 0, 0, 520, 529, 515, 553, 512, 515, 465, 543, 0, 0, 0, 0, 0, 0, 0, 0, 496, 478, 545, 490, 500, 513, 550, 523, 2, 1, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4015, 4143, 4097, 4137, 4047, 4152, 4099] avg=4096.0000 max=4152 max_rank=6 min=4015 min_rank=1 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16435 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3968, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3968, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[509, 523, 502, 492, 489, 540, 507, 526, 479, 501, 507, 493, 492, 530, 497, 542, 493, 507, 535, 521, 550, 499, 489, 449, 530, 527, 533, 534, 499, 493, 542, 514, 505, 508, 492, 540, 514, 524, 522, 543, 503, 527, 525, 517, 526, 487, 504, 487, 518, 502, 527, 505, 499, 501, 485, 557, 527, 507, 537, 501, 517, 485, 516, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[537, 513, 456, 468, 529, 510, 458, 493, 487, 520, 551, 515, 478, 519, 518, 500, 519, 502, 500, 511, 507, 499, 484, 507, 482, 522, 522, 527, 525, 543, 489, 456, 518, 553, 508, 515, 525, 520, 532, 515, 544, 460, 544, 534, 544, 491, 491, 520, 509, 513, 517, 526, 513, 532, 561, 505, 514, 530, 540, 477, 511, 527, 515, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[518, 485, 524, 544, 549, 521, 492, 500, 530, 496, 516, 519, 476, 480, 500, 510, 483, 508, 515, 543, 505, 503, 490, 495, 482, 533, 508, 507, 532, 531, 526, 507, 498, 500, 502, 538, 515, 533, 522, 553, 494, 513, 539, 549, 499, 509, 512, 522, 512, 497, 526, 467, 503, 529, 484, 513, 532, 468, 498, 511, 517, 508, 519, 558] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[493, 520, 522, 527, 499, 528, 508, 509, 509, 541, 528, 537, 488, 522, 524, 519, 529, 538, 498, 494, 487, 548, 485, 503, 532, 515, 486, 537, 543, 549, 518, 498, 495, 528, 495, 495, 496, 503, 446, 546, 521, 504, 503, 518, 490, 471, 528, 496, 518, 522, 531, 518, 535, 515, 483, 546, 520, 493, 527, 497, 501, 514, 486, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[519, 523, 506, 517, 513, 521, 501, 536, 504, 521, 508, 487, 569, 487, 472, 491, 515, 482, 509, 518, 489, 526, 506, 518, 546, 476, 538, 477, 494, 485, 556, 530, 526, 505, 497, 522, 504, 521, 513, 511, 550, 474, 526, 467, 485, 528, 508, 527, 560, 511, 505, 492, 510, 501, 528, 532, 517, 503, 515, 532, 528, 513, 518, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[496, 512, 519, 489, 535, 524, 495, 509, 534, 492, 511, 531, 533, 513, 509, 497, 520, 556, 505, 524, 526, 537, 487, 507, 514, 494, 532, 493, 502, 508, 510, 500, 518, 487, 492, 528, 494, 546, 506, 493, 522, 520, 515, 512, 519, 537, 499, 526, 518, 482, 520, 495, 484, 533, 506, 507, 539, 480, 528, 530, 524, 498, 487, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[544, 505, 507, 525, 518, 523, 516, 478, 495, 478, 520, 502, 532, 518, 528, 498, 514, 559, 523, 513, 541, 544, 471, 508, 540, 466, 481, 493, 484, 491, 505, 513, 495, 532, 493, 529, 478, 525, 534, 528, 524, 522, 517, 533, 524, 529, 478, 480, 536, 515, 540, 494, 481, 505, 499, 509, 533, 518, 480, 544, 552, 528, 476, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[537, 496, 469, 469, 492, 481, 508, 546, 465, 514, 493, 539, 489, 486, 529, 482, 516, 502, 524, 532, 482, 498, 513, 501, 526, 519, 530, 546, 531, 519, 535, 528, 488, 521, 495, 488, 509, 538, 523, 550, 509, 491, 489, 522, 524, 504, 489, 483, 515, 531, 487, 565, 523, 515, 540, 520, 495, 562, 531, 531, 497, 471, 540, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[537, 513, 456, 468, 529, 510, 458, 493, 9, 8, 0, 0, 0, 0, 0, 0, 487, 520, 551, 515, 478, 519, 518, 500, 19, 6, 0, 0, 0, 0, 0, 0, 519, 502, 500, 511, 507, 499, 484, 507, 6, 6, 0, 0, 0, 0, 0, 0, 482, 522, 522, 527, 525, 543, 474, 456, 0, 0, 0, 0, 0, 0, 0, 0, 518, 553, 508, 515, 525, 520, 532, 501, 0, 0, 0, 0, 0, 0, 0, 0, 544, 460, 544, 534, 544, 491, 491, 520, 7, 0, 0, 0, 0, 0, 0, 0, 509, 513, 517, 526, 513, 532, 561, 486, 0, 0, 0, 0, 0, 0, 0, 0, 501, 530, 540, 477, 511, 527, 515, 517, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3981, 4113, 4041, 4051, 4172, 4135, 4157, 4118] avg=4096.0000 max=4172 max_rank=4 min=3981 min_rank=0 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16582 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[509, 523, 502, 492, 489, 540, 507, 526, 10, 8, 0, 0, 0, 0, 0, 0, 479, 501, 507, 493, 492, 530, 497, 542, 21, 7, 0, 0, 0, 0, 0, 0, 493, 507, 535, 521, 550, 499, 489, 449, 6, 6, 0, 0, 0, 0, 0, 0, 530, 527, 533, 534, 499, 493, 525, 514, 0, 0, 0, 0, 0, 0, 0, 0, 505, 508, 492, 540, 514, 524, 522, 529, 0, 0, 0, 0, 0, 0, 0, 0, 503, 527, 525, 517, 526, 487, 504, 487, 7, 0, 0, 0, 0, 0, 0, 0, 518, 502, 527, 505, 499, 501, 485, 536, 0, 0, 0, 0, 0, 0, 0, 0, 514, 507, 537, 501, 517, 485, 516, 516, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4069, 4055, 4155, 4134, 4083, 4073, 4093] avg=4096.0000 max=4155 max_rank=3 min=4055 min_rank=2 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[496, 512, 519, 489, 535, 524, 495, 509, 9, 8, 0, 0, 0, 0, 0, 0, 534, 492, 511, 531, 533, 513, 509, 497, 19, 6, 0, 0, 0, 0, 0, 0, 520, 556, 505, 524, 526, 537, 487, 507, 7, 5, 0, 0, 0, 0, 0, 0, 514, 494, 532, 493, 502, 508, 495, 500, 0, 0, 0, 0, 0, 0, 0, 0, 518, 487, 492, 528, 494, 546, 506, 480, 0, 0, 0, 0, 0, 0, 0, 0, 522, 520, 515, 512, 519, 537, 499, 526, 8, 0, 0, 0, 0, 0, 0, 0, 518, 482, 520, 495, 484, 533, 506, 488, 0, 0, 0, 0, 0, 0, 0, 0, 524, 480, 528, 530, 524, 498, 487, 509, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4145, 4174, 4038, 4051, 4158, 4026, 4080] avg=4096.0000 max=4174 max_rank=2 min=4026 min_rank=6 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16315 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[518, 485, 524, 544, 549, 521, 492, 500, 9, 9, 0, 0, 0, 0, 0, 0, 530, 496, 516, 519, 476, 480, 500, 510, 19, 7, 0, 0, 0, 0, 0, 0, 483, 508, 515, 543, 505, 503, 490, 495, 7, 6, 0, 0, 0, 0, 0, 0, 482, 533, 508, 507, 532, 531, 510, 507, 0, 0, 0, 0, 0, 0, 0, 0, 498, 500, 502, 538, 515, 533, 522, 538, 0, 0, 0, 0, 0, 0, 0, 0, 494, 513, 539, 549, 499, 509, 512, 522, 8, 0, 0, 0, 0, 0, 0, 0, 512, 497, 526, 467, 503, 529, 484, 494, 0, 0, 0, 0, 0, 0, 0, 0, 517, 468, 498, 511, 517, 508, 519, 558, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4151, 4053, 4055, 4110, 4146, 4145, 4012, 4096] avg=4096.0000 max=4151 max_rank=0 min=4012 min_rank=6 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[493, 520, 522, 527, 499, 528, 508, 509, 9, 8, 0, 0, 0, 0, 0, 0, 509, 541, 528, 537, 488, 522, 524, 519, 20, 7, 0, 0, 0, 0, 0, 0, 529, 538, 498, 494, 487, 548, 485, 503, 6, 6, 0, 0, 0, 0, 0, 0, 532, 515, 486, 537, 543, 549, 502, 498, 0, 0, 0, 0, 0, 0, 0, 0, 495, 528, 495, 495, 496, 503, 446, 532, 0, 0, 0, 0, 0, 0, 0, 0, 521, 504, 503, 518, 490, 471, 528, 496, 7, 0, 0, 0, 0, 0, 0, 0, 518, 522, 531, 518, 535, 515, 483, 526, 0, 0, 0, 0, 0, 0, 0, 0, 507, 493, 527, 497, 501, 514, 486, 493, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4123, 4195, 4094, 4162, 3990, 4038, 4148, 4018] avg=4096.0000 max=4195 max_rank=1 min=3990 min_rank=4 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16194 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[519, 523, 506, 517, 513, 521, 501, 536, 10, 8, 0, 0, 0, 0, 0, 0, 504, 521, 508, 487, 569, 487, 472, 491, 20, 7, 0, 0, 0, 0, 0, 0, 515, 482, 509, 518, 489, 526, 506, 518, 6, 6, 0, 0, 0, 0, 0, 0, 546, 476, 538, 477, 494, 485, 539, 530, 0, 0, 0, 0, 0, 0, 0, 0, 526, 505, 497, 522, 504, 521, 513, 497, 0, 0, 0, 0, 0, 0, 0, 0, 550, 474, 526, 467, 485, 528, 508, 527, 7, 0, 0, 0, 0, 0, 0, 0, 560, 511, 505, 492, 510, 501, 528, 512, 0, 0, 0, 0, 0, 0, 0, 0, 504, 503, 515, 532, 528, 513, 518, 499, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4154, 4066, 4075, 4085, 4085, 4072, 4119, 4112] avg=4096.0000 max=4154 max_rank=0 min=4066 min_rank=1 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[544, 505, 507, 525, 518, 523, 516, 478, 13, 12, 0, 0, 0, 0, 0, 0, 495, 478, 520, 502, 532, 518, 528, 498, 23, 10, 0, 0, 0, 0, 0, 0, 514, 559, 523, 513, 541, 544, 471, 508, 11, 10, 0, 0, 0, 0, 0, 0, 540, 466, 481, 493, 484, 491, 482, 513, 0, 0, 0, 0, 0, 0, 0, 0, 495, 532, 493, 529, 478, 525, 534, 506, 0, 0, 0, 0, 0, 0, 0, 0, 524, 522, 517, 533, 524, 529, 478, 480, 12, 0, 0, 0, 0, 0, 0, 0, 536, 515, 540, 494, 481, 505, 499, 486, 0, 0, 0, 0, 0, 0, 0, 0, 510, 518, 480, 544, 552, 528, 476, 504, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4141, 4104, 4194, 3950, 4092, 4119, 4056, 4112] avg=4096.0000 max=4194 max_rank=2 min=3950 min_rank=3 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[537, 496, 469, 469, 492, 481, 508, 546, 10, 8, 0, 0, 0, 0, 0, 0, 465, 514, 493, 539, 489, 486, 529, 482, 19, 7, 0, 0, 0, 0, 0, 0, 516, 502, 524, 532, 482, 498, 513, 501, 6, 6, 0, 0, 0, 0, 0, 0, 526, 519, 530, 546, 531, 519, 518, 528, 0, 0, 0, 0, 0, 0, 0, 0, 488, 521, 495, 488, 509, 538, 523, 536, 0, 0, 0, 0, 0, 0, 0, 0, 509, 491, 489, 522, 524, 504, 489, 483, 7, 0, 0, 0, 0, 0, 0, 0, 515, 531, 487, 565, 523, 515, 540, 501, 0, 0, 0, 0, 0, 0, 0, 0, 482, 562, 531, 531, 497, 471, 540, 525, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4016, 4023, 4080, 4217, 4098, 4018, 4177, 4139] avg=4096.0000 max=4217 max_rank=3 min=4016 min_rank=0 imbalance_ratio=1.029541 max_load_reduction=28551 max_load_reduction_pct=0.871307 moved_to_echo=16432 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 3968, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[496, 503, 542, 498, 525, 480, 537, 502, 482, 466, 545, 513, 520, 505, 489, 511, 520, 553, 524, 495, 541, 506, 474, 537, 494, 462, 505, 511, 501, 539, 517, 522, 519, 549, 500, 530, 486, 502, 489, 498, 490, 510, 520, 509, 507, 517, 514, 542, 531, 510, 522, 510, 530, 492, 501, 551, 532, 538, 519, 488, 507, 542, 510, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[529, 484, 498, 512, 540, 493, 534, 500, 483, 496, 552, 494, 518, 538, 495, 501, 514, 523, 484, 496, 514, 540, 511, 506, 507, 513, 549, 517, 502, 486, 503, 513, 507, 507, 517, 552, 536, 502, 515, 514, 518, 502, 512, 516, 505, 489, 542, 508, 510, 485, 521, 515, 562, 513, 532, 537, 475, 467, 506, 499, 509, 520, 512, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[514, 507, 506, 501, 564, 520, 554, 526, 536, 495, 483, 536, 535, 507, 508, 548, 485, 537, 462, 511, 504, 532, 490, 534, 534, 527, 456, 509, 522, 464, 512, 527, 506, 488, 506, 517, 527, 506, 510, 494, 511, 514, 528, 505, 520, 504, 546, 533, 468, 501, 527, 516, 506, 492, 512, 505, 492, 515, 510, 503, 542, 524, 515, 479] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[522, 513, 526, 496, 476, 515, 476, 484, 527, 517, 516, 490, 502, 533, 513, 525, 499, 514, 524, 537, 564, 488, 536, 514, 516, 544, 509, 550, 502, 520, 504, 517, 506, 492, 501, 497, 534, 528, 489, 519, 518, 473, 536, 516, 503, 475, 489, 503, 523, 553, 506, 545, 512, 511, 503, 515, 505, 526, 510, 490, 511, 514, 492, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[538, 480, 513, 508, 486, 509, 525, 494, 511, 525, 529, 516, 493, 507, 511, 532, 490, 523, 491, 508, 521, 530, 531, 529, 540, 534, 519, 501, 492, 551, 475, 495, 510, 529, 506, 515, 517, 517, 442, 552, 523, 486, 486, 503, 491, 480, 503, 557, 518, 515, 518, 513, 526, 510, 516, 503, 494, 549, 501, 505, 530, 506, 488, 552] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[514, 516, 489, 551, 514, 485, 498, 528, 513, 509, 513, 483, 545, 527, 500, 501, 485, 528, 535, 498, 558, 476, 539, 481, 514, 504, 485, 519, 524, 523, 524, 506, 509, 542, 468, 491, 526, 542, 551, 488, 529, 473, 502, 514, 514, 545, 501, 497, 531, 469, 507, 513, 526, 554, 554, 515, 477, 517, 512, 517, 507, 498, 510, 484] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[518, 512, 470, 537, 526, 527, 504, 487, 490, 515, 480, 496, 519, 491, 517, 516, 503, 533, 523, 515, 507, 515, 512, 488, 505, 532, 494, 506, 529, 502, 469, 522, 479, 540, 535, 509, 505, 502, 577, 577, 520, 529, 504, 545, 535, 474, 531, 498, 539, 474, 515, 470, 497, 513, 525, 524, 539, 532, 535, 488, 524, 479, 523, 471] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[474, 521, 523, 521, 534, 509, 516, 521, 493, 530, 484, 500, 520, 507, 496, 515, 514, 523, 519, 546, 492, 488, 487, 536, 515, 507, 501, 478, 519, 516, 491, 529, 518, 514, 512, 500, 496, 533, 560, 486, 483, 514, 514, 486, 496, 550, 512, 508, 503, 509, 502, 525, 528, 497, 499, 508, 523, 509, 536, 498, 529, 511, 528, 556] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[496, 503, 542, 498, 525, 480, 537, 502, 6, 0, 0, 0, 0, 0, 0, 0, 482, 466, 545, 513, 520, 505, 489, 511, 13, 0, 0, 0, 0, 0, 0, 0, 520, 538, 524, 495, 541, 506, 474, 537, 0, 0, 0, 0, 0, 0, 0, 0, 494, 462, 505, 511, 501, 539, 517, 522, 7, 4, 0, 0, 0, 0, 0, 0, 519, 533, 500, 530, 486, 502, 489, 498, 0, 0, 0, 0, 0, 0, 0, 0, 490, 510, 520, 509, 507, 517, 514, 542, 11, 0, 0, 0, 0, 0, 0, 0, 531, 510, 522, 510, 510, 492, 501, 551, 0, 0, 0, 0, 0, 0, 0, 0, 532, 538, 519, 488, 507, 542, 510, 488, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4044, 4135, 4062, 4057, 4120, 4127, 4134] avg=4096.0000 max=4135 max_rank=2 min=4044 min_rank=1 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16438 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[529, 484, 498, 512, 540, 493, 534, 500, 5, 0, 0, 0, 0, 0, 0, 0, 483, 496, 552, 494, 518, 538, 495, 501, 14, 0, 0, 0, 0, 0, 0, 0, 514, 508, 484, 496, 514, 540, 511, 506, 0, 0, 0, 0, 0, 0, 0, 0, 507, 513, 549, 517, 502, 486, 503, 513, 8, 4, 0, 0, 0, 0, 0, 0, 507, 493, 517, 552, 536, 502, 515, 514, 0, 0, 0, 0, 0, 0, 0, 0, 518, 502, 512, 516, 505, 489, 542, 508, 11, 0, 0, 0, 0, 0, 0, 0, 510, 485, 521, 515, 540, 513, 532, 537, 0, 0, 0, 0, 0, 0, 0, 0, 475, 467, 506, 499, 509, 520, 512, 518, 9, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4091, 4073, 4102, 4136, 4103, 4153, 4015] avg=4096.0000 max=4153 max_rank=6 min=4015 min_rank=7 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16407 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[514, 516, 489, 551, 514, 485, 498, 528, 6, 0, 0, 0, 0, 0, 0, 0, 513, 509, 513, 483, 545, 527, 500, 501, 13, 0, 0, 0, 0, 0, 0, 0, 485, 513, 535, 498, 558, 476, 539, 481, 0, 0, 0, 0, 0, 0, 0, 0, 514, 504, 485, 519, 524, 523, 524, 506, 7, 4, 0, 0, 0, 0, 0, 0, 509, 527, 468, 491, 526, 542, 551, 488, 0, 0, 0, 0, 0, 0, 0, 0, 529, 473, 502, 514, 514, 545, 501, 497, 11, 0, 0, 0, 0, 0, 0, 0, 531, 469, 507, 513, 506, 554, 554, 515, 0, 0, 0, 0, 0, 0, 0, 0, 477, 517, 512, 517, 507, 498, 510, 484, 9, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4104, 4085, 4110, 4102, 4086, 4149, 4031] avg=4096.0000 max=4149 max_rank=6 min=4031 min_rank=7 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[514, 507, 506, 501, 564, 520, 554, 526, 5, 0, 0, 0, 0, 0, 0, 0, 536, 495, 483, 536, 535, 507, 508, 548, 12, 0, 0, 0, 0, 0, 0, 0, 485, 522, 462, 511, 504, 532, 490, 534, 0, 0, 0, 0, 0, 0, 0, 0, 534, 527, 456, 509, 522, 464, 512, 527, 7, 4, 0, 0, 0, 0, 0, 0, 506, 475, 506, 517, 527, 506, 510, 494, 0, 0, 0, 0, 0, 0, 0, 0, 511, 514, 528, 505, 520, 504, 546, 533, 11, 0, 0, 0, 0, 0, 0, 0, 468, 501, 527, 516, 487, 492, 512, 505, 0, 0, 0, 0, 0, 0, 0, 0, 492, 515, 510, 503, 542, 524, 515, 479, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4197, 4160, 4040, 4062, 4041, 4172, 4008, 4088] avg=4096.0000 max=4197 max_rank=0 min=4008 min_rank=6 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16309 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[522, 513, 526, 496, 476, 515, 476, 484, 9, 0, 0, 0, 0, 0, 0, 0, 527, 517, 516, 490, 502, 533, 513, 525, 15, 0, 0, 0, 0, 0, 0, 0, 499, 494, 524, 537, 564, 488, 536, 514, 0, 0, 0, 0, 0, 0, 0, 0, 516, 544, 509, 550, 502, 520, 504, 517, 10, 5, 0, 0, 0, 0, 0, 0, 506, 470, 501, 497, 534, 528, 489, 519, 0, 0, 0, 0, 0, 0, 0, 0, 518, 473, 536, 516, 503, 475, 489, 503, 13, 0, 0, 0, 0, 0, 0, 0, 523, 553, 506, 545, 487, 511, 503, 515, 0, 0, 0, 0, 0, 0, 0, 0, 505, 526, 510, 490, 511, 514, 492, 504, 13, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4017, 4138, 4156, 4177, 4044, 4026, 4143, 4067] avg=4096.0000 max=4177 max_rank=3 min=4017 min_rank=0 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16280 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[538, 480, 513, 508, 486, 509, 525, 494, 5, 0, 0, 0, 0, 0, 0, 0, 511, 525, 529, 516, 493, 507, 511, 532, 13, 0, 0, 0, 0, 0, 0, 0, 490, 508, 491, 508, 521, 530, 531, 529, 0, 0, 0, 0, 0, 0, 0, 0, 540, 534, 519, 501, 492, 551, 475, 495, 7, 4, 0, 0, 0, 0, 0, 0, 510, 515, 506, 515, 517, 517, 442, 552, 0, 0, 0, 0, 0, 0, 0, 0, 523, 486, 486, 503, 491, 480, 503, 557, 11, 0, 0, 0, 0, 0, 0, 0, 518, 515, 518, 513, 506, 510, 516, 503, 0, 0, 0, 0, 0, 0, 0, 0, 494, 549, 501, 505, 530, 506, 488, 552, 9, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4058, 4137, 4108, 4118, 4074, 4040, 4099, 4134] avg=4096.0000 max=4137 max_rank=1 min=4040 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[518, 512, 470, 537, 526, 527, 504, 487, 6, 0, 0, 0, 0, 0, 0, 0, 490, 515, 480, 496, 519, 491, 517, 516, 12, 0, 0, 0, 0, 0, 0, 0, 503, 518, 523, 515, 507, 515, 512, 488, 0, 0, 0, 0, 0, 0, 0, 0, 505, 532, 494, 506, 529, 502, 469, 522, 7, 4, 0, 0, 0, 0, 0, 0, 479, 525, 535, 509, 505, 502, 577, 577, 0, 0, 0, 0, 0, 0, 0, 0, 520, 529, 504, 545, 535, 474, 531, 498, 11, 0, 0, 0, 0, 0, 0, 0, 539, 474, 515, 470, 478, 513, 525, 524, 0, 0, 0, 0, 0, 0, 0, 0, 539, 532, 535, 488, 524, 479, 523, 471, 9, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4036, 4081, 4070, 4209, 4147, 4038, 4100] avg=4096.0000 max=4209 max_rank=4 min=4036 min_rank=1 imbalance_ratio=1.027588 max_load_reduction=28559 max_load_reduction_pct=0.871552 moved_to_echo=16494 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[474, 521, 523, 521, 534, 509, 516, 521, 5, 0, 0, 0, 0, 0, 0, 0, 493, 530, 484, 500, 520, 507, 496, 515, 13, 0, 0, 0, 0, 0, 0, 0, 514, 508, 519, 546, 492, 488, 487, 536, 0, 0, 0, 0, 0, 0, 0, 0, 515, 507, 501, 478, 519, 516, 491, 529, 7, 4, 0, 0, 0, 0, 0, 0, 518, 500, 512, 500, 496, 533, 560, 486, 0, 0, 0, 0, 0, 0, 0, 0, 483, 514, 514, 486, 496, 550, 512, 508, 11, 0, 0, 0, 0, 0, 0, 0, 503, 509, 502, 525, 508, 497, 499, 508, 0, 0, 0, 0, 0, 0, 0, 0, 523, 509, 536, 498, 529, 511, 528, 556, 9, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4124, 4058, 4090, 4067, 4105, 4074, 4051, 4199] avg=4096.0000 max=4199 max_rank=7 min=4051 min_rank=6 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[557, 521, 530, 511, 525, 495, 540, 467, 506, 525, 537, 501, 529, 535, 498, 521, 475, 531, 525, 508, 509, 560, 520, 494, 460, 536, 531, 529, 505, 494, 501, 532, 526, 527, 500, 526, 564, 509, 470, 479, 529, 505, 489, 473, 497, 493, 481, 543, 495, 521, 527, 513, 501, 489, 498, 533, 532, 534, 471, 528, 473, 492, 552, 490] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[517, 534, 498, 499, 522, 504, 541, 505, 505, 513, 504, 491, 512, 524, 492, 519, 490, 526, 503, 502, 539, 496, 527, 538, 515, 497, 500, 542, 500, 472, 547, 516, 502, 551, 499, 509, 519, 508, 525, 522, 481, 511, 479, 492, 494, 473, 512, 490, 555, 512, 518, 556, 462, 510, 534, 513, 522, 530, 538, 525, 509, 514, 506, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[494, 516, 508, 505, 497, 548, 582, 502, 512, 493, 489, 530, 497, 545, 493, 487, 525, 553, 515, 475, 535, 503, 556, 513, 507, 513, 500, 480, 533, 535, 505, 504, 483, 523, 469, 507, 461, 516, 498, 515, 517, 492, 526, 512, 505, 525, 494, 527, 517, 512, 516, 489, 544, 527, 479, 560, 507, 503, 505, 532, 543, 478, 510, 526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[538, 477, 517, 484, 528, 504, 522, 534, 501, 507, 523, 522, 512, 534, 459, 496, 500, 497, 533, 524, 485, 525, 486, 529, 537, 461, 502, 575, 538, 535, 527, 504, 556, 497, 542, 499, 494, 495, 526, 502, 494, 476, 533, 500, 528, 486, 511, 502, 521, 506, 495, 502, 496, 514, 506, 493, 538, 516, 535, 483, 526, 551, 545, 484] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[561, 516, 497, 495, 536, 505, 504, 517, 506, 488, 540, 509, 522, 526, 523, 493, 503, 513, 509, 521, 518, 473, 555, 487, 500, 512, 509, 504, 530, 473, 509, 507, 493, 505, 517, 460, 544, 525, 503, 506, 537, 513, 531, 527, 495, 513, 497, 515, 542, 486, 491, 536, 517, 478, 525, 509, 486, 506, 525, 526, 473, 551, 525, 550] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[490, 524, 510, 543, 532, 486, 512, 496, 515, 514, 515, 535, 539, 531, 492, 534, 487, 557, 502, 524, 519, 438, 524, 490, 531, 528, 536, 541, 479, 501, 480, 522, 543, 502, 590, 517, 482, 496, 504, 490, 493, 497, 493, 500, 529, 496, 528, 517, 484, 482, 498, 517, 544, 523, 502, 519, 499, 526, 514, 494, 501, 520, 531, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[487, 521, 520, 528, 515, 531, 486, 514, 502, 489, 521, 475, 543, 501, 500, 528, 526, 524, 495, 515, 521, 507, 530, 469, 489, 477, 485, 515, 550, 551, 514, 542, 481, 492, 514, 498, 522, 520, 534, 499, 519, 531, 475, 506, 490, 527, 490, 532, 490, 509, 512, 523, 507, 516, 566, 517, 459, 538, 531, 511, 510, 541, 528, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[518, 526, 515, 523, 517, 497, 523, 513, 511, 529, 507, 530, 496, 504, 519, 504, 487, 509, 501, 501, 541, 515, 508, 515, 501, 514, 541, 538, 517, 516, 539, 512, 501, 521, 473, 514, 472, 516, 506, 538, 529, 508, 496, 494, 489, 539, 452, 526, 509, 502, 478, 511, 543, 513, 532, 495, 525, 501, 505, 498, 521, 553, 520, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[517, 534, 498, 499, 522, 504, 515, 505, 0, 0, 0, 0, 0, 0, 0, 0, 505, 513, 504, 491, 512, 524, 492, 519, 1, 0, 0, 0, 0, 0, 0, 0, 490, 525, 503, 502, 539, 496, 527, 538, 0, 0, 0, 0, 0, 0, 0, 0, 515, 497, 500, 526, 500, 472, 547, 516, 0, 0, 0, 0, 0, 0, 0, 0, 502, 551, 499, 509, 519, 508, 525, 522, 16, 2, 2, 0, 0, 0, 0, 0, 481, 511, 479, 492, 494, 473, 512, 490, 26, 24, 0, 0, 0, 0, 0, 0, 555, 512, 518, 554, 462, 510, 534, 513, 0, 0, 0, 0, 0, 0, 0, 0, 522, 530, 538, 525, 509, 514, 480, 507, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4061, 4120, 4073, 4155, 3982, 4158, 4125] avg=4096.0000 max=4158 max_rank=6 min=3982 min_rank=5 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16420 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[557, 521, 530, 511, 525, 495, 514, 467, 0, 0, 0, 0, 0, 0, 0, 0, 506, 525, 537, 501, 529, 535, 498, 521, 1, 0, 0, 0, 0, 0, 0, 0, 475, 530, 525, 508, 509, 560, 520, 494, 0, 0, 0, 0, 0, 0, 0, 0, 460, 536, 531, 513, 505, 494, 501, 532, 0, 0, 0, 0, 0, 0, 0, 0, 526, 527, 500, 526, 564, 509, 470, 479, 16, 2, 2, 0, 0, 0, 0, 0, 529, 505, 489, 473, 497, 493, 481, 543, 29, 24, 0, 0, 0, 0, 0, 0, 495, 521, 527, 511, 501, 489, 498, 533, 0, 0, 0, 0, 0, 0, 0, 0, 532, 534, 471, 528, 473, 492, 523, 490, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4153, 4121, 4072, 4121, 4063, 4075, 4043] avg=4096.0000 max=4153 max_rank=1 min=4043 min_rank=7 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16302 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[538, 477, 517, 484, 528, 504, 497, 534, 0, 0, 0, 0, 0, 0, 0, 0, 501, 507, 523, 522, 512, 534, 459, 496, 1, 0, 0, 0, 0, 0, 0, 0, 500, 496, 533, 524, 485, 525, 486, 529, 0, 0, 0, 0, 0, 0, 0, 0, 537, 461, 502, 558, 538, 535, 527, 504, 0, 0, 0, 0, 0, 0, 0, 0, 556, 497, 542, 499, 494, 495, 526, 502, 17, 2, 2, 0, 0, 0, 0, 0, 494, 476, 533, 500, 528, 486, 511, 502, 29, 23, 0, 0, 0, 0, 0, 0, 521, 506, 495, 500, 496, 514, 506, 493, 0, 0, 0, 0, 0, 0, 0, 0, 538, 516, 535, 483, 526, 551, 516, 484, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4055, 4078, 4162, 4132, 4082, 4031, 4149] avg=4096.0000 max=4162 max_rank=3 min=4031 min_rank=6 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16394 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[494, 516, 508, 505, 497, 548, 543, 502, 0, 0, 0, 0, 0, 0, 0, 0, 512, 493, 489, 530, 497, 545, 493, 487, 6, 2, 0, 0, 0, 0, 0, 0, 525, 547, 515, 475, 535, 503, 556, 513, 0, 0, 0, 0, 0, 0, 0, 0, 507, 513, 500, 463, 533, 535, 505, 504, 0, 0, 0, 0, 0, 0, 0, 0, 483, 523, 469, 507, 461, 516, 498, 515, 17, 9, 6, 0, 0, 0, 0, 0, 517, 492, 526, 512, 505, 525, 494, 527, 30, 30, 0, 0, 0, 0, 0, 0, 517, 512, 516, 481, 544, 527, 479, 560, 0, 0, 0, 0, 0, 0, 0, 0, 507, 503, 505, 532, 543, 478, 480, 526, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4054, 4169, 4060, 4004, 4158, 4136, 4074] avg=4096.0000 max=4169 max_rank=2 min=4004 min_rank=4 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[561, 516, 497, 495, 536, 505, 480, 517, 0, 0, 0, 0, 0, 0, 0, 0, 506, 488, 540, 509, 522, 526, 523, 493, 1, 0, 0, 0, 0, 0, 0, 0, 503, 512, 509, 521, 518, 473, 555, 487, 0, 0, 0, 0, 0, 0, 0, 0, 500, 512, 509, 489, 530, 473, 509, 507, 0, 0, 0, 0, 0, 0, 0, 0, 493, 505, 517, 460, 544, 525, 503, 506, 15, 2, 2, 0, 0, 0, 0, 0, 537, 513, 531, 527, 495, 513, 497, 515, 28, 22, 0, 0, 0, 0, 0, 0, 542, 486, 491, 534, 517, 478, 525, 509, 0, 0, 0, 0, 0, 0, 0, 0, 486, 506, 525, 526, 473, 551, 497, 550, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4108, 4078, 4029, 4072, 4178, 4082, 4114] avg=4096.0000 max=4178 max_rank=5 min=4029 min_rank=3 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16446 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[487, 521, 520, 528, 515, 531, 463, 514, 0, 0, 0, 0, 0, 0, 0, 0, 502, 489, 521, 475, 543, 501, 500, 528, 1, 0, 0, 0, 0, 0, 0, 0, 526, 523, 495, 515, 521, 507, 530, 469, 0, 0, 0, 0, 0, 0, 0, 0, 489, 477, 485, 500, 550, 551, 514, 542, 0, 0, 0, 0, 0, 0, 0, 0, 481, 492, 514, 498, 522, 520, 534, 499, 15, 2, 2, 0, 0, 0, 0, 0, 519, 531, 475, 506, 490, 527, 490, 532, 28, 21, 0, 0, 0, 0, 0, 0, 490, 509, 512, 521, 507, 516, 566, 517, 0, 0, 0, 0, 0, 0, 0, 0, 459, 538, 531, 511, 510, 541, 500, 509, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4060, 4086, 4108, 4079, 4119, 4138, 4099] avg=4096.0000 max=4138 max_rank=6 min=4060 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16435 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[490, 524, 510, 543, 532, 486, 488, 496, 0, 0, 0, 0, 0, 0, 0, 0, 515, 514, 515, 535, 539, 531, 492, 534, 1, 0, 0, 0, 0, 0, 0, 0, 487, 556, 502, 524, 519, 438, 524, 490, 0, 0, 0, 0, 0, 0, 0, 0, 531, 528, 536, 525, 479, 501, 480, 522, 0, 0, 0, 0, 0, 0, 0, 0, 543, 502, 590, 517, 482, 496, 504, 490, 16, 2, 2, 0, 0, 0, 0, 0, 493, 497, 493, 500, 529, 496, 528, 517, 28, 22, 0, 0, 0, 0, 0, 0, 484, 482, 498, 515, 544, 523, 502, 519, 0, 0, 0, 0, 0, 0, 0, 0, 499, 526, 514, 494, 501, 520, 503, 510, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4176, 4040, 4102, 4144, 4103, 4067, 4067] avg=4096.0000 max=4176 max_rank=1 min=4040 min_rank=2 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[518, 526, 515, 523, 517, 497, 498, 513, 0, 0, 0, 0, 0, 0, 0, 0, 511, 529, 507, 530, 496, 504, 519, 504, 1, 0, 0, 0, 0, 0, 0, 0, 487, 508, 501, 501, 541, 515, 508, 515, 0, 0, 0, 0, 0, 0, 0, 0, 501, 514, 541, 522, 517, 516, 539, 512, 0, 0, 0, 0, 0, 0, 0, 0, 501, 521, 473, 514, 472, 516, 506, 538, 16, 2, 2, 0, 0, 0, 0, 0, 529, 508, 496, 494, 489, 539, 452, 526, 27, 23, 0, 0, 0, 0, 0, 0, 509, 502, 478, 509, 543, 513, 532, 495, 0, 0, 0, 0, 0, 0, 0, 0, 525, 501, 505, 498, 521, 553, 493, 501, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4101, 4076, 4162, 4061, 4083, 4081, 4097] avg=4096.0000 max=4162 max_rank=3 min=4061 min_rank=4 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16322 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 3968, 4224, 256, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[524, 512, 499, 501, 479, 526, 499, 511, 486, 479, 492, 513, 539, 517, 508, 519, 519, 519, 555, 504, 519, 552, 557, 527, 515, 514, 512, 512, 543, 510, 515, 503, 504, 520, 524, 549, 497, 542, 506, 498, 518, 535, 494, 516, 485, 482, 507, 527, 470, 562, 512, 511, 483, 514, 510, 514, 492, 503, 526, 470, 466, 485, 544, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[495, 520, 507, 460, 481, 511, 531, 534, 570, 482, 496, 536, 546, 503, 534, 530, 493, 466, 519, 514, 508, 546, 536, 507, 527, 499, 502, 532, 502, 499, 500, 486, 489, 487, 530, 489, 539, 483, 522, 525, 487, 532, 508, 530, 539, 508, 504, 528, 511, 535, 514, 519, 496, 508, 515, 501, 531, 521, 515, 500, 502, 510, 497, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[545, 531, 538, 531, 497, 494, 498, 520, 527, 506, 489, 463, 497, 502, 519, 510, 543, 528, 511, 491, 496, 484, 525, 522, 522, 504, 519, 492, 513, 515, 475, 502, 513, 515, 512, 533, 545, 521, 510, 519, 481, 513, 503, 497, 521, 554, 553, 567, 511, 501, 506, 515, 503, 508, 502, 474, 533, 487, 499, 495, 505, 511, 546, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[524, 497, 519, 525, 505, 516, 543, 514, 539, 495, 503, 540, 517, 484, 542, 533, 508, 554, 518, 511, 537, 529, 527, 516, 542, 518, 530, 499, 547, 516, 473, 509, 517, 497, 511, 515, 491, 482, 523, 486, 534, 517, 512, 527, 515, 479, 463, 500, 482, 492, 548, 477, 506, 494, 483, 506, 530, 493, 522, 484, 534, 502, 523, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[532, 519, 509, 498, 489, 520, 510, 473, 482, 509, 512, 550, 501, 521, 491, 530, 502, 546, 509, 518, 517, 539, 527, 539, 531, 528, 541, 495, 556, 521, 487, 490, 514, 529, 514, 527, 505, 492, 494, 497, 528, 524, 501, 496, 495, 530, 522, 515, 491, 510, 515, 486, 523, 512, 509, 508, 512, 499, 482, 520, 518, 491, 505, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[511, 508, 482, 543, 523, 524, 515, 534, 502, 503, 526, 515, 496, 496, 505, 506, 516, 533, 491, 507, 526, 523, 525, 520, 460, 519, 511, 488, 532, 502, 505, 522, 564, 524, 502, 535, 504, 504, 464, 525, 517, 504, 533, 502, 499, 562, 529, 453, 541, 496, 519, 485, 526, 509, 487, 512, 507, 488, 499, 543, 515, 517, 506, 528] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[523, 470, 511, 538, 556, 507, 541, 510, 463, 504, 499, 534, 523, 523, 494, 487, 526, 486, 501, 533, 486, 504, 474, 528, 536, 516, 498, 551, 521, 508, 515, 486, 505, 502, 505, 495, 494, 520, 462, 529, 505, 535, 535, 516, 533, 512, 533, 515, 517, 494, 553, 503, 506, 527, 524, 520, 493, 501, 506, 472, 528, 514, 535, 527] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[502, 509, 539, 484, 504, 523, 490, 543, 512, 477, 537, 510, 485, 501, 487, 523, 529, 510, 511, 538, 509, 527, 515, 532, 489, 520, 516, 500, 507, 551, 504, 490, 509, 505, 509, 493, 461, 530, 483, 513, 512, 522, 538, 501, 488, 509, 561, 524, 536, 528, 514, 517, 539, 472, 540, 485, 506, 543, 507, 499, 529, 503, 511, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[517, 512, 499, 501, 479, 526, 499, 511, 0, 0, 0, 0, 0, 0, 0, 0, 486, 479, 492, 513, 539, 517, 508, 519, 14, 4, 0, 0, 0, 0, 0, 0, 519, 519, 555, 504, 519, 498, 557, 527, 0, 0, 0, 0, 0, 0, 0, 0, 515, 514, 512, 512, 543, 510, 515, 503, 3, 0, 0, 0, 0, 0, 0, 0, 504, 520, 524, 549, 497, 542, 506, 498, 13, 8, 0, 0, 0, 0, 0, 0, 518, 508, 494, 516, 485, 482, 507, 527, 0, 0, 0, 0, 0, 0, 0, 0, 470, 562, 512, 511, 483, 514, 510, 514, 23, 0, 0, 0, 0, 0, 0, 0, 492, 503, 526, 470, 466, 485, 544, 522, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4044, 4071, 4198, 4127, 4161, 4037, 4099, 4031] avg=4096.0000 max=4198 max_rank=2 min=4031 min_rank=7 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16328 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[538, 531, 538, 531, 497, 494, 498, 520, 0, 0, 0, 0, 0, 0, 0, 0, 527, 506, 489, 463, 497, 502, 519, 510, 13, 4, 0, 0, 0, 0, 0, 0, 543, 528, 511, 491, 496, 437, 525, 522, 0, 0, 0, 0, 0, 0, 0, 0, 522, 504, 519, 492, 513, 515, 475, 502, 3, 0, 0, 0, 0, 0, 0, 0, 513, 515, 512, 533, 545, 521, 510, 519, 12, 7, 0, 0, 0, 0, 0, 0, 481, 488, 503, 497, 521, 554, 553, 567, 0, 0, 0, 0, 0, 0, 0, 0, 511, 501, 506, 515, 503, 508, 502, 474, 20, 0, 0, 0, 0, 0, 0, 0, 533, 487, 499, 495, 505, 511, 546, 506, 20, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4147, 4030, 4053, 4045, 4187, 4164, 4040, 4102] avg=4096.0000 max=4187 max_rank=4 min=4030 min_rank=1 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16493 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[489, 520, 507, 460, 481, 511, 531, 534, 0, 0, 0, 0, 0, 0, 0, 0, 570, 482, 496, 536, 546, 503, 534, 530, 14, 4, 0, 0, 0, 0, 0, 0, 493, 466, 519, 514, 508, 493, 536, 507, 0, 0, 0, 0, 0, 0, 0, 0, 527, 499, 502, 532, 502, 499, 500, 486, 2, 0, 0, 0, 0, 0, 0, 0, 489, 487, 530, 489, 539, 483, 522, 525, 13, 8, 0, 0, 0, 0, 0, 0, 487, 505, 508, 530, 539, 508, 504, 528, 0, 0, 0, 0, 0, 0, 0, 0, 511, 535, 514, 519, 496, 508, 515, 501, 23, 0, 0, 0, 0, 0, 0, 0, 531, 521, 515, 500, 502, 510, 497, 521, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4033, 4215, 4036, 4049, 4085, 4109, 4122, 4119] avg=4096.0000 max=4215 max_rank=1 min=4033 min_rank=0 imbalance_ratio=1.029053 max_load_reduction=28553 max_load_reduction_pct=0.871368 moved_to_echo=16435 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[500, 508, 482, 543, 523, 524, 515, 534, 0, 0, 0, 0, 0, 0, 0, 0, 502, 503, 526, 515, 496, 496, 505, 506, 17, 6, 0, 0, 0, 0, 0, 0, 516, 533, 491, 507, 526, 460, 525, 520, 0, 0, 0, 0, 0, 0, 0, 0, 460, 519, 511, 488, 532, 502, 505, 522, 5, 0, 0, 0, 0, 0, 0, 0, 564, 524, 502, 535, 504, 504, 464, 525, 16, 11, 0, 0, 0, 0, 0, 0, 517, 471, 533, 502, 499, 562, 529, 453, 0, 0, 0, 0, 0, 0, 0, 0, 541, 496, 519, 485, 526, 509, 487, 512, 27, 0, 0, 0, 0, 0, 0, 0, 507, 488, 499, 543, 515, 517, 506, 528, 25, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4129, 4072, 4078, 4044, 4149, 4066, 4102, 4128] avg=4096.0000 max=4149 max_rank=4 min=4044 min_rank=3 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16445 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[525, 519, 509, 498, 489, 520, 510, 473, 0, 0, 0, 0, 0, 0, 0, 0, 482, 509, 512, 550, 501, 521, 491, 530, 14, 4, 0, 0, 0, 0, 0, 0, 502, 546, 509, 518, 517, 487, 527, 539, 0, 0, 0, 0, 0, 0, 0, 0, 531, 528, 541, 495, 556, 521, 487, 490, 3, 0, 0, 0, 0, 0, 0, 0, 514, 529, 514, 527, 505, 492, 494, 497, 12, 7, 0, 0, 0, 0, 0, 0, 528, 498, 501, 496, 495, 530, 522, 515, 0, 0, 0, 0, 0, 0, 0, 0, 491, 510, 515, 486, 523, 512, 509, 508, 23, 0, 0, 0, 0, 0, 0, 0, 512, 499, 482, 520, 518, 491, 505, 512, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4043, 4114, 4145, 4152, 4091, 4085, 4077, 4061] avg=4096.0000 max=4152 max_rank=3 min=4043 min_rank=0 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16314 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[517, 497, 519, 525, 505, 516, 543, 514, 0, 0, 0, 0, 0, 0, 0, 0, 539, 495, 503, 540, 517, 484, 542, 533, 14, 4, 0, 0, 0, 0, 0, 0, 508, 554, 518, 511, 537, 478, 527, 516, 0, 0, 0, 0, 0, 0, 0, 0, 542, 518, 530, 499, 547, 516, 473, 509, 3, 0, 0, 0, 0, 0, 0, 0, 517, 497, 511, 515, 491, 482, 523, 486, 12, 7, 0, 0, 0, 0, 0, 0, 534, 491, 512, 527, 515, 479, 463, 500, 0, 0, 0, 0, 0, 0, 0, 0, 482, 492, 548, 477, 506, 494, 483, 506, 22, 0, 0, 0, 0, 0, 0, 0, 530, 493, 522, 484, 534, 502, 523, 493, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4136, 4171, 4149, 4137, 4041, 4021, 4010, 4103] avg=4096.0000 max=4171 max_rank=1 min=4010 min_rank=6 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16175 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[516, 470, 511, 538, 556, 507, 541, 510, 0, 0, 0, 0, 0, 0, 0, 0, 463, 504, 499, 534, 523, 523, 494, 487, 14, 4, 0, 0, 0, 0, 0, 0, 526, 486, 501, 533, 486, 455, 474, 528, 0, 0, 0, 0, 0, 0, 0, 0, 536, 516, 498, 551, 521, 508, 515, 486, 3, 0, 0, 0, 0, 0, 0, 0, 505, 502, 505, 495, 494, 520, 462, 529, 13, 7, 0, 0, 0, 0, 0, 0, 505, 508, 535, 516, 533, 512, 533, 515, 0, 0, 0, 0, 0, 0, 0, 0, 517, 494, 553, 503, 506, 527, 524, 520, 21, 0, 0, 0, 0, 0, 0, 0, 493, 501, 506, 472, 528, 514, 535, 527, 21, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4149, 4045, 3989, 4134, 4032, 4157, 4165, 4097] avg=4096.0000 max=4165 max_rank=6 min=3989 min_rank=2 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16451 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[495, 509, 539, 484, 504, 523, 490, 543, 0, 0, 0, 0, 0, 0, 0, 0, 512, 477, 537, 510, 485, 501, 487, 523, 14, 4, 0, 0, 0, 0, 0, 0, 529, 510, 511, 538, 509, 476, 515, 532, 0, 0, 0, 0, 0, 0, 0, 0, 489, 520, 516, 500, 507, 551, 504, 490, 3, 0, 0, 0, 0, 0, 0, 0, 509, 505, 509, 493, 461, 530, 483, 513, 12, 7, 0, 0, 0, 0, 0, 0, 512, 496, 538, 501, 488, 509, 561, 524, 0, 0, 0, 0, 0, 0, 0, 0, 536, 528, 514, 517, 539, 472, 540, 485, 22, 0, 0, 0, 0, 0, 0, 0, 506, 543, 507, 499, 529, 503, 511, 507, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4050, 4120, 4080, 4022, 4129, 4153, 4127] avg=4096.0000 max=4153 max_rank=6 min=4022 min_rank=4 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 3968, 4096, 4224, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 3840, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 3968, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 3968, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[518, 516, 525, 525, 475, 536, 519, 481, 516, 491, 552, 507, 478, 525, 527, 532, 500, 519, 520, 504, 505, 540, 508, 547, 516, 511, 526, 511, 507, 504, 490, 520, 471, 483, 544, 515, 502, 472, 531, 491, 503, 534, 509, 526, 489, 499, 545, 511, 546, 526, 515, 498, 485, 492, 509, 512, 530, 502, 480, 515, 488, 522, 536, 536] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[541, 513, 487, 493, 507, 518, 534, 527, 516, 501, 504, 483, 506, 515, 508, 516, 482, 500, 512, 549, 543, 542, 506, 529, 475, 487, 556, 531, 521, 496, 509, 516, 540, 524, 499, 461, 476, 546, 543, 539, 456, 549, 512, 505, 542, 485, 532, 489, 508, 516, 510, 502, 556, 514, 483, 503, 524, 511, 498, 519, 508, 471, 515, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[499, 499, 553, 505, 560, 479, 495, 521, 523, 549, 505, 523, 520, 534, 486, 482, 479, 498, 509, 505, 525, 519, 545, 510, 503, 517, 501, 511, 506, 533, 537, 502, 476, 496, 489, 546, 471, 465, 504, 514, 473, 472, 525, 548, 470, 552, 483, 511, 506, 487, 521, 498, 539, 556, 524, 515, 504, 493, 538, 522, 525, 522, 537, 553] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[467, 467, 535, 476, 536, 489, 511, 498, 513, 518, 537, 513, 506, 527, 524, 529, 527, 539, 539, 505, 503, 498, 480, 521, 480, 500, 498, 477, 505, 517, 509, 492, 489, 540, 495, 507, 513, 500, 482, 515, 492, 525, 512, 534, 527, 527, 515, 503, 520, 557, 508, 518, 525, 534, 491, 571, 530, 508, 508, 525, 517, 525, 493, 526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[516, 503, 502, 529, 486, 499, 520, 491, 520, 499, 514, 526, 535, 518, 523, 505, 491, 498, 528, 528, 516, 519, 520, 468, 501, 509, 540, 487, 480, 546, 511, 524, 523, 536, 515, 459, 517, 490, 496, 513, 517, 428, 496, 506, 509, 545, 496, 529, 517, 500, 539, 502, 513, 529, 523, 550, 556, 516, 505, 504, 541, 504, 520, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[512, 504, 528, 556, 502, 496, 512, 519, 530, 534, 490, 509, 502, 477, 484, 527, 523, 511, 515, 521, 528, 532, 516, 521, 514, 453, 548, 492, 453, 499, 503, 548, 558, 516, 515, 498, 526, 490, 515, 503, 500, 543, 538, 532, 542, 517, 524, 520, 545, 496, 531, 474, 513, 508, 492, 530, 511, 516, 486, 485, 511, 511, 489, 474] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[520, 505, 536, 516, 486, 535, 522, 483, 536, 519, 502, 496, 486, 511, 526, 497, 546, 453, 526, 528, 542, 497, 526, 492, 526, 516, 471, 492, 526, 551, 497, 519, 482, 510, 482, 527, 518, 553, 521, 543, 480, 529, 492, 511, 532, 522, 502, 513, 532, 500, 495, 487, 502, 502, 552, 516, 531, 500, 539, 518, 484, 486, 512, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[493, 520, 459, 491, 514, 487, 531, 532, 523, 546, 547, 460, 489, 514, 534, 561, 551, 533, 513, 530, 486, 504, 510, 516, 521, 533, 536, 536, 489, 491, 504, 516, 508, 518, 498, 518, 497, 489, 528, 524, 498, 487, 494, 517, 499, 523, 522, 508, 498, 489, 460, 516, 492, 514, 514, 488, 566, 519, 508, 506, 524, 555, 506, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[518, 516, 525, 525, 475, 536, 519, 481, 12, 6, 0, 0, 0, 0, 0, 0, 500, 491, 552, 507, 478, 525, 527, 532, 0, 0, 0, 0, 0, 0, 0, 0, 500, 519, 520, 477, 505, 540, 508, 547, 0, 0, 0, 0, 0, 0, 0, 0, 516, 511, 526, 511, 507, 504, 490, 520, 17, 4, 0, 0, 0, 0, 0, 0, 471, 483, 544, 515, 502, 472, 531, 491, 27, 1, 0, 0, 0, 0, 0, 0, 503, 534, 509, 526, 489, 499, 545, 511, 5, 0, 0, 0, 0, 0, 0, 0, 546, 526, 515, 498, 485, 492, 509, 501, 0, 0, 0, 0, 0, 0, 0, 0, 512, 502, 480, 515, 488, 522, 536, 536, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4112, 4116, 4106, 4037, 4121, 4072, 4091] avg=4096.0000 max=4121 max_rank=5 min=4037 min_rank=4 imbalance_ratio=1.006104 max_load_reduction=28647 max_load_reduction_pct=0.874237 moved_to_echo=16321 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[499, 499, 553, 505, 560, 479, 495, 521, 13, 6, 0, 0, 0, 0, 0, 0, 506, 549, 505, 523, 520, 534, 486, 482, 0, 0, 0, 0, 0, 0, 0, 0, 479, 498, 509, 478, 525, 519, 545, 510, 0, 0, 0, 0, 0, 0, 0, 0, 503, 517, 501, 511, 506, 533, 537, 502, 16, 4, 0, 0, 0, 0, 0, 0, 476, 496, 489, 546, 471, 465, 504, 514, 27, 1, 0, 0, 0, 0, 0, 0, 473, 472, 525, 548, 470, 552, 483, 511, 5, 0, 0, 0, 0, 0, 0, 0, 506, 487, 521, 498, 539, 556, 524, 504, 0, 0, 0, 0, 0, 0, 0, 0, 487, 493, 538, 522, 525, 522, 537, 553, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4130, 4105, 4063, 4130, 3989, 4039, 4135, 4177] avg=4096.0000 max=4177 max_rank=7 min=3989 min_rank=4 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16340 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[541, 513, 487, 493, 507, 518, 534, 527, 12, 6, 0, 0, 0, 0, 0, 0, 500, 501, 504, 483, 506, 515, 508, 516, 0, 0, 0, 0, 0, 0, 0, 0, 482, 500, 512, 519, 543, 542, 506, 529, 0, 0, 0, 0, 0, 0, 0, 0, 475, 487, 556, 531, 521, 496, 509, 516, 17, 4, 0, 0, 0, 0, 0, 0, 540, 524, 499, 461, 476, 546, 543, 539, 30, 1, 0, 0, 0, 0, 0, 0, 456, 549, 512, 505, 542, 485, 532, 489, 5, 0, 0, 0, 0, 0, 0, 0, 508, 516, 510, 502, 556, 514, 483, 492, 0, 0, 0, 0, 0, 0, 0, 0, 506, 511, 498, 519, 508, 471, 515, 509, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4138, 4033, 4133, 4112, 4159, 4075, 4081, 4037] avg=4096.0000 max=4159 max_rank=4 min=4033 min_rank=1 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16352 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[512, 504, 528, 556, 502, 496, 512, 519, 13, 6, 0, 0, 0, 0, 0, 0, 513, 534, 490, 509, 502, 477, 484, 527, 0, 0, 0, 0, 0, 0, 0, 0, 523, 511, 515, 493, 528, 532, 516, 521, 0, 0, 0, 0, 0, 0, 0, 0, 514, 453, 548, 492, 453, 499, 503, 548, 16, 4, 0, 0, 0, 0, 0, 0, 558, 516, 515, 498, 526, 490, 515, 503, 28, 1, 0, 0, 0, 0, 0, 0, 500, 543, 538, 532, 542, 517, 524, 520, 5, 0, 0, 0, 0, 0, 0, 0, 545, 496, 531, 474, 513, 508, 492, 519, 0, 0, 0, 0, 0, 0, 0, 0, 494, 516, 486, 485, 511, 511, 489, 474, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4148, 4036, 4139, 4030, 4150, 4221, 4078, 3966] avg=4096.0000 max=4221 max_rank=5 min=3966 min_rank=7 imbalance_ratio=1.030518 max_load_reduction=28547 max_load_reduction_pct=0.871185 moved_to_echo=16415 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[467, 467, 535, 476, 536, 489, 511, 498, 16, 11, 0, 0, 0, 0, 0, 0, 491, 518, 537, 513, 506, 527, 524, 529, 0, 0, 0, 0, 0, 0, 0, 0, 527, 539, 539, 473, 503, 498, 480, 521, 0, 0, 0, 0, 0, 0, 0, 0, 480, 500, 498, 477, 505, 517, 509, 492, 19, 6, 0, 0, 0, 0, 0, 0, 489, 540, 495, 507, 513, 500, 482, 515, 32, 8, 0, 0, 0, 0, 0, 0, 492, 525, 512, 534, 527, 527, 515, 503, 8, 0, 0, 0, 0, 0, 0, 0, 520, 557, 508, 518, 525, 534, 491, 552, 0, 0, 0, 0, 0, 0, 0, 0, 503, 508, 508, 525, 517, 525, 493, 526, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4006, 4145, 4080, 4003, 4081, 4143, 4205, 4105] avg=4096.0000 max=4205 max_rank=6 min=4003 min_rank=3 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16534 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[520, 505, 536, 516, 486, 535, 522, 483, 13, 6, 0, 0, 0, 0, 0, 0, 519, 519, 502, 496, 486, 511, 526, 497, 0, 0, 0, 0, 0, 0, 0, 0, 546, 453, 526, 500, 542, 497, 526, 492, 0, 0, 0, 0, 0, 0, 0, 0, 526, 516, 471, 492, 526, 551, 497, 519, 17, 4, 0, 0, 0, 0, 0, 0, 482, 510, 482, 527, 518, 553, 521, 543, 28, 1, 0, 0, 0, 0, 0, 0, 480, 529, 492, 511, 532, 522, 502, 513, 5, 0, 0, 0, 0, 0, 0, 0, 532, 500, 495, 487, 502, 502, 552, 505, 0, 0, 0, 0, 0, 0, 0, 0, 513, 500, 539, 518, 484, 486, 512, 511, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4122, 4056, 4082, 4119, 4165, 4086, 4075, 4063] avg=4096.0000 max=4165 max_rank=4 min=4056 min_rank=1 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[493, 520, 459, 491, 514, 487, 531, 532, 13, 6, 0, 0, 0, 0, 0, 0, 506, 546, 547, 460, 489, 514, 534, 561, 0, 0, 0, 0, 0, 0, 0, 0, 551, 533, 513, 502, 486, 504, 510, 516, 0, 0, 0, 0, 0, 0, 0, 0, 521, 533, 536, 536, 489, 491, 504, 516, 18, 4, 0, 0, 0, 0, 0, 0, 508, 518, 498, 518, 497, 489, 528, 524, 28, 1, 0, 0, 0, 0, 0, 0, 498, 487, 494, 517, 499, 523, 522, 508, 4, 0, 0, 0, 0, 0, 0, 0, 498, 489, 460, 516, 492, 514, 514, 478, 0, 0, 0, 0, 0, 0, 0, 0, 547, 519, 508, 506, 524, 555, 506, 515, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4046, 4157, 4115, 4148, 4109, 4052, 3961, 4180] avg=4096.0000 max=4180 max_rank=7 min=3961 min_rank=6 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16302 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[516, 503, 502, 529, 486, 499, 520, 491, 12, 7, 0, 0, 0, 0, 0, 0, 504, 499, 514, 526, 535, 518, 523, 505, 0, 0, 0, 0, 0, 0, 0, 0, 491, 498, 528, 500, 516, 519, 520, 468, 0, 0, 0, 0, 0, 0, 0, 0, 501, 509, 540, 487, 480, 546, 511, 524, 18, 4, 0, 0, 0, 0, 0, 0, 523, 536, 515, 459, 517, 490, 496, 513, 28, 1, 0, 0, 0, 0, 0, 0, 517, 428, 496, 506, 509, 545, 496, 529, 5, 0, 0, 0, 0, 0, 0, 0, 517, 500, 539, 502, 513, 529, 523, 538, 0, 0, 0, 0, 0, 0, 0, 0, 537, 516, 505, 504, 541, 504, 520, 522, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4065, 4124, 4040, 4120, 4078, 4031, 4161, 4149] avg=4096.0000 max=4161 max_rank=6 min=4031 min_rank=5 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16419 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 3968, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3968, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[515, 511, 498, 482, 506, 470, 493, 543, 528, 543, 511, 511, 529, 505, 511, 533, 498, 515, 532, 473, 517, 519, 515, 523, 498, 511, 518, 466, 512, 516, 510, 549, 479, 542, 492, 495, 555, 491, 504, 528, 524, 552, 563, 532, 478, 501, 523, 498, 483, 501, 530, 502, 522, 474, 470, 509, 502, 525, 494, 533, 510, 532, 533, 530] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[488, 507, 498, 488, 501, 546, 536, 525, 490, 508, 490, 522, 545, 472, 538, 502, 551, 531, 495, 558, 522, 446, 484, 559, 500, 494, 519, 521, 478, 488, 472, 472, 486, 536, 496, 523, 520, 533, 502, 544, 538, 502, 534, 496, 545, 504, 513, 518, 512, 539, 517, 551, 512, 524, 519, 503, 518, 503, 490, 469, 544, 481, 540, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[511, 479, 518, 506, 538, 480, 506, 513, 536, 514, 509, 546, 537, 516, 515, 506, 502, 550, 501, 567, 511, 511, 517, 476, 536, 491, 496, 545, 509, 539, 472, 541, 516, 542, 498, 509, 541, 464, 513, 553, 509, 490, 495, 528, 500, 496, 514, 511, 507, 497, 496, 502, 467, 495, 487, 528, 531, 537, 542, 511, 472, 480, 508, 535] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[515, 517, 548, 504, 495, 502, 499, 540, 521, 522, 535, 495, 483, 520, 529, 514, 516, 490, 529, 483, 491, 502, 489, 505, 497, 519, 522, 528, 491, 518, 488, 504, 499, 517, 526, 512, 495, 531, 534, 534, 539, 479, 510, 498, 500, 522, 456, 499, 524, 512, 524, 498, 503, 507, 513, 513, 529, 496, 492, 540, 540, 561, 497, 557] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[521, 526, 500, 494, 485, 493, 536, 509, 476, 533, 542, 490, 506, 552, 505, 550, 485, 508, 491, 514, 556, 505, 493, 524, 487, 520, 504, 510, 496, 532, 457, 496, 475, 483, 515, 496, 555, 525, 511, 532, 494, 510, 554, 495, 537, 470, 463, 501, 537, 527, 542, 506, 513, 555, 504, 525, 525, 525, 487, 534, 514, 527, 510, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[503, 535, 532, 540, 489, 542, 488, 551, 539, 504, 546, 475, 474, 520, 549, 493, 538, 463, 515, 536, 513, 532, 509, 518, 487, 477, 497, 530, 488, 547, 510, 514, 526, 514, 472, 522, 514, 513, 517, 495, 496, 508, 553, 457, 522, 520, 519, 498, 526, 536, 551, 487, 484, 462, 505, 541, 533, 533, 461, 506, 517, 519, 504, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[547, 542, 478, 506, 537, 537, 505, 540, 506, 556, 527, 512, 522, 500, 504, 491, 541, 500, 528, 498, 494, 496, 494, 535, 538, 546, 509, 507, 555, 489, 495, 517, 466, 489, 493, 534, 515, 526, 525, 486, 535, 513, 521, 467, 528, 517, 505, 480, 531, 543, 509, 514, 514, 516, 500, 509, 493, 530, 467, 493, 567, 493, 447, 490] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[526, 499, 496, 485, 538, 494, 544, 490, 506, 510, 544, 519, 473, 461, 509, 541, 462, 493, 541, 488, 520, 569, 521, 492, 509, 473, 537, 561, 504, 475, 506, 543, 550, 514, 525, 526, 476, 524, 549, 496, 515, 522, 522, 511, 517, 511, 503, 526, 525, 486, 524, 504, 511, 503, 540, 497, 536, 488, 481, 492, 479, 545, 513, 528] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[515, 511, 498, 482, 506, 470, 493, 541, 0, 0, 0, 0, 0, 0, 0, 0, 528, 543, 487, 511, 529, 505, 511, 533, 0, 0, 0, 0, 0, 0, 0, 0, 498, 515, 532, 473, 517, 519, 515, 523, 2, 0, 0, 0, 0, 0, 0, 0, 498, 511, 518, 466, 512, 516, 510, 549, 24, 3, 0, 0, 0, 0, 0, 0, 479, 542, 492, 495, 543, 491, 504, 528, 0, 0, 0, 0, 0, 0, 0, 0, 524, 552, 563, 532, 478, 501, 523, 498, 9, 4, 0, 0, 0, 0, 0, 0, 483, 501, 530, 502, 522, 474, 470, 509, 0, 0, 0, 0, 0, 0, 0, 0, 502, 525, 494, 533, 510, 532, 533, 526, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4016, 4147, 4094, 4107, 4074, 4184, 3991, 4155] avg=4096.0000 max=4184 max_rank=5 min=3991 min_rank=6 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16404 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[515, 517, 548, 504, 495, 502, 499, 538, 0, 0, 0, 0, 0, 0, 0, 0, 521, 522, 510, 495, 483, 520, 529, 514, 0, 0, 0, 0, 0, 0, 0, 0, 516, 490, 529, 483, 491, 502, 489, 505, 2, 0, 0, 0, 0, 0, 0, 0, 497, 519, 522, 528, 491, 518, 488, 504, 25, 3, 0, 0, 0, 0, 0, 0, 499, 517, 526, 512, 484, 531, 534, 534, 0, 0, 0, 0, 0, 0, 0, 0, 539, 479, 510, 498, 500, 522, 456, 499, 8, 5, 0, 0, 0, 0, 0, 0, 524, 512, 524, 498, 503, 507, 513, 513, 0, 0, 0, 0, 0, 0, 0, 0, 529, 496, 492, 540, 540, 561, 497, 552, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4094, 4007, 4095, 4137, 4016, 4094, 4207] avg=4096.0000 max=4207 max_rank=7 min=4007 min_rank=2 imbalance_ratio=1.027100 max_load_reduction=28561 max_load_reduction_pct=0.871613 moved_to_echo=16454 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[488, 507, 498, 488, 501, 546, 536, 523, 0, 0, 0, 0, 0, 0, 0, 0, 490, 508, 467, 522, 545, 472, 538, 502, 0, 0, 0, 0, 0, 0, 0, 0, 551, 531, 495, 558, 522, 446, 484, 559, 2, 0, 0, 0, 0, 0, 0, 0, 500, 494, 519, 521, 478, 488, 472, 472, 23, 3, 0, 0, 0, 0, 0, 0, 486, 536, 496, 523, 509, 533, 502, 544, 0, 0, 0, 0, 0, 0, 0, 0, 538, 502, 534, 496, 545, 504, 513, 518, 8, 4, 0, 0, 0, 0, 0, 0, 512, 539, 517, 551, 512, 524, 519, 503, 0, 0, 0, 0, 0, 0, 0, 0, 518, 503, 490, 469, 544, 481, 540, 506, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4044, 4148, 3970, 4129, 4162, 4177, 4051] avg=4096.0000 max=4177 max_rank=6 min=3970 min_rank=3 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16519 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[521, 526, 500, 494, 485, 493, 536, 505, 0, 0, 0, 0, 0, 0, 0, 0, 476, 533, 512, 490, 506, 552, 505, 550, 0, 0, 0, 0, 0, 0, 0, 0, 485, 508, 491, 514, 556, 505, 493, 524, 4, 0, 0, 0, 0, 0, 0, 0, 487, 520, 504, 510, 496, 532, 457, 496, 30, 8, 0, 0, 0, 0, 0, 0, 475, 483, 515, 496, 533, 525, 511, 532, 0, 0, 0, 0, 0, 0, 0, 0, 494, 510, 554, 495, 537, 470, 463, 501, 14, 10, 0, 0, 0, 0, 0, 0, 537, 527, 542, 506, 513, 555, 504, 525, 0, 0, 0, 0, 0, 0, 0, 0, 525, 525, 487, 534, 514, 527, 510, 515, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4124, 4080, 4040, 4070, 4048, 4209, 4137] avg=4096.0000 max=4209 max_rank=6 min=4040 min_rank=3 imbalance_ratio=1.027588 max_load_reduction=28559 max_load_reduction_pct=0.871552 moved_to_echo=16464 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[511, 479, 518, 506, 538, 480, 506, 511, 0, 0, 0, 0, 0, 0, 0, 0, 536, 514, 485, 546, 537, 516, 515, 506, 0, 0, 0, 0, 0, 0, 0, 0, 502, 550, 501, 567, 511, 511, 517, 476, 2, 0, 0, 0, 0, 0, 0, 0, 536, 491, 496, 545, 509, 539, 472, 541, 24, 3, 0, 0, 0, 0, 0, 0, 516, 542, 498, 509, 529, 464, 513, 553, 0, 0, 0, 0, 0, 0, 0, 0, 509, 490, 495, 528, 500, 496, 514, 511, 9, 4, 0, 0, 0, 0, 0, 0, 507, 497, 496, 502, 467, 495, 487, 528, 0, 0, 0, 0, 0, 0, 0, 0, 531, 537, 542, 511, 472, 480, 508, 531, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4049, 4155, 4137, 4156, 4124, 4056, 3979, 4112] avg=4096.0000 max=4156 max_rank=3 min=3979 min_rank=6 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16271 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[547, 542, 478, 506, 537, 537, 505, 538, 0, 0, 0, 0, 0, 0, 0, 0, 506, 556, 502, 512, 522, 500, 504, 491, 0, 0, 0, 0, 0, 0, 0, 0, 541, 500, 528, 498, 494, 496, 494, 535, 2, 0, 0, 0, 0, 0, 0, 0, 538, 546, 509, 507, 555, 489, 495, 517, 25, 3, 0, 0, 0, 0, 0, 0, 466, 489, 493, 534, 504, 526, 525, 486, 0, 0, 0, 0, 0, 0, 0, 0, 535, 513, 521, 467, 528, 517, 505, 480, 8, 4, 0, 0, 0, 0, 0, 0, 531, 543, 509, 514, 514, 516, 500, 509, 0, 0, 0, 0, 0, 0, 0, 0, 493, 530, 467, 493, 567, 493, 447, 486, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4190, 4093, 4088, 4184, 4023, 4078, 4136, 3976] avg=4096.0000 max=4190 max_rank=0 min=3976 min_rank=7 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16213 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[503, 535, 532, 540, 489, 542, 488, 549, 0, 0, 0, 0, 0, 0, 0, 0, 539, 504, 520, 475, 474, 520, 549, 493, 0, 0, 0, 0, 0, 0, 0, 0, 538, 463, 515, 536, 513, 532, 509, 518, 2, 0, 0, 0, 0, 0, 0, 0, 487, 477, 497, 530, 488, 547, 510, 514, 26, 3, 0, 0, 0, 0, 0, 0, 526, 514, 472, 522, 503, 513, 517, 495, 0, 0, 0, 0, 0, 0, 0, 0, 496, 508, 553, 457, 522, 520, 519, 498, 8, 4, 0, 0, 0, 0, 0, 0, 526, 536, 551, 487, 484, 462, 505, 541, 0, 0, 0, 0, 0, 0, 0, 0, 533, 533, 461, 506, 517, 519, 504, 499, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4178, 4074, 4126, 4079, 4062, 4085, 4092, 4072] avg=4096.0000 max=4178 max_rank=0 min=4062 min_rank=4 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16311 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[526, 499, 496, 485, 538, 494, 544, 488, 0, 0, 0, 0, 0, 0, 0, 0, 506, 510, 518, 519, 473, 461, 509, 541, 0, 0, 0, 0, 0, 0, 0, 0, 462, 493, 541, 488, 520, 569, 521, 492, 2, 0, 0, 0, 0, 0, 0, 0, 509, 473, 537, 561, 504, 475, 506, 543, 26, 3, 0, 0, 0, 0, 0, 0, 550, 514, 525, 526, 465, 524, 549, 496, 0, 0, 0, 0, 0, 0, 0, 0, 515, 522, 522, 511, 517, 511, 503, 526, 8, 4, 0, 0, 0, 0, 0, 0, 525, 486, 524, 504, 511, 503, 540, 497, 0, 0, 0, 0, 0, 0, 0, 0, 536, 488, 481, 492, 479, 545, 513, 524, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4070, 4037, 4088, 4137, 4149, 4139, 4090, 4058] avg=4096.0000 max=4149 max_rank=4 min=4037 min_rank=1 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16436 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 3968, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 3968, 4224, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4352, 4096, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[540, 527, 540, 488, 537, 534, 499, 499, 503, 510, 505, 519, 519, 537, 508, 544, 501, 547, 520, 516, 525, 498, 546, 511, 524, 484, 494, 497, 490, 522, 524, 516, 508, 517, 536, 507, 476, 508, 481, 494, 510, 525, 498, 512, 518, 506, 507, 533, 499, 486, 488, 476, 493, 533, 558, 493, 522, 510, 518, 519, 512, 506, 496, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[504, 511, 513, 532, 499, 529, 496, 495, 510, 492, 501, 483, 535, 514, 511, 481, 520, 554, 496, 488, 541, 465, 477, 518, 535, 494, 474, 525, 566, 520, 514, 536, 506, 504, 513, 479, 489, 537, 543, 515, 592, 503, 481, 532, 507, 533, 513, 525, 487, 492, 534, 528, 517, 535, 503, 486, 522, 485, 521, 512, 498, 499, 522, 526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[509, 535, 536, 487, 517, 510, 468, 532, 517, 543, 507, 499, 469, 537, 534, 502, 505, 539, 482, 474, 514, 553, 518, 519, 505, 538, 513, 481, 496, 497, 547, 533, 524, 517, 529, 504, 511, 501, 518, 486, 501, 499, 529, 501, 497, 521, 514, 531, 555, 517, 497, 482, 508, 498, 503, 515, 505, 513, 523, 496, 518, 532, 509, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[507, 469, 494, 538, 490, 509, 509, 505, 500, 559, 501, 488, 491, 481, 501, 502, 531, 485, 487, 485, 521, 512, 509, 503, 524, 506, 512, 536, 534, 494, 525, 505, 519, 542, 526, 530, 525, 539, 507, 518, 518, 530, 525, 489, 503, 503, 481, 540, 527, 529, 519, 521, 505, 517, 480, 505, 553, 547, 495, 543, 525, 504, 497, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[512, 506, 509, 495, 483, 499, 498, 508, 500, 476, 507, 542, 526, 492, 510, 534, 508, 524, 497, 512, 500, 499, 505, 523, 503, 493, 564, 535, 529, 537, 501, 527, 530, 486, 528, 514, 522, 509, 510, 539, 521, 490, 511, 473, 539, 523, 525, 502, 486, 562, 487, 541, 514, 505, 523, 512, 509, 499, 497, 515, 511, 557, 455, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[487, 522, 545, 559, 539, 474, 517, 510, 495, 540, 548, 492, 508, 531, 511, 496, 511, 476, 527, 495, 526, 506, 477, 461, 491, 483, 525, 474, 531, 497, 506, 520, 516, 517, 515, 536, 532, 482, 509, 519, 529, 543, 492, 536, 537, 518, 494, 515, 481, 505, 540, 525, 535, 527, 535, 527, 492, 519, 479, 525, 502, 496, 502, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[511, 530, 515, 512, 496, 516, 519, 497, 513, 472, 499, 526, 501, 521, 518, 498, 572, 495, 472, 474, 498, 544, 531, 516, 518, 488, 547, 490, 503, 514, 501, 515, 522, 531, 497, 499, 480, 512, 500, 523, 529, 543, 521, 494, 518, 480, 523, 479, 537, 516, 537, 501, 473, 518, 502, 520, 501, 531, 531, 541, 525, 516, 517, 529] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[479, 533, 500, 536, 483, 522, 518, 477, 465, 494, 508, 515, 530, 528, 532, 494, 520, 507, 485, 492, 508, 528, 464, 488, 461, 494, 508, 537, 495, 533, 516, 507, 550, 530, 501, 493, 486, 547, 544, 529, 525, 545, 515, 580, 490, 530, 526, 513, 498, 497, 511, 513, 531, 497, 573, 511, 488, 486, 531, 493, 528, 484, 526, 540] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[509, 535, 536, 487, 517, 510, 468, 532, 6, 4, 0, 0, 0, 0, 0, 0, 517, 543, 507, 499, 469, 537, 534, 502, 17, 0, 0, 0, 0, 0, 0, 0, 505, 539, 482, 474, 514, 553, 518, 519, 31, 0, 0, 0, 0, 0, 0, 0, 505, 538, 513, 481, 496, 497, 547, 529, 0, 0, 0, 0, 0, 0, 0, 0, 507, 517, 529, 504, 511, 501, 518, 486, 0, 0, 0, 0, 0, 0, 0, 0, 470, 499, 529, 501, 497, 521, 514, 531, 0, 0, 0, 0, 0, 0, 0, 0, 555, 517, 497, 482, 508, 498, 497, 515, 0, 0, 0, 0, 0, 0, 0, 0, 505, 513, 523, 496, 518, 532, 509, 498, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4125, 4135, 4106, 4073, 4062, 4069, 4094] avg=4096.0000 max=4135 max_rank=2 min=4062 min_rank=5 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16298 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[507, 469, 494, 538, 490, 509, 509, 505, 6, 4, 0, 0, 0, 0, 0, 0, 500, 559, 501, 488, 491, 481, 501, 502, 17, 0, 0, 0, 0, 0, 0, 0, 531, 485, 487, 485, 521, 512, 509, 503, 32, 0, 0, 0, 0, 0, 0, 0, 524, 506, 512, 536, 534, 494, 525, 501, 0, 0, 0, 0, 0, 0, 0, 0, 502, 542, 526, 530, 525, 539, 507, 518, 0, 0, 0, 0, 0, 0, 0, 0, 486, 530, 525, 489, 503, 503, 481, 540, 0, 0, 0, 0, 0, 0, 0, 0, 527, 529, 519, 521, 505, 517, 474, 505, 0, 0, 0, 0, 0, 0, 0, 0, 553, 547, 495, 543, 525, 504, 497, 493, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4031, 4040, 4065, 4132, 4189, 4057, 4097, 4157] avg=4096.0000 max=4189 max_rank=4 min=4031 min_rank=0 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16500 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[487, 522, 545, 559, 539, 474, 517, 510, 7, 4, 0, 0, 0, 0, 0, 0, 495, 540, 548, 492, 508, 531, 511, 496, 17, 0, 0, 0, 0, 0, 0, 0, 511, 476, 527, 495, 526, 506, 477, 461, 33, 0, 0, 0, 0, 0, 0, 0, 491, 483, 525, 474, 531, 497, 506, 516, 0, 0, 0, 0, 0, 0, 0, 0, 499, 517, 515, 536, 532, 482, 509, 519, 0, 0, 0, 0, 0, 0, 0, 0, 496, 543, 492, 536, 537, 518, 494, 515, 0, 0, 0, 0, 0, 0, 0, 0, 481, 505, 540, 525, 535, 527, 528, 527, 0, 0, 0, 0, 0, 0, 0, 0, 492, 519, 479, 525, 502, 496, 502, 508, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4164, 4138, 4012, 4023, 4109, 4131, 4168, 4023] avg=4096.0000 max=4168 max_rank=6 min=4012 min_rank=2 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[504, 511, 513, 532, 499, 529, 496, 495, 6, 4, 0, 0, 0, 0, 0, 0, 510, 492, 501, 483, 535, 514, 511, 481, 17, 0, 0, 0, 0, 0, 0, 0, 520, 554, 496, 488, 541, 465, 477, 518, 37, 0, 0, 0, 0, 0, 0, 0, 535, 494, 474, 525, 566, 520, 514, 532, 0, 0, 0, 0, 0, 0, 0, 0, 489, 504, 513, 479, 489, 537, 543, 515, 0, 0, 0, 0, 0, 0, 0, 0, 555, 503, 481, 532, 507, 533, 513, 525, 0, 0, 0, 0, 0, 0, 0, 0, 487, 492, 534, 528, 517, 535, 497, 486, 0, 0, 0, 0, 0, 0, 0, 0, 522, 485, 521, 512, 498, 499, 522, 526, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4044, 4096, 4160, 4069, 4149, 4076, 4085] avg=4096.0000 max=4160 max_rank=3 min=4044 min_rank=1 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[540, 527, 540, 488, 537, 534, 499, 499, 7, 4, 0, 0, 0, 0, 0, 0, 503, 510, 505, 519, 519, 537, 508, 544, 17, 0, 0, 0, 0, 0, 0, 0, 501, 547, 520, 516, 525, 498, 546, 511, 32, 0, 0, 0, 0, 0, 0, 0, 524, 484, 494, 497, 490, 522, 524, 512, 0, 0, 0, 0, 0, 0, 0, 0, 491, 517, 536, 507, 476, 508, 481, 494, 0, 0, 0, 0, 0, 0, 0, 0, 478, 525, 498, 512, 518, 506, 507, 533, 0, 0, 0, 0, 0, 0, 0, 0, 499, 486, 488, 476, 493, 533, 551, 493, 0, 0, 0, 0, 0, 0, 0, 0, 522, 510, 518, 519, 512, 506, 496, 499, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4175, 4162, 4196, 4047, 4010, 4077, 4019, 4082] avg=4096.0000 max=4196 max_rank=2 min=4010 min_rank=4 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16188 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[512, 506, 509, 495, 483, 499, 498, 508, 7, 4, 0, 0, 0, 0, 0, 0, 500, 476, 507, 542, 526, 492, 510, 534, 18, 0, 0, 0, 0, 0, 0, 0, 508, 524, 497, 512, 500, 499, 505, 523, 32, 0, 0, 0, 0, 0, 0, 0, 503, 493, 564, 535, 529, 537, 501, 523, 0, 0, 0, 0, 0, 0, 0, 0, 512, 486, 528, 514, 522, 509, 510, 539, 0, 0, 0, 0, 0, 0, 0, 0, 489, 490, 511, 473, 539, 523, 525, 502, 0, 0, 0, 0, 0, 0, 0, 0, 486, 562, 487, 541, 514, 505, 516, 512, 0, 0, 0, 0, 0, 0, 0, 0, 509, 499, 497, 515, 511, 557, 455, 519, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4021, 4105, 4100, 4185, 4120, 4052, 4123, 4062] avg=4096.0000 max=4185 max_rank=3 min=4021 min_rank=0 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16357 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[511, 530, 515, 512, 496, 516, 519, 497, 12, 8, 7, 2, 0, 0, 0, 0, 513, 472, 499, 526, 501, 521, 518, 498, 21, 1, 0, 0, 0, 0, 0, 0, 572, 495, 472, 474, 498, 544, 531, 516, 37, 0, 0, 0, 0, 0, 0, 0, 518, 488, 547, 490, 503, 514, 501, 507, 0, 0, 0, 0, 0, 0, 0, 0, 494, 531, 497, 499, 480, 512, 500, 523, 0, 0, 0, 0, 0, 0, 0, 0, 491, 543, 521, 494, 518, 480, 523, 479, 0, 0, 0, 0, 0, 0, 0, 0, 537, 516, 537, 501, 473, 518, 490, 520, 0, 0, 0, 0, 0, 0, 0, 0, 501, 531, 531, 539, 525, 516, 517, 529, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4070, 4139, 4068, 4036, 4049, 4092, 4189] avg=4096.0000 max=4189 max_rank=7 min=4036 min_rank=4 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[479, 533, 500, 536, 483, 522, 518, 477, 7, 4, 0, 0, 0, 0, 0, 0, 465, 494, 508, 515, 530, 528, 532, 494, 18, 0, 0, 0, 0, 0, 0, 0, 520, 507, 485, 492, 508, 528, 464, 488, 33, 0, 0, 0, 0, 0, 0, 0, 461, 494, 508, 537, 495, 533, 516, 503, 0, 0, 0, 0, 0, 0, 0, 0, 532, 530, 501, 493, 486, 547, 544, 529, 0, 0, 0, 0, 0, 0, 0, 0, 492, 545, 515, 580, 490, 530, 526, 513, 0, 0, 0, 0, 0, 0, 0, 0, 498, 497, 511, 513, 531, 497, 566, 511, 0, 0, 0, 0, 0, 0, 0, 0, 488, 486, 531, 493, 528, 484, 526, 540, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4059, 4084, 4025, 4047, 4162, 4191, 4124, 4076] avg=4096.0000 max=4191 max_rank=5 min=4025 min_rank=2 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16553 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 3968, 3968, 4224, 4224, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 128, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3968, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[504, 532, 507, 478, 505, 485, 547, 534, 530, 482, 522, 485, 484, 525, 490, 514, 498, 536, 504, 514, 489, 515, 469, 549, 542, 517, 532, 541, 484, 527, 502, 522, 500, 517, 455, 500, 522, 538, 508, 486, 528, 535, 479, 530, 536, 560, 535, 533, 510, 537, 509, 487, 527, 500, 517, 479, 493, 497, 488, 504, 552, 486, 538, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[508, 499, 499, 519, 496, 469, 497, 525, 471, 485, 482, 494, 488, 510, 504, 501, 538, 524, 501, 528, 503, 562, 527, 483, 497, 512, 506, 517, 558, 530, 533, 539, 536, 518, 503, 529, 549, 523, 512, 525, 487, 525, 512, 489, 467, 508, 517, 521, 527, 535, 516, 490, 507, 492, 554, 516, 499, 505, 537, 502, 538, 511, 476, 537] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[524, 544, 525, 514, 509, 523, 500, 523, 517, 507, 509, 518, 513, 527, 490, 517, 523, 490, 564, 498, 523, 544, 478, 506, 479, 492, 534, 541, 525, 543, 538, 494, 530, 455, 508, 501, 474, 549, 531, 536, 524, 527, 471, 527, 510, 505, 476, 497, 507, 526, 520, 536, 492, 488, 492, 505, 493, 510, 508, 481, 551, 478, 490, 538] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[493, 532, 482, 509, 497, 496, 520, 544, 511, 541, 490, 491, 500, 522, 526, 514, 505, 514, 494, 528, 523, 509, 525, 520, 523, 503, 514, 506, 550, 498, 535, 492, 485, 514, 480, 520, 492, 518, 505, 502, 510, 520, 509, 539, 492, 533, 507, 515, 502, 532, 526, 497, 523, 513, 504, 525, 514, 497, 540, 500, 527, 500, 514, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[497, 525, 505, 525, 504, 504, 520, 531, 486, 543, 482, 509, 516, 507, 492, 520, 484, 477, 537, 526, 570, 508, 532, 535, 533, 537, 497, 520, 514, 486, 551, 492, 528, 515, 500, 495, 491, 483, 534, 545, 525, 523, 483, 532, 497, 531, 508, 533, 464, 515, 503, 491, 454, 538, 500, 506, 517, 545, 521, 477, 539, 499, 490, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[549, 537, 511, 499, 518, 534, 524, 505, 503, 543, 529, 496, 462, 519, 535, 533, 509, 518, 513, 522, 524, 499, 486, 527, 530, 507, 513, 502, 543, 515, 484, 489, 477, 492, 523, 529, 492, 515, 495, 493, 506, 492, 480, 476, 516, 508, 534, 504, 522, 499, 513, 561, 494, 511, 489, 484, 523, 514, 532, 536, 502, 536, 494, 548] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[538, 510, 509, 529, 522, 529, 499, 501, 502, 515, 501, 522, 533, 516, 512, 538, 512, 527, 521, 524, 490, 470, 534, 531, 522, 525, 500, 538, 510, 517, 538, 497, 502, 556, 497, 528, 492, 544, 509, 556, 548, 479, 476, 481, 537, 494, 499, 531, 531, 466, 483, 527, 513, 520, 482, 513, 530, 482, 485, 513, 481, 502, 474, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[466, 508, 521, 520, 535, 529, 543, 534, 471, 524, 514, 524, 503, 526, 492, 473, 513, 499, 490, 535, 521, 515, 491, 555, 511, 509, 532, 511, 504, 471, 532, 486, 529, 518, 494, 473, 504, 507, 513, 534, 511, 504, 536, 549, 539, 514, 531, 514, 490, 460, 475, 515, 501, 525, 541, 505, 511, 513, 512, 527, 510, 559, 482, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[504, 532, 507, 478, 505, 485, 547, 516, 0, 0, 0, 0, 0, 0, 0, 0, 530, 482, 522, 485, 484, 525, 490, 514, 25, 8, 0, 0, 0, 0, 0, 0, 498, 536, 504, 514, 489, 515, 469, 523, 0, 0, 0, 0, 0, 0, 0, 0, 542, 517, 532, 541, 484, 527, 470, 522, 0, 0, 0, 0, 0, 0, 0, 0, 500, 517, 455, 500, 522, 538, 508, 486, 10, 0, 0, 0, 0, 0, 0, 0, 528, 535, 479, 530, 536, 554, 535, 533, 0, 0, 0, 0, 0, 0, 0, 0, 510, 537, 509, 487, 527, 500, 517, 479, 32, 1, 0, 0, 0, 0, 0, 0, 493, 497, 488, 504, 552, 486, 538, 517, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4065, 4048, 4135, 4036, 4230, 4099, 4081] avg=4096.0000 max=4230 max_rank=5 min=4036 min_rank=4 imbalance_ratio=1.032715 max_load_reduction=28538 max_load_reduction_pct=0.870911 moved_to_echo=16446 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[493, 532, 482, 509, 497, 496, 520, 526, 0, 0, 0, 0, 0, 0, 0, 0, 511, 541, 490, 491, 500, 522, 526, 514, 24, 8, 0, 0, 0, 0, 0, 0, 505, 514, 494, 528, 523, 509, 525, 495, 0, 0, 0, 0, 0, 0, 0, 0, 523, 503, 514, 506, 550, 498, 501, 492, 0, 0, 0, 0, 0, 0, 0, 0, 485, 514, 480, 520, 492, 518, 505, 502, 10, 0, 0, 0, 0, 0, 0, 0, 510, 520, 509, 539, 492, 527, 507, 515, 0, 0, 0, 0, 0, 0, 0, 0, 502, 532, 526, 497, 523, 513, 504, 525, 34, 1, 0, 0, 0, 0, 0, 0, 514, 497, 540, 500, 527, 500, 514, 506, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4055, 4127, 4093, 4087, 4026, 4119, 4157, 4104] avg=4096.0000 max=4157 max_rank=6 min=4026 min_rank=4 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16406 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[497, 525, 505, 525, 504, 504, 520, 513, 0, 0, 0, 0, 0, 0, 0, 0, 486, 543, 482, 509, 516, 507, 492, 520, 25, 8, 0, 0, 0, 0, 0, 0, 484, 477, 537, 526, 570, 508, 532, 509, 0, 0, 0, 0, 0, 0, 0, 0, 533, 537, 497, 520, 514, 486, 516, 492, 0, 0, 0, 0, 0, 0, 0, 0, 528, 515, 500, 495, 491, 483, 534, 545, 10, 0, 0, 0, 0, 0, 0, 0, 525, 523, 483, 532, 497, 525, 508, 533, 0, 0, 0, 0, 0, 0, 0, 0, 464, 515, 503, 491, 454, 538, 500, 506, 35, 1, 0, 0, 0, 0, 0, 0, 517, 545, 521, 477, 539, 499, 490, 521, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4088, 4143, 4095, 4101, 4126, 4007, 4115] avg=4096.0000 max=4143 max_rank=2 min=4007 min_rank=6 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16349 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[508, 499, 499, 519, 496, 469, 497, 507, 0, 0, 0, 0, 0, 0, 0, 0, 471, 485, 482, 494, 488, 510, 504, 501, 22, 8, 0, 0, 0, 0, 0, 0, 538, 524, 501, 528, 503, 562, 527, 460, 0, 0, 0, 0, 0, 0, 0, 0, 497, 512, 506, 517, 558, 530, 499, 539, 0, 0, 0, 0, 0, 0, 0, 0, 536, 518, 503, 529, 549, 523, 512, 525, 10, 0, 0, 0, 0, 0, 0, 0, 487, 525, 512, 489, 467, 503, 517, 521, 0, 0, 0, 0, 0, 0, 0, 0, 527, 535, 516, 490, 507, 492, 554, 516, 34, 1, 0, 0, 0, 0, 0, 0, 499, 505, 537, 502, 538, 511, 476, 537, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3994, 3965, 4143, 4158, 4205, 4021, 4172, 4110] avg=4096.0000 max=4205 max_rank=4 min=3965 min_rank=1 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16508 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[524, 544, 525, 514, 509, 523, 500, 505, 0, 0, 0, 0, 0, 0, 0, 0, 517, 507, 509, 518, 513, 527, 490, 517, 23, 8, 0, 0, 0, 0, 0, 0, 523, 490, 564, 498, 523, 544, 478, 482, 0, 0, 0, 0, 0, 0, 0, 0, 479, 492, 534, 541, 525, 543, 504, 494, 0, 0, 0, 0, 0, 0, 0, 0, 530, 455, 508, 501, 474, 549, 531, 536, 10, 0, 0, 0, 0, 0, 0, 0, 524, 527, 471, 527, 510, 500, 476, 497, 0, 0, 0, 0, 0, 0, 0, 0, 507, 526, 520, 536, 492, 488, 492, 505, 34, 1, 0, 0, 0, 0, 0, 0, 493, 510, 508, 481, 551, 478, 490, 538, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4144, 4129, 4102, 4112, 4094, 4032, 4101, 4054] avg=4096.0000 max=4144 max_rank=0 min=4032 min_rank=5 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16281 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[549, 537, 511, 499, 518, 534, 524, 479, 0, 0, 0, 0, 0, 0, 0, 0, 503, 543, 529, 496, 462, 519, 535, 533, 29, 12, 0, 0, 0, 0, 0, 0, 509, 518, 513, 522, 524, 499, 486, 493, 0, 0, 0, 0, 0, 0, 0, 0, 530, 507, 513, 502, 543, 515, 447, 489, 0, 0, 0, 0, 0, 0, 0, 0, 477, 492, 523, 529, 492, 515, 495, 493, 14, 0, 0, 0, 0, 0, 0, 0, 506, 492, 480, 476, 516, 499, 534, 504, 0, 0, 0, 0, 0, 0, 0, 0, 522, 499, 513, 561, 494, 511, 489, 484, 37, 5, 0, 0, 0, 0, 0, 0, 523, 514, 532, 536, 502, 536, 494, 548, 9, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4151, 4161, 4064, 4046, 4030, 4007, 4115, 4194] avg=4096.0000 max=4194 max_rank=7 min=4007 min_rank=5 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16346 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[466, 508, 521, 520, 535, 529, 543, 516, 0, 0, 0, 0, 0, 0, 0, 0, 471, 524, 514, 524, 503, 526, 492, 473, 26, 8, 0, 0, 0, 0, 0, 0, 513, 499, 490, 535, 521, 515, 491, 528, 0, 0, 0, 0, 0, 0, 0, 0, 511, 509, 532, 511, 504, 471, 498, 486, 0, 0, 0, 0, 0, 0, 0, 0, 529, 518, 494, 473, 504, 507, 513, 534, 10, 0, 0, 0, 0, 0, 0, 0, 511, 504, 536, 549, 539, 509, 531, 514, 0, 0, 0, 0, 0, 0, 0, 0, 490, 460, 475, 515, 501, 525, 541, 505, 34, 1, 0, 0, 0, 0, 0, 0, 511, 513, 512, 527, 510, 559, 482, 514, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4138, 4061, 4092, 4022, 4082, 4193, 4047, 4133] avg=4096.0000 max=4193 max_rank=5 min=4022 min_rank=3 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16455 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[538, 510, 509, 529, 522, 529, 499, 483, 0, 0, 0, 0, 0, 0, 0, 0, 502, 515, 501, 522, 533, 516, 512, 538, 25, 8, 0, 0, 0, 0, 0, 0, 512, 527, 521, 524, 490, 470, 534, 505, 0, 0, 0, 0, 0, 0, 0, 0, 522, 525, 500, 538, 510, 517, 504, 497, 0, 0, 0, 0, 0, 0, 0, 0, 502, 556, 497, 528, 492, 544, 509, 556, 10, 0, 0, 0, 0, 0, 0, 0, 548, 479, 476, 481, 537, 489, 499, 531, 0, 0, 0, 0, 0, 0, 0, 0, 531, 466, 483, 527, 513, 520, 482, 513, 34, 1, 0, 0, 0, 0, 0, 0, 530, 482, 485, 513, 481, 502, 474, 505, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4172, 4083, 4113, 4194, 4040, 4070, 3977] avg=4096.0000 max=4194 max_rank=4 min=3977 min_rank=7 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16281 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 3968, 4096, 4096, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 3968, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 3968, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 3968, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[517, 512, 488, 531, 533, 476, 519, 536, 529, 499, 531, 522, 485, 520, 465, 510, 530, 539, 495, 516, 491, 490, 488, 480, 524, 484, 517, 555, 517, 516, 533, 532, 504, 498, 524, 501, 553, 519, 499, 492, 527, 523, 526, 528, 496, 500, 512, 517, 534, 543, 485, 508, 512, 506, 522, 534, 480, 482, 490, 484, 512, 521, 530, 526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[484, 512, 550, 544, 497, 515, 532, 499, 534, 481, 484, 481, 526, 490, 521, 517, 535, 507, 513, 523, 477, 540, 493, 474, 541, 521, 538, 533, 518, 503, 534, 523, 500, 553, 527, 483, 530, 524, 528, 529, 491, 523, 519, 501, 522, 512, 476, 498, 536, 468, 529, 516, 502, 542, 496, 498, 508, 510, 519, 519, 468, 526, 473, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[511, 507, 522, 522, 505, 530, 518, 492, 496, 489, 505, 518, 543, 504, 493, 537, 528, 519, 514, 545, 505, 489, 499, 483, 499, 537, 499, 496, 497, 510, 511, 491, 489, 506, 524, 510, 494, 512, 512, 472, 544, 546, 547, 483, 515, 560, 492, 489, 535, 531, 512, 517, 529, 477, 542, 539, 493, 491, 493, 507, 517, 527, 507, 542] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[520, 508, 540, 496, 486, 516, 498, 486, 498, 521, 527, 503, 497, 462, 518, 507, 556, 494, 468, 487, 539, 504, 485, 502, 539, 521, 495, 514, 508, 519, 542, 483, 503, 505, 540, 529, 533, 554, 497, 536, 524, 488, 498, 513, 518, 509, 496, 490, 519, 501, 515, 511, 513, 511, 479, 525, 519, 525, 471, 589, 551, 485, 553, 529] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[505, 483, 480, 507, 502, 509, 533, 519, 459, 517, 525, 520, 481, 522, 514, 497, 523, 515, 510, 509, 512, 517, 516, 521, 555, 493, 520, 501, 490, 497, 478, 525, 533, 480, 511, 513, 546, 504, 572, 511, 525, 487, 544, 531, 526, 482, 492, 543, 486, 515, 523, 525, 513, 506, 544, 532, 544, 490, 501, 542, 483, 514, 501, 494] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[496, 485, 528, 531, 521, 509, 538, 534, 531, 510, 516, 541, 497, 515, 474, 509, 514, 537, 502, 514, 496, 474, 536, 535, 495, 572, 491, 524, 524, 482, 478, 504, 547, 484, 501, 524, 501, 451, 539, 531, 522, 554, 519, 501, 498, 538, 528, 532, 537, 514, 501, 513, 512, 467, 541, 544, 462, 487, 477, 471, 498, 518, 536, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[538, 510, 539, 503, 500, 480, 497, 513, 494, 487, 504, 545, 493, 555, 531, 533, 469, 510, 512, 533, 504, 500, 500, 513, 485, 483, 492, 521, 553, 509, 501, 497, 483, 507, 465, 487, 550, 538, 537, 526, 529, 547, 517, 503, 524, 541, 483, 537, 470, 556, 525, 509, 523, 470, 536, 507, 497, 472, 537, 547, 503, 513, 511, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[528, 544, 512, 508, 466, 505, 542, 518, 480, 490, 492, 489, 535, 529, 503, 529, 506, 499, 498, 494, 572, 516, 511, 520, 503, 532, 486, 516, 532, 520, 506, 480, 543, 494, 511, 508, 520, 495, 501, 502, 535, 519, 522, 493, 528, 524, 522, 501, 536, 548, 556, 495, 483, 504, 477, 529, 533, 501, 501, 484, 503, 539, 477, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[517, 512, 488, 531, 533, 476, 517, 536, 0, 0, 0, 0, 0, 0, 0, 0, 529, 499, 531, 522, 485, 520, 465, 510, 27, 2, 0, 0, 0, 0, 0, 0, 530, 539, 495, 516, 491, 490, 488, 480, 16, 3, 2, 0, 0, 0, 0, 0, 524, 484, 517, 552, 517, 516, 533, 532, 0, 0, 0, 0, 0, 0, 0, 0, 504, 498, 524, 501, 537, 519, 499, 492, 0, 0, 0, 0, 0, 0, 0, 0, 495, 523, 526, 528, 496, 500, 512, 517, 0, 0, 0, 0, 0, 0, 0, 0, 534, 543, 485, 508, 512, 506, 522, 507, 0, 0, 0, 0, 0, 0, 0, 0, 480, 482, 490, 484, 512, 521, 530, 526, 30, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4090, 4050, 4175, 4074, 4097, 4117, 4055] avg=4096.0000 max=4175 max_rank=3 min=4050 min_rank=2 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16343 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[511, 507, 522, 522, 505, 530, 516, 492, 0, 0, 0, 0, 0, 0, 0, 0, 496, 489, 505, 518, 543, 504, 493, 537, 27, 2, 0, 0, 0, 0, 0, 0, 528, 519, 514, 545, 505, 489, 499, 483, 14, 3, 2, 0, 0, 0, 0, 0, 499, 537, 499, 493, 497, 510, 511, 491, 0, 0, 0, 0, 0, 0, 0, 0, 489, 506, 524, 510, 480, 512, 512, 472, 0, 0, 0, 0, 0, 0, 0, 0, 512, 546, 547, 483, 515, 560, 492, 489, 0, 0, 0, 0, 0, 0, 0, 0, 535, 531, 512, 517, 529, 477, 542, 512, 0, 0, 0, 0, 0, 0, 0, 0, 493, 491, 493, 507, 517, 527, 507, 542, 30, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4114, 4101, 4037, 4005, 4144, 4155, 4107] avg=4096.0000 max=4155 max_rank=6 min=4005 min_rank=4 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16411 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[484, 512, 550, 544, 497, 515, 530, 499, 0, 0, 0, 0, 0, 0, 0, 0, 534, 481, 484, 481, 526, 490, 521, 517, 25, 2, 0, 0, 0, 0, 0, 0, 535, 507, 513, 523, 477, 540, 493, 474, 15, 3, 2, 0, 0, 0, 0, 0, 541, 521, 538, 530, 518, 503, 534, 523, 0, 0, 0, 0, 0, 0, 0, 0, 500, 553, 527, 483, 515, 524, 528, 529, 0, 0, 0, 0, 0, 0, 0, 0, 462, 523, 519, 501, 522, 512, 476, 498, 0, 0, 0, 0, 0, 0, 0, 0, 536, 468, 529, 516, 502, 542, 496, 473, 0, 0, 0, 0, 0, 0, 0, 0, 508, 510, 519, 519, 468, 526, 473, 502, 27, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4131, 4061, 4082, 4208, 4159, 4013, 4062, 4052] avg=4096.0000 max=4208 max_rank=3 min=4013 min_rank=5 imbalance_ratio=1.027344 max_load_reduction=28560 max_load_reduction_pct=0.871582 moved_to_echo=16286 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[505, 483, 480, 507, 502, 509, 531, 519, 0, 0, 0, 0, 0, 0, 0, 0, 459, 517, 525, 520, 481, 522, 514, 497, 27, 2, 0, 0, 0, 0, 0, 0, 523, 515, 510, 509, 512, 517, 516, 521, 16, 3, 2, 0, 0, 0, 0, 0, 555, 493, 520, 498, 490, 497, 478, 525, 0, 0, 0, 0, 0, 0, 0, 0, 533, 480, 511, 513, 530, 504, 572, 511, 0, 0, 0, 0, 0, 0, 0, 0, 494, 487, 544, 531, 526, 482, 492, 543, 0, 0, 0, 0, 0, 0, 0, 0, 486, 515, 523, 525, 513, 506, 544, 505, 0, 0, 0, 0, 0, 0, 0, 0, 544, 490, 501, 542, 483, 514, 501, 494, 29, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4036, 4064, 4144, 4056, 4154, 4099, 4117, 4098] avg=4096.0000 max=4154 max_rank=4 min=4036 min_rank=0 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16468 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[520, 508, 540, 496, 486, 516, 496, 486, 0, 0, 0, 0, 0, 0, 0, 0, 498, 521, 527, 503, 497, 462, 518, 507, 26, 2, 0, 0, 0, 0, 0, 0, 556, 494, 468, 487, 539, 504, 485, 502, 16, 3, 2, 0, 0, 0, 0, 0, 539, 521, 495, 511, 508, 519, 542, 483, 0, 0, 0, 0, 0, 0, 0, 0, 503, 505, 540, 529, 517, 554, 497, 536, 0, 0, 0, 0, 0, 0, 0, 0, 493, 488, 498, 513, 518, 509, 496, 490, 0, 0, 0, 0, 0, 0, 0, 0, 519, 501, 515, 511, 513, 511, 479, 499, 0, 0, 0, 0, 0, 0, 0, 0, 519, 525, 471, 589, 551, 485, 553, 529, 29, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4048, 4061, 4056, 4118, 4181, 4005, 4048, 4251] avg=4096.0000 max=4251 max_rank=7 min=4005 min_rank=5 imbalance_ratio=1.037842 max_load_reduction=28517 max_load_reduction_pct=0.870270 moved_to_echo=16485 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[496, 485, 528, 531, 521, 509, 535, 534, 0, 0, 0, 0, 0, 0, 0, 0, 531, 510, 516, 541, 497, 515, 474, 509, 31, 9, 0, 0, 0, 0, 0, 0, 514, 537, 502, 514, 496, 474, 536, 535, 19, 6, 3, 1, 0, 0, 0, 0, 495, 572, 491, 518, 524, 482, 478, 504, 0, 0, 0, 0, 0, 0, 0, 0, 547, 484, 501, 524, 482, 451, 539, 531, 0, 0, 0, 0, 0, 0, 0, 0, 479, 554, 519, 501, 498, 538, 528, 532, 0, 0, 0, 0, 0, 0, 0, 0, 537, 514, 501, 513, 512, 467, 541, 512, 0, 0, 0, 0, 0, 0, 0, 0, 462, 487, 477, 471, 498, 518, 536, 507, 34, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4139, 4133, 4137, 4064, 4059, 4149, 4097, 3990] avg=4096.0000 max=4149 max_rank=5 min=3990 min_rank=7 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16295 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[538, 510, 539, 503, 500, 480, 495, 513, 0, 0, 0, 0, 0, 0, 0, 0, 494, 487, 504, 545, 493, 555, 531, 533, 25, 2, 0, 0, 0, 0, 0, 0, 469, 510, 512, 533, 504, 500, 500, 513, 16, 3, 2, 0, 0, 0, 0, 0, 485, 483, 492, 518, 553, 509, 501, 497, 0, 0, 0, 0, 0, 0, 0, 0, 483, 507, 465, 487, 534, 538, 537, 526, 0, 0, 0, 0, 0, 0, 0, 0, 497, 547, 517, 503, 524, 541, 483, 537, 0, 0, 0, 0, 0, 0, 0, 0, 470, 556, 525, 509, 523, 470, 536, 482, 0, 0, 0, 0, 0, 0, 0, 0, 497, 472, 537, 547, 503, 513, 511, 514, 30, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4169, 4062, 4038, 4077, 4149, 4071, 4124] avg=4096.0000 max=4169 max_rank=1 min=4038 min_rank=3 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16421 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[528, 544, 512, 508, 466, 505, 540, 518, 0, 0, 0, 0, 0, 0, 0, 0, 480, 490, 492, 489, 535, 529, 503, 529, 27, 2, 0, 0, 0, 0, 0, 0, 506, 499, 498, 494, 572, 516, 511, 520, 15, 3, 2, 0, 0, 0, 0, 0, 503, 532, 486, 513, 532, 520, 506, 480, 0, 0, 0, 0, 0, 0, 0, 0, 543, 494, 511, 508, 505, 495, 501, 502, 0, 0, 0, 0, 0, 0, 0, 0, 503, 519, 522, 493, 528, 524, 522, 501, 0, 0, 0, 0, 0, 0, 0, 0, 536, 548, 556, 495, 483, 504, 477, 502, 0, 0, 0, 0, 0, 0, 0, 0, 533, 501, 501, 484, 503, 539, 477, 523, 30, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4076, 4136, 4072, 4059, 4112, 4101, 4091] avg=4096.0000 max=4136 max_rank=2 min=4059 min_rank=4 imbalance_ratio=1.009766 max_load_reduction=28632 max_load_reduction_pct=0.873779 moved_to_echo=16363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 3968, 4096, 4224, 4096, 4224, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 128, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3968, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[509, 477, 487, 526, 482, 484, 528, 507, 496, 514, 545, 479, 492, 528, 509, 540, 489, 524, 481, 556, 523, 552, 507, 483, 479, 494, 506, 473, 524, 529, 537, 523, 471, 521, 520, 486, 519, 535, 558, 564, 517, 505, 501, 503, 519, 507, 515, 550, 512, 501, 537, 492, 514, 522, 485, 505, 511, 546, 532, 535, 496, 523, 470, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[482, 511, 497, 485, 508, 540, 508, 493, 519, 521, 507, 537, 518, 504, 500, 543, 517, 510, 510, 531, 509, 505, 514, 558, 522, 480, 497, 526, 481, 477, 494, 545, 473, 475, 525, 508, 504, 508, 523, 536, 516, 512, 489, 522, 531, 540, 547, 518, 478, 532, 468, 512, 535, 499, 544, 516, 526, 503, 495, 499, 505, 533, 529, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[471, 487, 517, 552, 519, 534, 536, 547, 536, 487, 542, 486, 508, 528, 529, 502, 499, 494, 494, 568, 477, 509, 515, 496, 521, 494, 525, 530, 498, 527, 535, 481, 517, 487, 501, 518, 462, 503, 497, 519, 511, 482, 500, 488, 517, 517, 530, 529, 579, 532, 529, 528, 518, 476, 484, 536, 515, 519, 496, 507, 543, 467, 503, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[510, 523, 532, 513, 528, 502, 523, 497, 515, 543, 537, 510, 534, 497, 502, 507, 513, 468, 554, 515, 507, 550, 512, 512, 529, 496, 532, 502, 483, 511, 495, 475, 547, 492, 484, 502, 488, 521, 485, 512, 537, 515, 512, 498, 533, 507, 523, 485, 520, 506, 508, 527, 534, 529, 486, 541, 485, 473, 513, 522, 522, 520, 509, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[471, 518, 493, 518, 530, 524, 527, 520, 505, 524, 523, 539, 490, 517, 477, 517, 521, 529, 542, 504, 524, 491, 516, 484, 480, 486, 525, 572, 534, 487, 506, 556, 531, 543, 525, 483, 523, 540, 489, 488, 528, 533, 476, 505, 531, 472, 491, 511, 521, 504, 488, 514, 514, 525, 428, 536, 533, 529, 493, 512, 493, 488, 539, 532] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[511, 492, 514, 551, 502, 507, 471, 515, 505, 538, 499, 513, 481, 516, 508, 526, 519, 497, 503, 564, 531, 493, 522, 561, 513, 505, 511, 535, 494, 517, 504, 529, 495, 526, 565, 509, 500, 523, 498, 498, 528, 485, 494, 478, 491, 529, 531, 518, 519, 515, 512, 483, 488, 523, 515, 520, 505, 532, 509, 475, 498, 500, 547, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[507, 514, 509, 476, 555, 532, 530, 489, 528, 550, 517, 522, 553, 513, 515, 502, 493, 509, 548, 491, 501, 495, 525, 526, 512, 486, 509, 524, 514, 538, 534, 534, 498, 486, 520, 519, 510, 522, 513, 492, 496, 497, 529, 483, 507, 489, 483, 516, 501, 469, 530, 501, 535, 546, 536, 505, 515, 510, 509, 527, 483, 508, 482, 500] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[522, 502, 519, 482, 480, 484, 528, 515, 505, 480, 491, 497, 548, 515, 525, 511, 496, 536, 545, 509, 508, 517, 531, 507, 497, 491, 495, 509, 497, 490, 485, 538, 535, 464, 499, 502, 481, 485, 500, 549, 543, 510, 504, 551, 469, 519, 510, 459, 543, 541, 516, 488, 525, 498, 559, 517, 510, 550, 542, 509, 551, 532, 522, 530] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[509, 477, 487, 526, 482, 484, 528, 507, 15, 3, 0, 0, 0, 0, 0, 0, 496, 514, 520, 479, 492, 528, 509, 540, 0, 0, 0, 0, 0, 0, 0, 0, 489, 524, 481, 528, 523, 552, 507, 483, 0, 0, 0, 0, 0, 0, 0, 0, 479, 494, 506, 473, 524, 529, 537, 523, 10, 7, 0, 0, 0, 0, 0, 0, 471, 521, 520, 486, 519, 535, 558, 564, 25, 0, 0, 0, 0, 0, 0, 0, 517, 505, 501, 503, 519, 507, 515, 550, 8, 7, 0, 0, 0, 0, 0, 0, 512, 501, 537, 492, 514, 522, 485, 490, 0, 0, 0, 0, 0, 0, 0, 0, 511, 539, 532, 535, 496, 523, 470, 513, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4018, 4078, 4087, 4082, 4199, 4132, 4053, 4119] avg=4096.0000 max=4199 max_rank=4 min=4018 min_rank=0 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16503 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[471, 487, 517, 552, 519, 534, 536, 547, 15, 3, 0, 0, 0, 0, 0, 0, 536, 487, 517, 486, 508, 528, 529, 502, 0, 0, 0, 0, 0, 0, 0, 0, 499, 494, 494, 539, 477, 509, 515, 496, 0, 0, 0, 0, 0, 0, 0, 0, 521, 494, 525, 530, 498, 527, 535, 481, 10, 7, 0, 0, 0, 0, 0, 0, 517, 487, 501, 518, 462, 503, 497, 519, 26, 0, 0, 0, 0, 0, 0, 0, 511, 482, 500, 488, 517, 517, 530, 529, 8, 6, 0, 0, 0, 0, 0, 0, 579, 532, 529, 528, 518, 476, 484, 521, 0, 0, 0, 0, 0, 0, 0, 0, 515, 513, 496, 507, 543, 467, 503, 514, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4181, 4093, 4023, 4128, 4030, 4088, 4167, 4058] avg=4096.0000 max=4181 max_rank=0 min=4023 min_rank=2 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16343 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[482, 511, 497, 485, 508, 540, 508, 493, 14, 3, 0, 0, 0, 0, 0, 0, 519, 521, 484, 537, 518, 504, 500, 543, 0, 0, 0, 0, 0, 0, 0, 0, 517, 510, 510, 504, 509, 505, 514, 558, 0, 0, 0, 0, 0, 0, 0, 0, 522, 480, 497, 526, 481, 477, 494, 545, 9, 7, 0, 0, 0, 0, 0, 0, 473, 475, 525, 508, 504, 508, 523, 536, 24, 0, 0, 0, 0, 0, 0, 0, 516, 512, 489, 522, 531, 540, 547, 518, 8, 6, 0, 0, 0, 0, 0, 0, 478, 532, 468, 512, 535, 499, 544, 501, 0, 0, 0, 0, 0, 0, 0, 0, 526, 497, 495, 499, 505, 533, 529, 518, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4041, 4126, 4127, 4038, 4076, 4189, 4069, 4102] avg=4096.0000 max=4189 max_rank=5 min=4038 min_rank=3 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16436 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[510, 523, 532, 513, 528, 502, 523, 497, 15, 3, 0, 0, 0, 0, 0, 0, 515, 543, 512, 510, 534, 497, 502, 507, 0, 0, 0, 0, 0, 0, 0, 0, 513, 468, 554, 489, 507, 550, 512, 512, 0, 0, 0, 0, 0, 0, 0, 0, 529, 496, 532, 502, 483, 511, 495, 475, 10, 8, 0, 0, 0, 0, 0, 0, 547, 492, 484, 502, 488, 521, 485, 512, 23, 0, 0, 0, 0, 0, 0, 0, 537, 515, 512, 498, 533, 507, 523, 485, 9, 6, 0, 0, 0, 0, 0, 0, 520, 506, 508, 527, 534, 529, 486, 524, 0, 0, 0, 0, 0, 0, 0, 0, 485, 467, 513, 522, 522, 520, 509, 505, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4146, 4120, 4105, 4041, 4054, 4125, 4134, 4043] avg=4096.0000 max=4146 max_rank=0 min=4041 min_rank=3 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16356 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[471, 518, 493, 518, 530, 524, 527, 520, 18, 5, 0, 0, 0, 0, 0, 0, 505, 524, 493, 539, 490, 517, 477, 517, 0, 0, 0, 0, 0, 0, 0, 0, 521, 529, 542, 472, 524, 491, 516, 484, 0, 0, 0, 0, 0, 0, 0, 0, 480, 486, 525, 572, 534, 487, 506, 556, 12, 12, 0, 0, 0, 0, 0, 0, 531, 543, 525, 483, 523, 540, 489, 488, 27, 0, 0, 0, 0, 0, 0, 0, 528, 533, 476, 505, 531, 472, 491, 511, 13, 11, 0, 0, 0, 0, 0, 0, 521, 504, 488, 514, 514, 525, 428, 511, 0, 0, 0, 0, 0, 0, 0, 0, 533, 518, 493, 512, 493, 488, 539, 532, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4124, 4062, 4079, 4170, 4149, 4071, 4005, 4108] avg=4096.0000 max=4170 max_rank=3 min=4005 min_rank=6 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16333 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[511, 492, 514, 551, 502, 507, 471, 515, 14, 3, 0, 0, 0, 0, 0, 0, 505, 538, 476, 513, 481, 516, 508, 526, 0, 0, 0, 0, 0, 0, 0, 0, 519, 497, 503, 535, 531, 493, 522, 561, 0, 0, 0, 0, 0, 0, 0, 0, 513, 505, 511, 535, 494, 517, 504, 529, 9, 7, 0, 0, 0, 0, 0, 0, 495, 526, 565, 509, 500, 523, 498, 498, 26, 0, 0, 0, 0, 0, 0, 0, 528, 485, 494, 478, 491, 529, 531, 518, 8, 7, 0, 0, 0, 0, 0, 0, 519, 515, 512, 483, 488, 523, 515, 505, 0, 0, 0, 0, 0, 0, 0, 0, 505, 525, 509, 475, 498, 500, 547, 512, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4063, 4161, 4124, 4140, 4069, 4060, 4071] avg=4096.0000 max=4161 max_rank=2 min=4060 min_rank=6 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16340 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[507, 514, 509, 476, 555, 532, 530, 489, 14, 3, 0, 0, 0, 0, 0, 0, 528, 550, 494, 522, 553, 513, 515, 502, 0, 0, 0, 0, 0, 0, 0, 0, 493, 509, 548, 466, 501, 495, 525, 526, 0, 0, 0, 0, 0, 0, 0, 0, 512, 486, 509, 524, 514, 538, 534, 534, 9, 7, 0, 0, 0, 0, 0, 0, 498, 486, 520, 519, 510, 522, 513, 492, 22, 0, 0, 0, 0, 0, 0, 0, 496, 497, 529, 483, 507, 489, 483, 516, 8, 6, 0, 0, 0, 0, 0, 0, 501, 469, 530, 501, 535, 546, 536, 490, 0, 0, 0, 0, 0, 0, 0, 0, 515, 504, 509, 527, 483, 508, 482, 500, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4129, 4177, 4063, 4167, 4082, 4014, 4108, 4028] avg=4096.0000 max=4177 max_rank=1 min=4014 min_rank=5 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16232 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[522, 502, 519, 482, 480, 484, 528, 515, 14, 3, 0, 0, 0, 0, 0, 0, 505, 480, 468, 497, 548, 515, 525, 511, 0, 0, 0, 0, 0, 0, 0, 0, 496, 536, 545, 483, 508, 517, 531, 507, 0, 0, 0, 0, 0, 0, 0, 0, 497, 491, 495, 509, 497, 490, 485, 538, 9, 7, 0, 0, 0, 0, 0, 0, 535, 464, 499, 502, 481, 485, 500, 549, 23, 0, 0, 0, 0, 0, 0, 0, 543, 510, 504, 551, 469, 519, 510, 459, 8, 7, 0, 0, 0, 0, 0, 0, 543, 541, 516, 488, 525, 498, 559, 502, 0, 0, 0, 0, 0, 0, 0, 0, 510, 543, 542, 509, 551, 532, 522, 530, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4049, 4049, 4123, 4018, 4038, 4080, 4172, 4239] avg=4096.0000 max=4239 max_rank=7 min=4018 min_rank=3 imbalance_ratio=1.034912 max_load_reduction=28529 max_load_reduction_pct=0.870636 moved_to_echo=16529 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 3968, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 3968, 4224, 4224, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True + [2026-09-21 08:26:48] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 259.8 | throughput per GPU (TFLOP/s/GPU): 124.0 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.083672E+01 | loss scale: 1.0 | grad norm: 20.752 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[507, 515, 511, 543, 554, 494, 517, 468, 547, 526, 504, 500, 543, 508, 521, 525, 511, 529, 526, 517, 553, 530, 504, 479, 553, 506, 478, 496, 534, 479, 523, 492, 478, 458, 508, 504, 521, 535, 497, 516, 475, 530, 509, 483, 491, 493, 522, 535, 554, 488, 522, 524, 505, 486, 530, 550, 495, 500, 460, 502, 526, 544, 507, 527] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[516, 501, 517, 475, 508, 499, 562, 499, 500, 508, 567, 507, 515, 479, 500, 503, 485, 526, 536, 522, 508, 507, 512, 522, 537, 534, 516, 505, 534, 512, 502, 525, 522, 542, 533, 495, 542, 455, 498, 512, 515, 488, 495, 507, 537, 485, 490, 520, 493, 523, 520, 478, 534, 513, 515, 509, 529, 504, 513, 500, 498, 544, 497, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[474, 507, 475, 541, 483, 508, 529, 515, 527, 489, 487, 522, 475, 532, 567, 500, 483, 517, 507, 482, 543, 506, 527, 538, 508, 542, 513, 533, 466, 517, 538, 528, 502, 503, 555, 532, 506, 524, 527, 503, 506, 500, 521, 528, 504, 482, 520, 496, 480, 513, 502, 524, 528, 508, 492, 528, 526, 545, 495, 519, 516, 494, 512, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[485, 513, 509, 516, 513, 501, 536, 500, 541, 517, 514, 530, 492, 547, 505, 498, 498, 525, 504, 529, 542, 532, 518, 538, 499, 552, 529, 504, 492, 532, 486, 494, 528, 505, 519, 532, 498, 475, 496, 509, 525, 518, 474, 500, 490, 449, 525, 519, 536, 495, 546, 493, 492, 519, 523, 512, 546, 527, 510, 529, 519, 484, 496, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[532, 533, 511, 525, 515, 509, 493, 485, 535, 524, 506, 496, 509, 542, 540, 550, 476, 523, 480, 540, 523, 497, 497, 522, 472, 522, 515, 498, 504, 532, 519, 521, 495, 535, 514, 539, 488, 494, 487, 489, 480, 544, 525, 516, 502, 495, 525, 509, 552, 527, 540, 524, 498, 478, 534, 505, 505, 508, 510, 479, 506, 502, 496, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[550, 482, 508, 508, 528, 484, 500, 523, 490, 540, 501, 531, 526, 511, 521, 520, 546, 527, 485, 505, 531, 503, 504, 479, 479, 513, 484, 528, 526, 485, 525, 543, 543, 503, 483, 512, 540, 513, 465, 502, 528, 475, 536, 460, 473, 485, 493, 492, 526, 510, 544, 520, 512, 531, 482, 543, 526, 526, 541, 533, 548, 516, 497, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[568, 518, 538, 521, 518, 503, 483, 476, 532, 474, 501, 510, 518, 524, 545, 521, 518, 545, 510, 493, 521, 541, 544, 497, 497, 505, 531, 541, 521, 526, 487, 490, 496, 513, 467, 528, 530, 490, 523, 548, 480, 491, 514, 508, 499, 549, 493, 513, 503, 517, 493, 475, 521, 512, 511, 509, 556, 514, 494, 489, 506, 500, 488, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[507, 525, 529, 521, 518, 495, 526, 513, 485, 536, 542, 509, 545, 484, 478, 536, 530, 495, 553, 478, 515, 511, 477, 500, 526, 491, 555, 512, 523, 502, 560, 521, 520, 463, 505, 480, 539, 519, 534, 486, 498, 514, 522, 508, 471, 524, 476, 496, 514, 495, 513, 489, 516, 538, 490, 504, 507, 515, 493, 501, 585, 537, 528, 490] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[507, 515, 511, 543, 554, 494, 517, 468, 4, 0, 0, 0, 0, 0, 0, 0, 547, 526, 504, 500, 543, 508, 483, 525, 0, 0, 0, 0, 0, 0, 0, 0, 511, 529, 526, 517, 533, 530, 504, 479, 0, 0, 0, 0, 0, 0, 0, 0, 553, 489, 478, 496, 534, 479, 523, 492, 0, 0, 0, 0, 0, 0, 0, 0, 478, 458, 508, 504, 521, 535, 497, 516, 11, 8, 4, 0, 0, 0, 0, 0, 475, 530, 509, 483, 491, 493, 522, 535, 38, 20, 9, 0, 0, 0, 0, 0, 554, 488, 511, 524, 505, 486, 530, 550, 0, 0, 0, 0, 0, 0, 0, 0, 495, 500, 460, 502, 518, 544, 507, 527, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4136, 4129, 4044, 4040, 4105, 4148, 4053] avg=4096.0000 max=4148 max_rank=6 min=4040 min_rank=4 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16346 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[516, 501, 517, 475, 508, 499, 562, 499, 3, 0, 0, 0, 0, 0, 0, 0, 500, 508, 567, 507, 515, 479, 463, 503, 0, 0, 0, 0, 0, 0, 0, 0, 485, 526, 536, 522, 490, 507, 512, 522, 0, 0, 0, 0, 0, 0, 0, 0, 537, 516, 516, 505, 534, 512, 502, 525, 0, 0, 0, 0, 0, 0, 0, 0, 522, 542, 533, 495, 542, 455, 498, 512, 11, 8, 4, 0, 0, 0, 0, 0, 515, 488, 495, 507, 537, 485, 490, 520, 37, 18, 10, 0, 0, 0, 0, 0, 493, 523, 509, 478, 534, 513, 515, 509, 0, 0, 0, 0, 0, 0, 0, 0, 529, 504, 513, 500, 491, 544, 497, 523, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4042, 4100, 4147, 4122, 4102, 4074, 4101] avg=4096.0000 max=4147 max_rank=3 min=4042 min_rank=1 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[474, 507, 475, 541, 483, 508, 529, 515, 3, 0, 0, 0, 0, 0, 0, 0, 527, 489, 487, 522, 475, 532, 525, 500, 0, 0, 0, 0, 0, 0, 0, 0, 483, 517, 507, 482, 524, 506, 527, 538, 0, 0, 0, 0, 0, 0, 0, 0, 508, 524, 513, 533, 466, 517, 538, 528, 0, 0, 0, 0, 0, 0, 0, 0, 502, 503, 555, 532, 506, 524, 527, 503, 11, 8, 4, 0, 0, 0, 0, 0, 506, 500, 521, 528, 504, 482, 520, 496, 42, 19, 10, 0, 0, 0, 0, 0, 480, 513, 491, 524, 528, 508, 492, 528, 0, 0, 0, 0, 0, 0, 0, 0, 526, 545, 495, 519, 509, 494, 512, 498, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4035, 4057, 4084, 4127, 4175, 4128, 4064, 4098] avg=4096.0000 max=4175 max_rank=4 min=4035 min_rank=0 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16465 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[485, 513, 509, 516, 513, 501, 536, 500, 3, 0, 0, 0, 0, 0, 0, 0, 541, 517, 514, 530, 492, 547, 468, 498, 0, 0, 0, 0, 0, 0, 0, 0, 498, 525, 504, 529, 523, 532, 518, 538, 0, 0, 0, 0, 0, 0, 0, 0, 499, 534, 529, 504, 492, 532, 486, 494, 0, 0, 0, 0, 0, 0, 0, 0, 528, 505, 519, 532, 498, 475, 496, 509, 12, 8, 4, 0, 0, 0, 0, 0, 525, 518, 474, 500, 490, 449, 525, 519, 37, 19, 10, 0, 0, 0, 0, 0, 536, 495, 534, 493, 492, 519, 523, 512, 0, 0, 0, 0, 0, 0, 0, 0, 546, 527, 510, 529, 512, 484, 496, 488, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4107, 4167, 4070, 4086, 4066, 4104, 4092] avg=4096.0000 max=4167 max_rank=2 min=4066 min_rank=5 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16348 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[532, 533, 511, 525, 515, 509, 493, 485, 3, 0, 0, 0, 0, 0, 0, 0, 535, 524, 506, 496, 509, 542, 500, 550, 0, 0, 0, 0, 0, 0, 0, 0, 476, 523, 480, 540, 504, 497, 497, 522, 0, 0, 0, 0, 0, 0, 0, 0, 472, 505, 515, 498, 504, 532, 519, 521, 0, 0, 0, 0, 0, 0, 0, 0, 495, 535, 514, 539, 488, 494, 487, 489, 12, 8, 4, 0, 0, 0, 0, 0, 480, 544, 525, 516, 502, 495, 525, 509, 40, 19, 9, 0, 0, 0, 0, 0, 552, 527, 528, 524, 498, 478, 534, 505, 0, 0, 0, 0, 0, 0, 0, 0, 505, 508, 510, 479, 499, 502, 496, 521, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4162, 4039, 4066, 4065, 4164, 4146, 4020] avg=4096.0000 max=4164 max_rank=5 min=4020 min_rank=7 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16395 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[550, 482, 508, 508, 528, 484, 500, 523, 9, 0, 0, 0, 0, 0, 0, 0, 490, 540, 501, 531, 526, 511, 480, 520, 0, 0, 0, 0, 0, 0, 0, 0, 546, 527, 485, 505, 508, 503, 504, 479, 0, 0, 0, 0, 0, 0, 0, 0, 479, 489, 484, 528, 526, 485, 525, 543, 0, 0, 0, 0, 0, 0, 0, 0, 543, 503, 483, 512, 540, 513, 465, 502, 15, 12, 7, 0, 0, 0, 0, 0, 528, 475, 536, 460, 473, 485, 493, 492, 41, 23, 12, 0, 0, 0, 0, 0, 526, 510, 529, 520, 512, 531, 482, 543, 0, 0, 0, 0, 0, 0, 0, 0, 526, 526, 541, 533, 532, 516, 497, 524, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4099, 4057, 4059, 4095, 4018, 4153, 4195] avg=4096.0000 max=4195 max_rank=7 min=4018 min_rank=5 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16461 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[568, 518, 538, 521, 518, 503, 483, 476, 3, 0, 0, 0, 0, 0, 0, 0, 532, 474, 501, 510, 518, 524, 505, 521, 0, 0, 0, 0, 0, 0, 0, 0, 518, 545, 510, 493, 503, 541, 544, 497, 0, 0, 0, 0, 0, 0, 0, 0, 497, 488, 531, 541, 521, 526, 487, 490, 0, 0, 0, 0, 0, 0, 0, 0, 496, 513, 467, 528, 530, 490, 523, 548, 10, 8, 4, 0, 0, 0, 0, 0, 480, 491, 514, 508, 499, 549, 493, 513, 40, 18, 9, 0, 0, 0, 0, 0, 503, 517, 483, 475, 521, 512, 511, 509, 0, 0, 0, 0, 0, 0, 0, 0, 556, 514, 494, 489, 499, 500, 488, 521, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4085, 4151, 4081, 4117, 4114, 4031, 4061] avg=4096.0000 max=4151 max_rank=2 min=4031 min_rank=6 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16323 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[507, 525, 529, 521, 518, 495, 526, 513, 4, 0, 0, 0, 0, 0, 0, 0, 485, 536, 542, 509, 545, 484, 443, 536, 0, 0, 0, 0, 0, 0, 0, 0, 530, 495, 553, 478, 497, 511, 477, 500, 0, 0, 0, 0, 0, 0, 0, 0, 526, 475, 555, 512, 523, 502, 560, 521, 0, 0, 0, 0, 0, 0, 0, 0, 520, 463, 505, 480, 539, 519, 534, 486, 11, 7, 4, 0, 0, 0, 0, 0, 498, 514, 522, 508, 471, 524, 476, 496, 35, 18, 9, 0, 0, 0, 0, 0, 514, 495, 502, 489, 516, 538, 490, 504, 0, 0, 0, 0, 0, 0, 0, 0, 507, 515, 493, 501, 577, 537, 528, 490, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4138, 4080, 4041, 4174, 4068, 4071, 4048, 4148] avg=4096.0000 max=4174 max_rank=3 min=4041 min_rank=2 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16335 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 3968, 3968, 4096, 4096, 384, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4224, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[489, 503, 500, 519, 535, 525, 508, 530, 523, 497, 510, 513, 507, 497, 522, 513, 509, 529, 508, 564, 535, 510, 504, 498, 497, 507, 519, 525, 497, 532, 508, 511, 468, 504, 539, 501, 499, 470, 497, 519, 492, 500, 536, 508, 481, 497, 527, 500, 516, 493, 488, 506, 508, 528, 494, 522, 533, 493, 551, 530, 515, 541, 538, 530] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[526, 514, 528, 506, 503, 485, 532, 494, 507, 495, 501, 507, 490, 539, 512, 499, 539, 546, 512, 501, 491, 516, 536, 502, 506, 525, 498, 535, 512, 470, 505, 502, 526, 521, 527, 505, 491, 522, 512, 520, 514, 495, 498, 498, 526, 471, 508, 478, 500, 560, 541, 542, 509, 493, 470, 543, 504, 500, 491, 530, 551, 542, 502, 544] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[567, 501, 476, 523, 503, 517, 497, 526, 516, 515, 524, 532, 494, 563, 548, 498, 482, 480, 503, 540, 512, 514, 518, 508, 515, 499, 493, 505, 531, 498, 522, 490, 492, 513, 481, 520, 503, 526, 533, 518, 476, 517, 508, 493, 558, 531, 533, 539, 513, 518, 522, 506, 526, 503, 531, 509, 508, 506, 461, 526, 466, 514, 506, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[513, 535, 540, 498, 503, 504, 499, 538, 535, 503, 512, 509, 476, 520, 520, 517, 500, 528, 494, 499, 555, 491, 497, 522, 496, 486, 507, 482, 480, 514, 542, 479, 526, 518, 488, 517, 472, 539, 513, 529, 479, 499, 535, 490, 486, 530, 494, 475, 518, 513, 530, 533, 535, 512, 525, 495, 517, 565, 518, 537, 491, 539, 508, 548] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[539, 511, 532, 498, 500, 498, 536, 510, 503, 488, 504, 543, 510, 506, 514, 511, 510, 504, 538, 546, 509, 543, 533, 495, 489, 516, 499, 504, 520, 525, 527, 517, 520, 528, 539, 483, 470, 546, 494, 521, 489, 482, 490, 506, 500, 516, 541, 509, 524, 513, 518, 524, 472, 535, 482, 495, 568, 487, 526, 522, 480, 490, 491, 529] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[531, 466, 506, 522, 515, 486, 561, 505, 525, 548, 534, 537, 516, 539, 456, 538, 525, 473, 484, 524, 470, 534, 486, 517, 500, 511, 479, 500, 476, 547, 541, 504, 549, 470, 495, 515, 504, 518, 482, 497, 542, 544, 474, 480, 514, 528, 502, 555, 495, 524, 522, 517, 520, 523, 514, 558, 513, 515, 506, 523, 514, 511, 466, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[510, 494, 521, 504, 471, 527, 516, 496, 484, 496, 506, 512, 493, 531, 503, 537, 526, 479, 529, 516, 511, 486, 481, 526, 522, 506, 529, 499, 500, 524, 527, 530, 515, 494, 510, 510, 513, 486, 543, 525, 536, 472, 512, 518, 515, 512, 521, 535, 541, 477, 531, 518, 521, 513, 479, 519, 503, 529, 541, 496, 531, 526, 509, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[503, 529, 496, 539, 496, 516, 552, 522, 479, 484, 504, 530, 506, 491, 499, 484, 513, 542, 497, 516, 543, 480, 541, 513, 501, 521, 547, 528, 489, 495, 498, 502, 517, 496, 506, 504, 497, 530, 491, 531, 551, 542, 499, 533, 512, 537, 499, 481, 506, 495, 459, 454, 523, 508, 498, 498, 529, 511, 499, 545, 555, 529, 532, 545] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[531, 466, 506, 522, 515, 486, 551, 505, 0, 0, 0, 0, 0, 0, 0, 0, 525, 548, 534, 537, 516, 539, 456, 538, 4, 1, 0, 0, 0, 0, 0, 0, 525, 473, 484, 517, 470, 534, 486, 517, 0, 0, 0, 0, 0, 0, 0, 0, 500, 511, 479, 500, 476, 547, 541, 504, 25, 0, 0, 0, 0, 0, 0, 0, 549, 470, 495, 515, 504, 518, 482, 497, 19, 0, 0, 0, 0, 0, 0, 0, 542, 544, 474, 480, 514, 528, 502, 555, 10, 4, 3, 0, 0, 0, 0, 0, 495, 524, 522, 517, 520, 523, 514, 557, 0, 0, 0, 0, 0, 0, 0, 0, 513, 515, 506, 523, 514, 511, 466, 474, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4198, 4006, 4083, 4049, 4156, 4172, 4022] avg=4096.0000 max=4198 max_rank=1 min=4006 min_rank=2 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[539, 511, 532, 498, 500, 498, 527, 510, 0, 0, 0, 0, 0, 0, 0, 0, 503, 488, 504, 543, 510, 506, 514, 511, 4, 1, 0, 0, 0, 0, 0, 0, 510, 504, 538, 538, 509, 543, 533, 495, 0, 0, 0, 0, 0, 0, 0, 0, 489, 516, 499, 504, 520, 525, 527, 517, 25, 0, 0, 0, 0, 0, 0, 0, 520, 528, 539, 483, 470, 546, 494, 521, 19, 0, 0, 0, 0, 0, 0, 0, 489, 482, 490, 506, 500, 516, 541, 509, 9, 4, 4, 0, 0, 0, 0, 0, 524, 513, 518, 524, 472, 535, 482, 494, 0, 0, 0, 0, 0, 0, 0, 0, 568, 487, 526, 522, 480, 490, 491, 481, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4084, 4170, 4122, 4120, 4050, 4062, 4045] avg=4096.0000 max=4170 max_rank=2 min=4045 min_rank=7 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16277 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[513, 535, 540, 498, 503, 504, 490, 538, 0, 0, 0, 0, 0, 0, 0, 0, 535, 503, 512, 509, 476, 520, 520, 517, 3, 1, 0, 0, 0, 0, 0, 0, 500, 528, 494, 493, 555, 491, 497, 522, 0, 0, 0, 0, 0, 0, 0, 0, 496, 486, 507, 482, 480, 514, 542, 479, 26, 0, 0, 0, 0, 0, 0, 0, 526, 518, 488, 517, 472, 539, 513, 529, 20, 0, 0, 0, 0, 0, 0, 0, 479, 499, 535, 490, 486, 530, 494, 475, 9, 5, 3, 0, 0, 0, 0, 0, 518, 513, 530, 533, 535, 512, 525, 494, 0, 0, 0, 0, 0, 0, 0, 0, 517, 565, 518, 537, 491, 539, 508, 497, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4096, 4080, 4012, 4122, 4005, 4160, 4172] avg=4096.0000 max=4172 max_rank=7 min=4005 min_rank=5 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16459 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[503, 529, 496, 539, 496, 516, 542, 522, 0, 0, 0, 0, 0, 0, 0, 0, 479, 484, 504, 530, 506, 491, 499, 484, 4, 1, 0, 0, 0, 0, 0, 0, 513, 542, 497, 509, 543, 480, 541, 513, 0, 0, 0, 0, 0, 0, 0, 0, 501, 521, 547, 528, 489, 495, 498, 502, 26, 0, 0, 0, 0, 0, 0, 0, 517, 496, 506, 504, 497, 530, 491, 531, 20, 0, 0, 0, 0, 0, 0, 0, 551, 542, 499, 533, 512, 537, 499, 481, 10, 5, 3, 0, 0, 0, 0, 0, 506, 495, 459, 454, 523, 508, 498, 497, 0, 0, 0, 0, 0, 0, 0, 0, 529, 511, 499, 545, 555, 529, 532, 494, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4143, 3982, 4138, 4107, 4092, 4172, 3940, 4194] avg=4096.0000 max=4194 max_rank=7 min=3940 min_rank=6 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[526, 514, 528, 506, 503, 485, 523, 494, 0, 0, 0, 0, 0, 0, 0, 0, 507, 495, 501, 507, 490, 539, 512, 499, 3, 1, 0, 0, 0, 0, 0, 0, 539, 546, 512, 495, 491, 516, 536, 502, 0, 0, 0, 0, 0, 0, 0, 0, 506, 525, 498, 535, 512, 470, 505, 502, 26, 0, 0, 0, 0, 0, 0, 0, 526, 521, 527, 505, 491, 522, 512, 520, 20, 0, 0, 0, 0, 0, 0, 0, 514, 495, 498, 498, 526, 471, 508, 478, 9, 5, 3, 0, 0, 0, 0, 0, 500, 560, 541, 542, 509, 493, 470, 542, 0, 0, 0, 0, 0, 0, 0, 0, 504, 500, 491, 530, 551, 542, 502, 493, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4054, 4137, 4079, 4144, 4005, 4157, 4113] avg=4096.0000 max=4157 max_rank=6 min=4005 min_rank=5 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16419 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[510, 494, 521, 504, 471, 527, 504, 496, 0, 0, 0, 0, 0, 0, 0, 0, 484, 496, 506, 512, 493, 531, 503, 537, 7, 3, 0, 0, 0, 0, 0, 0, 526, 479, 529, 501, 511, 486, 481, 526, 0, 0, 0, 0, 0, 0, 0, 0, 522, 506, 529, 499, 500, 524, 527, 530, 30, 0, 0, 0, 0, 0, 0, 0, 515, 494, 510, 510, 513, 486, 543, 525, 25, 0, 0, 0, 0, 0, 0, 0, 536, 472, 512, 518, 515, 512, 521, 535, 12, 9, 8, 0, 0, 0, 0, 0, 541, 477, 531, 518, 521, 513, 479, 516, 0, 0, 0, 0, 0, 0, 0, 0, 503, 529, 541, 496, 531, 526, 509, 461, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4027, 4072, 4039, 4167, 4121, 4150, 4096, 4096] avg=4096.0000 max=4167 max_rank=3 min=4027 min_rank=0 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16463 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[489, 503, 500, 519, 535, 525, 499, 530, 0, 0, 0, 0, 0, 0, 0, 0, 523, 497, 510, 513, 507, 497, 522, 513, 4, 1, 0, 0, 0, 0, 0, 0, 509, 529, 508, 556, 535, 510, 504, 498, 0, 0, 0, 0, 0, 0, 0, 0, 497, 507, 519, 525, 497, 532, 508, 511, 25, 0, 0, 0, 0, 0, 0, 0, 468, 504, 539, 501, 499, 470, 497, 519, 19, 0, 0, 0, 0, 0, 0, 0, 492, 500, 536, 508, 481, 497, 527, 500, 9, 4, 4, 0, 0, 0, 0, 0, 516, 493, 488, 506, 508, 528, 494, 521, 0, 0, 0, 0, 0, 0, 0, 0, 533, 493, 551, 530, 515, 541, 538, 482, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4087, 4149, 4121, 4016, 4058, 4054, 4183] avg=4096.0000 max=4183 max_rank=7 min=4016 min_rank=4 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16311 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[567, 501, 476, 523, 503, 517, 488, 526, 0, 0, 0, 0, 0, 0, 0, 0, 516, 515, 524, 532, 494, 563, 548, 498, 4, 1, 0, 0, 0, 0, 0, 0, 482, 480, 503, 532, 512, 514, 518, 508, 0, 0, 0, 0, 0, 0, 0, 0, 515, 499, 493, 505, 531, 498, 522, 490, 24, 0, 0, 0, 0, 0, 0, 0, 492, 513, 481, 520, 503, 526, 533, 518, 18, 0, 0, 0, 0, 0, 0, 0, 476, 517, 508, 493, 558, 531, 533, 539, 9, 4, 4, 0, 0, 0, 0, 0, 513, 518, 522, 506, 526, 503, 531, 508, 0, 0, 0, 0, 0, 0, 0, 0, 508, 506, 461, 526, 466, 514, 506, 456, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4195, 4049, 4077, 4104, 4172, 4127, 3943] avg=4096.0000 max=4195 max_rank=1 min=3943 min_rank=7 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16346 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4224, 4096, 3840, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 3968, 4224, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[522, 507, 500, 504, 525, 538, 504, 508, 492, 549, 483, 450, 494, 530, 514, 519, 518, 505, 544, 495, 509, 458, 500, 500, 517, 505, 507, 528, 551, 528, 514, 557, 545, 500, 507, 496, 512, 479, 490, 507, 514, 535, 505, 520, 536, 520, 514, 504, 511, 520, 452, 508, 535, 503, 518, 556, 522, 502, 500, 515, 500, 535, 507, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[514, 524, 526, 526, 508, 496, 505, 499, 500, 524, 527, 492, 494, 500, 517, 523, 498, 515, 508, 473, 507, 511, 534, 530, 542, 541, 545, 522, 476, 507, 531, 531, 479, 521, 514, 508, 493, 525, 494, 514, 520, 500, 522, 503, 526, 570, 518, 490, 509, 514, 487, 510, 467, 506, 482, 529, 515, 536, 488, 502, 540, 528, 510, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[520, 530, 494, 512, 471, 552, 523, 493, 537, 524, 499, 512, 504, 551, 491, 534, 489, 537, 463, 504, 500, 477, 507, 518, 548, 461, 525, 534, 495, 492, 577, 528, 487, 489, 520, 496, 508, 536, 512, 510, 484, 474, 537, 458, 497, 526, 514, 516, 486, 531, 519, 524, 512, 539, 517, 540, 534, 493, 511, 529, 534, 531, 506, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[462, 511, 539, 523, 461, 524, 516, 522, 518, 543, 536, 488, 500, 522, 508, 514, 486, 470, 545, 492, 489, 526, 525, 512, 488, 512, 464, 489, 490, 504, 502, 511, 512, 493, 498, 539, 497, 528, 534, 519, 493, 504, 545, 527, 509, 480, 549, 482, 544, 549, 527, 517, 525, 531, 532, 500, 516, 544, 528, 526, 521, 524, 478, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[517, 515, 502, 518, 511, 482, 483, 508, 553, 513, 536, 518, 543, 510, 502, 526, 549, 532, 524, 555, 493, 471, 529, 503, 512, 499, 508, 531, 511, 494, 482, 501, 480, 543, 527, 480, 544, 543, 496, 485, 483, 510, 538, 514, 495, 513, 551, 498, 510, 513, 531, 513, 529, 496, 541, 473, 506, 515, 477, 524, 539, 495, 490, 485] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[474, 550, 546, 547, 501, 516, 521, 485, 499, 537, 502, 505, 508, 518, 522, 529, 499, 556, 488, 512, 539, 515, 488, 511, 495, 517, 479, 547, 507, 522, 521, 512, 504, 503, 499, 529, 495, 514, 485, 476, 539, 549, 492, 551, 524, 471, 523, 499, 493, 525, 506, 532, 487, 507, 537, 526, 503, 483, 502, 534, 484, 492, 502, 534] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[527, 475, 481, 525, 519, 501, 515, 533, 516, 555, 490, 508, 511, 527, 506, 499, 543, 503, 522, 488, 513, 500, 514, 513, 502, 517, 470, 509, 490, 504, 506, 524, 527, 505, 474, 543, 495, 503, 504, 571, 509, 542, 508, 545, 518, 482, 522, 529, 492, 533, 510, 510, 501, 511, 545, 504, 531, 512, 522, 464, 517, 531, 510, 492] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[509, 534, 482, 504, 515, 474, 514, 542, 506, 525, 543, 490, 503, 502, 491, 501, 536, 527, 491, 492, 507, 539, 517, 501, 496, 503, 500, 532, 494, 518, 492, 535, 521, 497, 524, 539, 517, 515, 470, 492, 550, 532, 549, 552, 490, 515, 486, 512, 516, 499, 550, 521, 489, 548, 492, 523, 501, 500, 511, 453, 527, 532, 519, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[522, 507, 500, 504, 525, 538, 504, 508, 6, 4, 0, 0, 0, 0, 0, 0, 492, 535, 483, 450, 494, 530, 514, 519, 0, 0, 0, 0, 0, 0, 0, 0, 518, 505, 544, 495, 509, 458, 500, 500, 17, 1, 0, 0, 0, 0, 0, 0, 517, 505, 507, 528, 551, 528, 514, 557, 2, 0, 0, 0, 0, 0, 0, 0, 545, 500, 507, 496, 512, 479, 490, 507, 24, 0, 0, 0, 0, 0, 0, 0, 514, 535, 480, 520, 536, 520, 514, 504, 0, 0, 0, 0, 0, 0, 0, 0, 511, 497, 452, 508, 535, 503, 518, 556, 0, 0, 0, 0, 0, 0, 0, 0, 522, 502, 500, 515, 500, 535, 507, 525, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4017, 4047, 4209, 4060, 4123, 4080, 4114] avg=4096.0000 max=4209 max_rank=3 min=4017 min_rank=1 imbalance_ratio=1.027588 max_load_reduction=28559 max_load_reduction_pct=0.871552 moved_to_echo=16377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[514, 524, 526, 526, 508, 496, 505, 499, 6, 3, 0, 0, 0, 0, 0, 0, 500, 512, 527, 492, 494, 500, 517, 523, 0, 0, 0, 0, 0, 0, 0, 0, 498, 515, 508, 473, 507, 511, 534, 530, 17, 1, 0, 0, 0, 0, 0, 0, 542, 541, 545, 522, 476, 507, 531, 531, 1, 0, 0, 0, 0, 0, 0, 0, 479, 521, 514, 508, 493, 525, 494, 514, 25, 0, 0, 0, 0, 0, 0, 0, 520, 500, 496, 503, 526, 570, 518, 490, 0, 0, 0, 0, 0, 0, 0, 0, 509, 491, 487, 510, 467, 506, 482, 529, 0, 0, 0, 0, 0, 0, 0, 0, 515, 536, 488, 502, 540, 528, 510, 502, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4065, 4094, 4196, 4073, 4123, 3981, 4129] avg=4096.0000 max=4196 max_rank=3 min=3981 min_rank=6 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16306 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[520, 530, 494, 512, 471, 552, 523, 493, 6, 3, 0, 0, 0, 0, 0, 0, 537, 512, 499, 512, 504, 551, 491, 534, 0, 0, 0, 0, 0, 0, 0, 0, 489, 537, 463, 504, 500, 477, 507, 518, 18, 1, 0, 0, 0, 0, 0, 0, 548, 461, 525, 534, 495, 492, 577, 528, 1, 0, 0, 0, 0, 0, 0, 0, 487, 489, 520, 496, 508, 536, 512, 510, 26, 0, 0, 0, 0, 0, 0, 0, 484, 474, 510, 458, 497, 526, 514, 516, 0, 0, 0, 0, 0, 0, 0, 0, 486, 507, 519, 524, 512, 539, 517, 540, 0, 0, 0, 0, 0, 0, 0, 0, 534, 493, 511, 529, 534, 531, 506, 496, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4140, 4014, 4161, 4084, 3979, 4144, 4142] avg=4096.0000 max=4161 max_rank=3 min=3979 min_rank=5 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16349 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[517, 515, 502, 518, 511, 482, 483, 508, 9, 7, 0, 0, 0, 0, 0, 0, 553, 490, 536, 518, 543, 510, 502, 526, 0, 0, 0, 0, 0, 0, 0, 0, 549, 532, 524, 555, 493, 471, 529, 503, 21, 2, 0, 0, 0, 0, 0, 0, 512, 499, 508, 531, 511, 494, 482, 501, 5, 0, 0, 0, 0, 0, 0, 0, 480, 543, 527, 480, 544, 543, 496, 485, 31, 0, 0, 0, 0, 0, 0, 0, 483, 510, 505, 514, 495, 513, 551, 498, 0, 0, 0, 0, 0, 0, 0, 0, 510, 483, 531, 513, 529, 496, 541, 473, 0, 0, 0, 0, 0, 0, 0, 0, 506, 515, 477, 524, 539, 495, 490, 485, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4052, 4178, 4179, 4043, 4129, 4069, 4076, 4042] avg=4096.0000 max=4179 max_rank=2 min=4042 min_rank=7 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16316 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[462, 511, 539, 523, 461, 524, 516, 522, 6, 4, 0, 0, 0, 0, 0, 0, 518, 529, 536, 488, 500, 522, 508, 514, 0, 0, 0, 0, 0, 0, 0, 0, 486, 470, 545, 492, 489, 526, 525, 512, 18, 1, 0, 0, 0, 0, 0, 0, 488, 512, 464, 489, 490, 504, 502, 511, 2, 0, 0, 0, 0, 0, 0, 0, 512, 493, 498, 539, 497, 528, 534, 519, 26, 0, 0, 0, 0, 0, 0, 0, 493, 504, 518, 527, 509, 480, 549, 482, 0, 0, 0, 0, 0, 0, 0, 0, 544, 525, 527, 517, 525, 531, 532, 500, 0, 0, 0, 0, 0, 0, 0, 0, 516, 544, 528, 526, 521, 524, 478, 505, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4115, 4064, 3962, 4146, 4062, 4201, 4150] avg=4096.0000 max=4201 max_rank=6 min=3962 min_rank=3 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16559 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[509, 534, 482, 504, 515, 474, 514, 542, 6, 3, 0, 0, 0, 0, 0, 0, 506, 513, 543, 490, 503, 502, 491, 501, 0, 0, 0, 0, 0, 0, 0, 0, 536, 527, 491, 492, 507, 539, 517, 501, 17, 1, 0, 0, 0, 0, 0, 0, 496, 503, 500, 532, 494, 518, 492, 535, 1, 0, 0, 0, 0, 0, 0, 0, 521, 497, 524, 539, 517, 515, 470, 492, 26, 0, 0, 0, 0, 0, 0, 0, 550, 532, 522, 552, 490, 515, 486, 512, 0, 0, 0, 0, 0, 0, 0, 0, 516, 476, 550, 521, 489, 548, 492, 523, 0, 0, 0, 0, 0, 0, 0, 0, 501, 500, 511, 453, 527, 532, 519, 511, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4049, 4128, 4071, 4101, 4159, 4115, 4062] avg=4096.0000 max=4159 max_rank=5 min=4049 min_rank=1 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16437 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[474, 550, 546, 547, 501, 516, 521, 485, 6, 4, 0, 0, 0, 0, 0, 0, 499, 523, 502, 505, 508, 518, 522, 529, 0, 0, 0, 0, 0, 0, 0, 0, 499, 556, 488, 512, 539, 515, 488, 511, 18, 1, 0, 0, 0, 0, 0, 0, 495, 517, 479, 547, 507, 522, 521, 512, 2, 0, 0, 0, 0, 0, 0, 0, 504, 503, 499, 529, 495, 514, 485, 476, 24, 0, 0, 0, 0, 0, 0, 0, 539, 549, 467, 551, 524, 471, 523, 499, 0, 0, 0, 0, 0, 0, 0, 0, 493, 501, 506, 532, 487, 507, 537, 526, 0, 0, 0, 0, 0, 0, 0, 0, 503, 483, 502, 534, 484, 492, 502, 534, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4150, 4106, 4127, 4102, 4029, 4123, 4089, 4042] avg=4096.0000 max=4150 max_rank=0 min=4029 min_rank=4 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16283 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[527, 475, 481, 525, 519, 501, 515, 533, 6, 4, 0, 0, 0, 0, 0, 0, 516, 541, 490, 508, 511, 527, 506, 499, 0, 0, 0, 0, 0, 0, 0, 0, 543, 503, 522, 488, 513, 500, 514, 513, 18, 1, 0, 0, 0, 0, 0, 0, 502, 517, 470, 509, 490, 504, 506, 524, 2, 0, 0, 0, 0, 0, 0, 0, 527, 505, 474, 543, 495, 503, 504, 571, 24, 0, 0, 0, 0, 0, 0, 0, 509, 542, 483, 545, 518, 482, 522, 529, 0, 0, 0, 0, 0, 0, 0, 0, 492, 509, 510, 510, 501, 511, 545, 504, 0, 0, 0, 0, 0, 0, 0, 0, 531, 512, 522, 464, 517, 531, 510, 492, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4098, 4115, 4024, 4146, 4130, 4082, 4087] avg=4096.0000 max=4146 max_rank=4 min=4024 min_rank=3 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16445 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 3968, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[518, 500, 548, 542, 497, 466, 514, 507, 539, 521, 535, 471, 507, 509, 505, 541, 517, 523, 493, 523, 507, 503, 531, 501, 492, 529, 531, 488, 465, 521, 483, 517, 487, 503, 547, 527, 494, 523, 511, 536, 508, 498, 521, 538, 519, 479, 501, 484, 560, 514, 509, 472, 507, 483, 537, 520, 522, 486, 498, 477, 560, 526, 525, 552] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[537, 498, 498, 517, 554, 476, 495, 504, 528, 505, 482, 557, 518, 516, 500, 525, 497, 559, 526, 509, 516, 515, 472, 510, 522, 514, 491, 509, 558, 511, 534, 513, 500, 516, 504, 479, 509, 504, 480, 518, 486, 545, 501, 547, 477, 502, 501, 498, 503, 511, 544, 532, 514, 515, 513, 509, 491, 520, 535, 521, 467, 492, 527, 541] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[513, 485, 525, 515, 519, 508, 484, 519, 514, 492, 520, 470, 507, 488, 533, 497, 504, 502, 545, 524, 480, 518, 523, 522, 582, 514, 526, 511, 501, 495, 536, 548, 514, 491, 551, 522, 517, 490, 516, 495, 510, 480, 518, 506, 528, 535, 520, 501, 472, 509, 484, 520, 516, 491, 511, 516, 516, 523, 513, 522, 514, 512, 488, 547] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[513, 552, 521, 469, 515, 508, 515, 472, 496, 507, 503, 496, 529, 526, 485, 523, 503, 472, 514, 509, 542, 495, 489, 551, 503, 539, 472, 564, 580, 519, 513, 543, 492, 486, 518, 531, 501, 518, 496, 526, 494, 538, 499, 487, 469, 558, 558, 481, 504, 520, 498, 505, 534, 534, 522, 499, 500, 530, 510, 498, 501, 523, 493, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[501, 457, 536, 513, 512, 493, 533, 495, 492, 477, 549, 524, 511, 519, 510, 517, 518, 524, 505, 487, 533, 512, 512, 481, 505, 521, 488, 540, 543, 504, 516, 515, 491, 526, 504, 488, 504, 504, 493, 547, 504, 513, 531, 507, 510, 495, 510, 516, 520, 512, 507, 499, 539, 557, 495, 516, 533, 530, 537, 491, 524, 514, 486, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[541, 485, 544, 559, 507, 524, 552, 520, 529, 490, 487, 488, 527, 489, 497, 491, 467, 518, 500, 486, 545, 515, 496, 506, 492, 514, 539, 496, 519, 490, 520, 510, 524, 539, 497, 509, 514, 507, 519, 471, 488, 538, 559, 497, 511, 547, 522, 494, 523, 505, 520, 538, 504, 532, 518, 514, 503, 495, 511, 504, 491, 514, 512, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[501, 527, 505, 533, 495, 482, 524, 504, 507, 509, 504, 489, 522, 482, 505, 553, 500, 508, 521, 550, 496, 529, 549, 506, 494, 543, 451, 509, 539, 512, 513, 494, 519, 520, 537, 532, 533, 509, 568, 500, 489, 518, 478, 508, 515, 486, 505, 510, 508, 548, 519, 529, 505, 480, 505, 497, 530, 500, 503, 510, 513, 519, 518, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[520, 476, 536, 497, 525, 492, 482, 529, 513, 504, 526, 492, 498, 490, 503, 496, 525, 509, 549, 513, 528, 508, 564, 495, 505, 511, 526, 518, 530, 496, 512, 496, 500, 510, 483, 501, 485, 525, 513, 481, 517, 511, 533, 506, 489, 523, 546, 547, 531, 504, 546, 494, 510, 489, 478, 546, 528, 524, 520, 522, 532, 515, 500, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[518, 500, 548, 542, 497, 466, 514, 507, 6, 1, 0, 0, 0, 0, 0, 0, 539, 521, 535, 471, 507, 509, 505, 541, 23, 1, 0, 0, 0, 0, 0, 0, 517, 523, 484, 523, 507, 503, 531, 501, 0, 0, 0, 0, 0, 0, 0, 0, 492, 529, 531, 488, 442, 521, 483, 517, 0, 0, 0, 0, 0, 0, 0, 0, 487, 503, 547, 527, 494, 523, 511, 536, 10, 3, 0, 0, 0, 0, 0, 0, 508, 498, 521, 538, 519, 479, 501, 484, 7, 0, 0, 0, 0, 0, 0, 0, 560, 514, 509, 472, 496, 483, 537, 520, 0, 0, 0, 0, 0, 0, 0, 0, 522, 486, 498, 477, 560, 526, 525, 544, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4152, 4089, 4003, 4141, 4055, 4091, 4138] avg=4096.0000 max=4152 max_rank=1 min=4003 min_rank=3 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16425 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[513, 485, 525, 515, 519, 508, 484, 519, 6, 1, 0, 0, 0, 0, 0, 0, 514, 492, 520, 470, 507, 488, 533, 497, 25, 1, 0, 0, 0, 0, 0, 0, 504, 502, 535, 524, 480, 518, 523, 522, 0, 0, 0, 0, 0, 0, 0, 0, 582, 514, 526, 511, 476, 495, 536, 548, 0, 0, 0, 0, 0, 0, 0, 0, 514, 491, 551, 522, 517, 490, 516, 495, 10, 4, 0, 0, 0, 0, 0, 0, 510, 480, 518, 506, 528, 535, 520, 501, 7, 0, 0, 0, 0, 0, 0, 0, 472, 509, 484, 520, 505, 491, 511, 516, 0, 0, 0, 0, 0, 0, 0, 0, 516, 523, 513, 522, 514, 512, 488, 539, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4047, 4108, 4188, 4110, 4105, 4008, 4127] avg=4096.0000 max=4188 max_rank=3 min=4008 min_rank=6 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16350 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[537, 498, 498, 517, 554, 476, 495, 504, 6, 1, 0, 0, 0, 0, 0, 0, 528, 505, 482, 557, 518, 516, 500, 525, 28, 1, 0, 0, 0, 0, 0, 0, 497, 559, 517, 509, 516, 515, 472, 510, 0, 0, 0, 0, 0, 0, 0, 0, 522, 514, 491, 509, 530, 511, 534, 513, 0, 0, 0, 0, 0, 0, 0, 0, 500, 516, 504, 479, 509, 504, 480, 518, 10, 3, 0, 0, 0, 0, 0, 0, 486, 545, 501, 547, 477, 502, 501, 498, 7, 0, 0, 0, 0, 0, 0, 0, 503, 511, 544, 532, 503, 515, 513, 509, 0, 0, 0, 0, 0, 0, 0, 0, 491, 520, 535, 521, 467, 492, 527, 533, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4160, 4095, 4124, 4023, 4064, 4130, 4086] avg=4096.0000 max=4160 max_rank=1 min=4023 min_rank=4 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16303 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[513, 552, 521, 469, 515, 508, 515, 472, 6, 1, 0, 0, 0, 0, 0, 0, 496, 507, 503, 496, 529, 526, 485, 523, 29, 2, 0, 0, 0, 0, 0, 0, 503, 472, 505, 509, 542, 495, 489, 551, 0, 0, 0, 0, 0, 0, 0, 0, 503, 539, 472, 564, 551, 519, 513, 543, 0, 0, 0, 0, 0, 0, 0, 0, 492, 486, 518, 531, 501, 518, 496, 526, 10, 3, 0, 0, 0, 0, 0, 0, 494, 538, 499, 487, 469, 558, 558, 481, 7, 0, 0, 0, 0, 0, 0, 0, 504, 520, 498, 505, 522, 534, 522, 499, 0, 0, 0, 0, 0, 0, 0, 0, 500, 530, 510, 498, 501, 523, 493, 499, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4096, 4066, 4204, 4081, 4091, 4104, 4054] avg=4096.0000 max=4204 max_rank=3 min=4054 min_rank=7 imbalance_ratio=1.026367 max_load_reduction=28564 max_load_reduction_pct=0.871704 moved_to_echo=16330 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[501, 527, 505, 533, 495, 482, 524, 504, 6, 1, 0, 0, 0, 0, 0, 0, 507, 509, 504, 489, 522, 482, 505, 553, 27, 1, 0, 0, 0, 0, 0, 0, 500, 508, 512, 550, 496, 529, 549, 506, 0, 0, 0, 0, 0, 0, 0, 0, 494, 543, 451, 509, 512, 512, 513, 494, 0, 0, 0, 0, 0, 0, 0, 0, 519, 520, 537, 532, 533, 509, 568, 500, 10, 3, 0, 0, 0, 0, 0, 0, 489, 518, 478, 508, 515, 486, 505, 510, 6, 0, 0, 0, 0, 0, 0, 0, 508, 548, 519, 529, 494, 480, 505, 497, 0, 0, 0, 0, 0, 0, 0, 0, 530, 500, 503, 510, 513, 519, 518, 494, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4099, 4150, 4028, 4231, 4015, 4080, 4087] avg=4096.0000 max=4231 max_rank=4 min=4015 min_rank=5 imbalance_ratio=1.032959 max_load_reduction=28537 max_load_reduction_pct=0.870880 moved_to_echo=16413 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[501, 457, 536, 513, 512, 493, 533, 495, 9, 6, 0, 0, 0, 0, 0, 0, 492, 477, 549, 524, 511, 519, 510, 517, 32, 8, 0, 0, 0, 0, 0, 0, 518, 524, 488, 487, 533, 512, 512, 481, 0, 0, 0, 0, 0, 0, 0, 0, 505, 521, 488, 540, 511, 504, 516, 515, 0, 0, 0, 0, 0, 0, 0, 0, 491, 526, 504, 488, 504, 504, 493, 547, 12, 8, 0, 0, 0, 0, 0, 0, 504, 513, 531, 507, 510, 495, 510, 516, 11, 0, 0, 0, 0, 0, 0, 0, 520, 512, 507, 499, 519, 557, 495, 516, 0, 0, 0, 0, 0, 0, 0, 0, 533, 530, 537, 491, 524, 514, 486, 505, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4055, 4139, 4055, 4100, 4077, 4097, 4125, 4120] avg=4096.0000 max=4139 max_rank=1 min=4055 min_rank=0 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16419 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[541, 485, 544, 559, 507, 524, 552, 520, 6, 1, 0, 0, 0, 0, 0, 0, 529, 490, 487, 488, 527, 489, 497, 491, 26, 1, 0, 0, 0, 0, 0, 0, 467, 518, 491, 486, 545, 515, 496, 506, 0, 0, 0, 0, 0, 0, 0, 0, 492, 514, 539, 496, 493, 490, 520, 510, 0, 0, 0, 0, 0, 0, 0, 0, 524, 539, 497, 509, 514, 507, 519, 471, 10, 3, 0, 0, 0, 0, 0, 0, 488, 538, 559, 497, 511, 547, 522, 494, 7, 0, 0, 0, 0, 0, 0, 0, 523, 505, 520, 538, 493, 532, 518, 514, 0, 0, 0, 0, 0, 0, 0, 0, 503, 495, 511, 504, 491, 514, 512, 497, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4239, 4025, 4024, 4054, 4093, 4163, 4143, 4027] avg=4096.0000 max=4239 max_rank=0 min=4024 min_rank=2 imbalance_ratio=1.034912 max_load_reduction=28529 max_load_reduction_pct=0.870636 moved_to_echo=16426 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[520, 476, 536, 497, 525, 492, 482, 529, 6, 1, 0, 0, 0, 0, 0, 0, 513, 504, 526, 492, 498, 490, 503, 496, 27, 1, 0, 0, 0, 0, 0, 0, 525, 509, 539, 513, 528, 508, 564, 495, 0, 0, 0, 0, 0, 0, 0, 0, 505, 511, 526, 518, 503, 496, 512, 496, 0, 0, 0, 0, 0, 0, 0, 0, 500, 510, 483, 501, 485, 525, 513, 481, 10, 4, 0, 0, 0, 0, 0, 0, 517, 511, 533, 506, 489, 523, 546, 547, 6, 0, 0, 0, 0, 0, 0, 0, 531, 504, 546, 494, 499, 489, 478, 546, 0, 0, 0, 0, 0, 0, 0, 0, 528, 524, 520, 522, 532, 515, 500, 488, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4064, 4050, 4181, 4067, 4012, 4178, 4087, 4129] avg=4096.0000 max=4181 max_rank=2 min=4012 min_rank=4 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16406 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 3968, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[500, 510, 514, 485, 513, 498, 516, 510, 498, 561, 536, 464, 509, 514, 535, 503, 526, 501, 488, 493, 547, 499, 517, 562, 475, 501, 543, 481, 534, 539, 499, 531, 527, 476, 486, 535, 504, 534, 491, 526, 488, 513, 496, 506, 500, 544, 534, 492, 517, 505, 528, 547, 537, 490, 532, 533, 501, 518, 540, 502, 463, 513, 496, 492] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[492, 502, 501, 531, 514, 526, 528, 505, 520, 482, 480, 493, 526, 536, 475, 520, 498, 507, 504, 535, 543, 482, 510, 513, 499, 548, 523, 506, 475, 487, 503, 532, 498, 541, 516, 505, 512, 552, 538, 500, 486, 482, 505, 491, 545, 535, 511, 519, 538, 552, 491, 538, 468, 489, 480, 497, 553, 512, 504, 520, 552, 499, 539, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[496, 527, 517, 461, 518, 506, 517, 507, 511, 497, 550, 505, 525, 515, 495, 516, 469, 518, 538, 494, 532, 490, 518, 504, 481, 490, 545, 531, 529, 502, 551, 510, 519, 504, 524, 514, 500, 501, 502, 549, 499, 539, 534, 558, 539, 469, 491, 532, 479, 540, 526, 494, 541, 500, 491, 512, 524, 519, 491, 516, 495, 491, 513, 497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[508, 491, 554, 521, 528, 471, 505, 511, 504, 520, 492, 535, 526, 473, 470, 515, 488, 511, 527, 511, 565, 513, 535, 482, 550, 492, 511, 533, 492, 501, 475, 515, 492, 489, 564, 513, 525, 516, 534, 500, 570, 534, 503, 500, 528, 475, 532, 484, 547, 487, 533, 490, 488, 495, 535, 511, 483, 536, 538, 539, 493, 499, 498, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[505, 485, 525, 549, 496, 505, 513, 497, 499, 486, 479, 491, 508, 515, 500, 509, 523, 509, 533, 502, 537, 491, 527, 496, 556, 515, 494, 555, 500, 504, 546, 517, 518, 489, 544, 517, 493, 513, 492, 524, 475, 516, 493, 512, 511, 491, 515, 510, 519, 498, 521, 532, 519, 485, 507, 516, 546, 487, 534, 521, 523, 557, 464, 559] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[504, 531, 493, 500, 548, 512, 489, 540, 491, 476, 505, 517, 534, 525, 532, 492, 519, 504, 535, 517, 485, 544, 498, 526, 527, 547, 491, 503, 535, 544, 449, 505, 510, 488, 503, 503, 502, 510, 511, 534, 511, 512, 551, 530, 502, 518, 477, 524, 517, 452, 491, 508, 496, 518, 535, 504, 490, 524, 528, 504, 544, 525, 558, 470] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[493, 506, 518, 498, 488, 510, 528, 508, 548, 487, 519, 525, 510, 535, 510, 499, 529, 540, 455, 513, 507, 539, 517, 534, 488, 503, 555, 500, 495, 508, 528, 512, 507, 487, 494, 489, 501, 521, 481, 521, 507, 561, 530, 509, 499, 551, 514, 530, 511, 526, 489, 534, 513, 517, 488, 483, 486, 515, 544, 483, 520, 544, 512, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[510, 520, 492, 521, 508, 469, 494, 536, 521, 511, 512, 516, 482, 545, 529, 540, 486, 534, 517, 521, 484, 480, 553, 511, 536, 482, 487, 534, 500, 514, 517, 510, 495, 493, 539, 517, 500, 540, 525, 494, 534, 542, 523, 544, 534, 513, 508, 482, 514, 551, 508, 501, 505, 497, 520, 511, 473, 533, 510, 523, 500, 492, 487, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[500, 510, 514, 485, 513, 498, 516, 510, 23, 0, 0, 0, 0, 0, 0, 0, 498, 561, 536, 464, 509, 514, 535, 503, 15, 0, 0, 0, 0, 0, 0, 0, 526, 501, 488, 493, 532, 499, 517, 562, 0, 0, 0, 0, 0, 0, 0, 0, 475, 501, 534, 481, 534, 539, 499, 531, 0, 0, 0, 0, 0, 0, 0, 0, 527, 476, 486, 535, 504, 534, 491, 526, 3, 0, 0, 0, 0, 0, 0, 0, 488, 490, 496, 506, 500, 544, 534, 492, 0, 0, 0, 0, 0, 0, 0, 0, 517, 505, 528, 547, 537, 490, 532, 533, 9, 0, 0, 0, 0, 0, 0, 0, 501, 518, 537, 502, 463, 513, 496, 492, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4135, 4118, 4094, 4082, 4050, 4198, 4022] avg=4096.0000 max=4198 max_rank=6 min=4022 min_rank=7 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16352 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[492, 502, 501, 531, 514, 526, 528, 505, 22, 0, 0, 0, 0, 0, 0, 0, 520, 482, 480, 493, 526, 536, 475, 520, 15, 0, 0, 0, 0, 0, 0, 0, 498, 507, 504, 535, 528, 482, 510, 513, 0, 0, 0, 0, 0, 0, 0, 0, 499, 548, 515, 506, 475, 487, 503, 532, 0, 0, 0, 0, 0, 0, 0, 0, 498, 541, 516, 505, 512, 552, 538, 500, 2, 0, 0, 0, 0, 0, 0, 0, 486, 460, 505, 491, 545, 535, 511, 519, 0, 0, 0, 0, 0, 0, 0, 0, 538, 552, 491, 538, 468, 489, 480, 497, 8, 0, 0, 0, 0, 0, 0, 0, 553, 512, 502, 520, 552, 499, 539, 504, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4047, 4077, 4065, 4164, 4052, 4061, 4181] avg=4096.0000 max=4181 max_rank=7 min=4047 min_rank=1 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16458 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[508, 491, 554, 521, 528, 471, 505, 511, 24, 0, 0, 0, 0, 0, 0, 0, 504, 520, 492, 535, 526, 473, 470, 515, 15, 0, 0, 0, 0, 0, 0, 0, 488, 511, 527, 511, 550, 513, 535, 482, 0, 0, 0, 0, 0, 0, 0, 0, 550, 492, 503, 533, 492, 501, 475, 515, 0, 0, 0, 0, 0, 0, 0, 0, 492, 489, 564, 513, 525, 516, 534, 500, 3, 0, 0, 0, 0, 0, 0, 0, 570, 510, 503, 500, 528, 475, 532, 484, 0, 0, 0, 0, 0, 0, 0, 0, 547, 487, 533, 490, 488, 495, 535, 511, 8, 0, 0, 0, 0, 0, 0, 0, 483, 536, 535, 539, 493, 499, 498, 512, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4050, 4117, 4061, 4136, 4102, 4094, 4095] avg=4096.0000 max=4136 max_rank=4 min=4050 min_rank=1 imbalance_ratio=1.009766 max_load_reduction=28632 max_load_reduction_pct=0.873779 moved_to_echo=16427 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[496, 527, 517, 461, 518, 506, 517, 507, 24, 0, 0, 0, 0, 0, 0, 0, 511, 497, 550, 505, 525, 515, 495, 516, 14, 0, 0, 0, 0, 0, 0, 0, 469, 518, 538, 494, 518, 490, 518, 504, 0, 0, 0, 0, 0, 0, 0, 0, 481, 490, 536, 531, 529, 502, 551, 510, 0, 0, 0, 0, 0, 0, 0, 0, 519, 504, 524, 514, 500, 501, 502, 549, 2, 0, 0, 0, 0, 0, 0, 0, 499, 515, 534, 558, 539, 469, 491, 532, 0, 0, 0, 0, 0, 0, 0, 0, 479, 540, 526, 494, 541, 500, 491, 512, 9, 0, 0, 0, 0, 0, 0, 0, 524, 519, 489, 516, 495, 491, 513, 497, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4128, 4049, 4130, 4115, 4137, 4092, 4044] avg=4096.0000 max=4137 max_rank=5 min=4044 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16388 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[504, 531, 493, 500, 548, 512, 489, 540, 23, 0, 0, 0, 0, 0, 0, 0, 491, 476, 505, 517, 534, 525, 532, 492, 13, 0, 0, 0, 0, 0, 0, 0, 519, 504, 535, 517, 472, 544, 498, 526, 0, 0, 0, 0, 0, 0, 0, 0, 527, 547, 483, 503, 535, 544, 449, 505, 0, 0, 0, 0, 0, 0, 0, 0, 510, 488, 503, 503, 502, 510, 511, 534, 3, 0, 0, 0, 0, 0, 0, 0, 511, 489, 551, 530, 502, 518, 477, 524, 0, 0, 0, 0, 0, 0, 0, 0, 517, 452, 491, 508, 496, 518, 535, 504, 8, 0, 0, 0, 0, 0, 0, 0, 490, 524, 525, 504, 544, 525, 558, 470, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4140, 4085, 4115, 4093, 4064, 4102, 4029, 4140] avg=4096.0000 max=4140 max_rank=0 min=4029 min_rank=6 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16335 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[505, 485, 525, 549, 496, 505, 513, 497, 23, 0, 0, 0, 0, 0, 0, 0, 499, 486, 479, 491, 508, 515, 500, 509, 15, 0, 0, 0, 0, 0, 0, 0, 523, 509, 533, 502, 522, 491, 527, 496, 0, 0, 0, 0, 0, 0, 0, 0, 556, 515, 486, 555, 500, 504, 546, 517, 0, 0, 0, 0, 0, 0, 0, 0, 518, 489, 544, 517, 493, 513, 492, 524, 3, 0, 0, 0, 0, 0, 0, 0, 475, 493, 493, 512, 511, 491, 515, 510, 0, 0, 0, 0, 0, 0, 0, 0, 519, 498, 521, 532, 519, 485, 507, 516, 8, 0, 0, 0, 0, 0, 0, 0, 546, 487, 531, 521, 523, 557, 464, 559, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4002, 4103, 4179, 4093, 4000, 4105, 4188] avg=4096.0000 max=4188 max_rank=7 min=4000 min_rank=5 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16386 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[493, 506, 518, 498, 488, 510, 528, 508, 30, 0, 0, 0, 0, 0, 0, 0, 548, 487, 519, 525, 510, 535, 510, 499, 18, 1, 0, 0, 0, 0, 0, 0, 529, 540, 455, 513, 489, 539, 517, 534, 0, 0, 0, 0, 0, 0, 0, 0, 488, 503, 540, 500, 495, 508, 528, 512, 0, 0, 0, 0, 0, 0, 0, 0, 507, 487, 494, 489, 501, 521, 481, 521, 6, 2, 0, 0, 0, 0, 0, 0, 507, 530, 530, 509, 499, 551, 514, 530, 0, 0, 0, 0, 0, 0, 0, 0, 511, 526, 489, 534, 513, 517, 488, 483, 13, 0, 0, 0, 0, 0, 0, 0, 486, 515, 538, 483, 520, 544, 512, 496, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4152, 4116, 4074, 4009, 4170, 4074, 4094] avg=4096.0000 max=4170 max_rank=5 min=4009 min_rank=4 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16347 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[510, 520, 492, 521, 508, 469, 494, 536, 25, 0, 0, 0, 0, 0, 0, 0, 521, 511, 512, 516, 482, 545, 529, 540, 13, 0, 0, 0, 0, 0, 0, 0, 486, 534, 517, 521, 471, 480, 553, 511, 0, 0, 0, 0, 0, 0, 0, 0, 536, 482, 479, 534, 500, 514, 517, 510, 0, 0, 0, 0, 0, 0, 0, 0, 495, 493, 539, 517, 500, 540, 525, 494, 2, 0, 0, 0, 0, 0, 0, 0, 534, 517, 523, 544, 534, 513, 508, 482, 0, 0, 0, 0, 0, 0, 0, 0, 514, 551, 508, 501, 505, 497, 520, 511, 8, 0, 0, 0, 0, 0, 0, 0, 473, 533, 508, 523, 500, 492, 487, 488, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4169, 4073, 4072, 4105, 4155, 4115, 4004] avg=4096.0000 max=4169 max_rank=1 min=4004 min_rank=7 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16379 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 3968, 4224, 4096, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[508, 485, 499, 522, 543, 521, 522, 537, 523, 516, 537, 495, 559, 530, 502, 483, 531, 528, 506, 501, 516, 512, 508, 492, 511, 491, 510, 518, 486, 496, 545, 494, 494, 510, 517, 536, 521, 473, 544, 511, 487, 497, 530, 493, 505, 518, 518, 514, 526, 513, 481, 532, 532, 492, 507, 518, 515, 516, 497, 495, 511, 526, 536, 476] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[508, 510, 531, 514, 534, 547, 531, 523, 503, 486, 514, 486, 512, 498, 499, 499, 535, 533, 517, 507, 528, 510, 509, 518, 496, 488, 481, 504, 526, 530, 533, 502, 504, 512, 505, 511, 465, 548, 492, 524, 525, 536, 514, 520, 482, 522, 540, 524, 520, 495, 490, 513, 520, 575, 543, 506, 517, 475, 497, 486, 468, 516, 474, 537] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[502, 487, 513, 567, 535, 516, 515, 541, 509, 496, 543, 519, 522, 505, 491, 515, 492, 502, 488, 497, 538, 562, 522, 519, 523, 522, 531, 543, 548, 491, 467, 491, 482, 502, 536, 486, 506, 496, 538, 486, 499, 530, 514, 517, 456, 488, 507, 530, 491, 526, 542, 527, 501, 521, 469, 527, 550, 498, 503, 485, 470, 548, 554, 471] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[550, 521, 516, 546, 495, 527, 497, 489, 511, 492, 515, 480, 487, 514, 538, 481, 540, 526, 552, 505, 534, 542, 504, 516, 539, 535, 518, 514, 530, 524, 507, 541, 480, 461, 483, 481, 502, 480, 516, 533, 504, 509, 547, 502, 524, 510, 499, 493, 467, 470, 490, 512, 545, 505, 531, 507, 521, 475, 477, 515, 539, 541, 539, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[495, 510, 538, 482, 543, 506, 547, 519, 501, 500, 495, 489, 473, 512, 546, 506, 510, 494, 509, 503, 531, 498, 487, 513, 548, 474, 499, 518, 464, 523, 459, 543, 515, 503, 503, 539, 534, 527, 518, 499, 516, 520, 525, 504, 518, 542, 555, 464, 516, 510, 523, 516, 521, 527, 513, 515, 503, 553, 527, 518, 499, 509, 493, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[510, 513, 506, 492, 511, 528, 497, 518, 490, 495, 561, 534, 502, 526, 528, 489, 509, 513, 510, 475, 577, 517, 478, 526, 516, 509, 489, 486, 518, 476, 509, 548, 528, 487, 522, 521, 529, 511, 516, 502, 536, 501, 499, 486, 536, 536, 515, 478, 544, 506, 528, 495, 509, 493, 484, 515, 533, 515, 503, 539, 516, 541, 509, 479] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[510, 536, 485, 542, 536, 508, 499, 525, 483, 535, 540, 545, 519, 531, 516, 520, 494, 528, 478, 574, 489, 543, 520, 504, 499, 506, 492, 486, 521, 552, 506, 512, 524, 533, 484, 491, 516, 499, 491, 541, 517, 487, 529, 470, 496, 492, 489, 521, 527, 504, 525, 505, 514, 500, 526, 508, 501, 471, 531, 536, 467, 514, 496, 529] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[497, 498, 506, 507, 492, 508, 522, 496, 481, 462, 499, 511, 505, 519, 499, 572, 560, 517, 521, 515, 531, 502, 495, 534, 524, 514, 447, 570, 526, 486, 495, 492, 534, 504, 523, 488, 508, 512, 527, 505, 494, 541, 521, 502, 504, 519, 469, 512, 509, 499, 516, 529, 531, 527, 476, 478, 540, 521, 542, 519, 570, 497, 521, 527] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[508, 485, 499, 522, 509, 521, 522, 537, 0, 0, 0, 0, 0, 0, 0, 0, 523, 516, 537, 495, 559, 530, 502, 483, 16, 0, 0, 0, 0, 0, 0, 0, 531, 528, 506, 501, 483, 512, 508, 492, 0, 0, 0, 0, 0, 0, 0, 0, 511, 491, 510, 518, 486, 496, 545, 494, 9, 6, 0, 0, 0, 0, 0, 0, 494, 510, 517, 536, 521, 473, 544, 511, 24, 0, 0, 0, 0, 0, 0, 0, 487, 497, 530, 493, 505, 518, 518, 514, 12, 1, 1, 0, 0, 0, 0, 0, 526, 513, 481, 532, 531, 492, 507, 518, 0, 0, 0, 0, 0, 0, 0, 0, 515, 516, 497, 495, 511, 525, 536, 476, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4161, 4061, 4066, 4130, 4076, 4100, 4071] avg=4096.0000 max=4161 max_rank=1 min=4061 min_rank=2 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[508, 510, 531, 514, 502, 547, 531, 523, 0, 0, 0, 0, 0, 0, 0, 0, 503, 486, 514, 486, 512, 498, 499, 499, 15, 0, 0, 0, 0, 0, 0, 0, 535, 533, 517, 507, 495, 510, 509, 518, 0, 0, 0, 0, 0, 0, 0, 0, 496, 488, 481, 504, 526, 530, 533, 502, 9, 6, 0, 0, 0, 0, 0, 0, 504, 512, 505, 511, 465, 548, 492, 524, 24, 0, 0, 0, 0, 0, 0, 0, 525, 536, 514, 520, 482, 522, 540, 524, 11, 1, 1, 0, 0, 0, 0, 0, 520, 495, 490, 513, 519, 575, 543, 506, 0, 0, 0, 0, 0, 0, 0, 0, 517, 475, 497, 486, 468, 515, 474, 537, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4166, 4012, 4124, 4075, 4085, 4176, 4161, 3969] avg=4096.0000 max=4176 max_rank=5 min=3969 min_rank=7 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[502, 487, 513, 567, 502, 516, 515, 541, 0, 0, 0, 0, 0, 0, 0, 0, 509, 496, 543, 519, 522, 505, 491, 515, 15, 0, 0, 0, 0, 0, 0, 0, 492, 502, 488, 497, 504, 562, 522, 519, 0, 0, 0, 0, 0, 0, 0, 0, 523, 522, 531, 543, 548, 491, 467, 491, 9, 6, 0, 0, 0, 0, 0, 0, 482, 502, 536, 486, 506, 496, 538, 486, 25, 0, 0, 0, 0, 0, 0, 0, 499, 530, 514, 517, 456, 488, 507, 530, 12, 1, 1, 0, 0, 0, 0, 0, 491, 526, 542, 527, 500, 521, 469, 527, 0, 0, 0, 0, 0, 0, 0, 0, 550, 498, 503, 485, 470, 547, 554, 471, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4143, 4115, 4086, 4131, 4057, 4055, 4103, 4078] avg=4096.0000 max=4143 max_rank=0 min=4055 min_rank=5 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16293 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[495, 510, 538, 482, 509, 506, 547, 519, 0, 0, 0, 0, 0, 0, 0, 0, 501, 500, 495, 489, 473, 512, 546, 506, 16, 0, 0, 0, 0, 0, 0, 0, 510, 494, 509, 503, 498, 498, 487, 513, 0, 0, 0, 0, 0, 0, 0, 0, 548, 474, 499, 518, 464, 523, 459, 543, 9, 6, 0, 0, 0, 0, 0, 0, 515, 503, 503, 539, 534, 527, 518, 499, 24, 0, 0, 0, 0, 0, 0, 0, 516, 520, 525, 504, 518, 542, 555, 464, 12, 1, 1, 0, 0, 0, 0, 0, 516, 510, 523, 516, 520, 527, 513, 515, 0, 0, 0, 0, 0, 0, 0, 0, 503, 553, 527, 518, 499, 508, 493, 508, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4038, 4012, 4043, 4162, 4158, 4140, 4109] avg=4096.0000 max=4162 max_rank=4 min=4012 min_rank=2 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16569 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[510, 536, 485, 542, 494, 508, 499, 525, 0, 0, 0, 0, 0, 0, 0, 0, 483, 535, 540, 545, 519, 531, 516, 520, 19, 0, 0, 0, 0, 0, 0, 0, 494, 528, 478, 574, 449, 543, 520, 504, 0, 0, 0, 0, 0, 0, 0, 0, 499, 506, 492, 486, 521, 552, 506, 512, 14, 9, 0, 0, 0, 0, 0, 0, 524, 533, 484, 491, 516, 499, 491, 541, 26, 0, 0, 0, 0, 0, 0, 0, 517, 487, 529, 470, 496, 492, 489, 521, 14, 8, 4, 0, 0, 0, 0, 0, 527, 504, 525, 505, 506, 500, 526, 508, 0, 0, 0, 0, 0, 0, 0, 0, 501, 471, 531, 536, 467, 510, 496, 529, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4208, 4090, 4097, 4105, 4027, 4101, 4041] avg=4096.0000 max=4208 max_rank=1 min=4027 min_rank=5 imbalance_ratio=1.027344 max_load_reduction=28560 max_load_reduction_pct=0.871582 moved_to_echo=16274 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[550, 521, 516, 546, 465, 527, 497, 489, 0, 0, 0, 0, 0, 0, 0, 0, 511, 492, 515, 480, 487, 514, 538, 481, 14, 0, 0, 0, 0, 0, 0, 0, 540, 526, 552, 505, 500, 542, 504, 516, 0, 0, 0, 0, 0, 0, 0, 0, 539, 535, 518, 514, 530, 524, 507, 541, 9, 5, 0, 0, 0, 0, 0, 0, 480, 461, 483, 481, 502, 480, 516, 533, 25, 0, 0, 0, 0, 0, 0, 0, 504, 509, 547, 502, 524, 510, 499, 493, 11, 1, 1, 0, 0, 0, 0, 0, 467, 470, 490, 512, 544, 505, 531, 507, 0, 0, 0, 0, 0, 0, 0, 0, 521, 475, 477, 515, 539, 540, 539, 524, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4032, 4185, 4222, 3961, 4101, 4026, 4130] avg=4096.0000 max=4222 max_rank=3 min=3961 min_rank=4 imbalance_ratio=1.030762 max_load_reduction=28546 max_load_reduction_pct=0.871155 moved_to_echo=16218 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[510, 513, 506, 492, 480, 528, 497, 518, 0, 0, 0, 0, 0, 0, 0, 0, 490, 495, 561, 534, 502, 526, 528, 489, 15, 0, 0, 0, 0, 0, 0, 0, 509, 513, 510, 475, 540, 517, 478, 526, 0, 0, 0, 0, 0, 0, 0, 0, 516, 509, 489, 486, 518, 476, 509, 548, 10, 5, 0, 0, 0, 0, 0, 0, 528, 487, 522, 521, 529, 511, 516, 502, 27, 0, 0, 0, 0, 0, 0, 0, 536, 501, 499, 486, 536, 536, 515, 478, 11, 1, 1, 0, 0, 0, 0, 0, 544, 506, 528, 495, 508, 493, 484, 515, 0, 0, 0, 0, 0, 0, 0, 0, 533, 515, 503, 539, 516, 540, 509, 479, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4044, 4140, 4068, 4066, 4143, 4100, 4073, 4134] avg=4096.0000 max=4143 max_rank=4 min=4044 min_rank=0 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16450 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[497, 498, 506, 507, 462, 508, 522, 496, 0, 0, 0, 0, 0, 0, 0, 0, 481, 462, 499, 511, 505, 519, 499, 572, 14, 0, 0, 0, 0, 0, 0, 0, 560, 517, 521, 515, 498, 502, 495, 534, 0, 0, 0, 0, 0, 0, 0, 0, 524, 514, 447, 570, 526, 486, 495, 492, 9, 5, 0, 0, 0, 0, 0, 0, 534, 504, 523, 488, 508, 512, 527, 505, 24, 0, 0, 0, 0, 0, 0, 0, 494, 541, 521, 502, 504, 519, 469, 512, 11, 1, 1, 0, 0, 0, 0, 0, 509, 499, 516, 529, 530, 527, 476, 478, 0, 0, 0, 0, 0, 0, 0, 0, 540, 521, 542, 519, 570, 496, 521, 527, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3996, 4062, 4142, 4068, 4125, 4075, 4064, 4236] avg=4096.0000 max=4236 max_rank=7 min=3996 min_rank=0 imbalance_ratio=1.034180 max_load_reduction=28532 max_load_reduction_pct=0.870728 moved_to_echo=16500 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4224, 3968, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4096, 4096, 3968, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 3968, 4224, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[503, 469, 519, 513, 517, 479, 561, 512, 523, 520, 500, 502, 489, 485, 535, 554, 471, 542, 485, 530, 478, 498, 527, 526, 492, 519, 522, 509, 509, 521, 502, 548, 517, 496, 517, 519, 501, 527, 498, 531, 544, 505, 512, 518, 522, 490, 464, 525, 455, 526, 509, 535, 526, 528, 480, 543, 546, 501, 503, 517, 546, 518, 498, 491] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[540, 495, 526, 547, 489, 499, 491, 511, 498, 489, 507, 514, 461, 485, 489, 540, 495, 497, 509, 516, 534, 524, 501, 536, 518, 544, 478, 524, 487, 501, 535, 495, 529, 502, 508, 497, 533, 509, 511, 525, 502, 519, 535, 546, 493, 547, 540, 512, 514, 517, 480, 500, 503, 489, 526, 506, 526, 527, 545, 493, 561, 484, 517, 497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[492, 531, 514, 474, 491, 501, 503, 488, 519, 498, 531, 553, 519, 525, 524, 536, 507, 500, 459, 485, 542, 497, 503, 505, 528, 516, 521, 523, 506, 472, 524, 529, 526, 509, 523, 483, 497, 486, 522, 481, 539, 548, 525, 509, 499, 491, 580, 508, 546, 500, 528, 534, 526, 540, 500, 533, 511, 507, 473, 511, 508, 491, 494, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[479, 506, 507, 535, 486, 500, 505, 509, 556, 524, 504, 504, 466, 554, 544, 523, 489, 530, 509, 493, 539, 492, 514, 476, 492, 526, 524, 502, 515, 495, 534, 470, 523, 500, 500, 503, 511, 504, 505, 526, 497, 556, 532, 505, 518, 492, 553, 502, 486, 488, 518, 551, 497, 517, 518, 511, 530, 508, 543, 481, 536, 511, 541, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[514, 522, 561, 498, 515, 487, 528, 527, 478, 536, 499, 517, 478, 527, 530, 534, 470, 524, 492, 536, 536, 530, 508, 507, 515, 511, 501, 510, 552, 496, 494, 464, 524, 514, 508, 526, 520, 491, 491, 490, 502, 551, 541, 539, 520, 515, 526, 510, 509, 525, 521, 502, 529, 534, 493, 503, 518, 461, 473, 532, 485, 487, 525, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[520, 516, 512, 485, 474, 480, 521, 510, 528, 519, 507, 510, 535, 521, 505, 524, 547, 528, 526, 499, 519, 493, 518, 515, 517, 509, 507, 544, 531, 553, 499, 518, 475, 517, 518, 502, 541, 496, 504, 507, 507, 508, 458, 483, 496, 504, 505, 507, 502, 529, 532, 545, 515, 474, 480, 568, 482, 494, 523, 502, 554, 532, 510, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[538, 564, 511, 458, 486, 532, 487, 532, 544, 549, 539, 448, 511, 523, 510, 472, 520, 512, 521, 504, 548, 520, 532, 534, 498, 512, 540, 451, 526, 527, 530, 503, 513, 544, 499, 524, 516, 521, 503, 504, 493, 500, 545, 470, 505, 471, 502, 530, 470, 480, 513, 533, 513, 486, 533, 486, 508, 500, 517, 518, 533, 523, 519, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[485, 513, 532, 500, 531, 531, 572, 525, 486, 543, 523, 531, 534, 535, 489, 532, 491, 551, 515, 539, 538, 540, 470, 496, 519, 507, 510, 570, 532, 509, 509, 557, 529, 507, 513, 492, 537, 522, 456, 486, 452, 485, 481, 491, 487, 465, 525, 507, 471, 506, 487, 521, 531, 526, 534, 532, 514, 520, 497, 496, 502, 509, 489, 483] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[503, 469, 519, 513, 517, 479, 561, 512, 8, 5, 0, 0, 0, 0, 0, 0, 523, 520, 500, 502, 489, 485, 535, 526, 0, 0, 0, 0, 0, 0, 0, 0, 471, 542, 485, 530, 476, 498, 527, 526, 0, 0, 0, 0, 0, 0, 0, 0, 492, 519, 522, 509, 494, 521, 502, 548, 0, 0, 0, 0, 0, 0, 0, 0, 517, 496, 517, 519, 501, 527, 498, 531, 20, 0, 0, 0, 0, 0, 0, 0, 544, 505, 512, 518, 522, 490, 464, 525, 4, 2, 0, 0, 0, 0, 0, 0, 455, 526, 509, 531, 526, 528, 480, 543, 0, 0, 0, 0, 0, 0, 0, 0, 546, 501, 503, 517, 546, 518, 498, 491, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4080, 4055, 4107, 4126, 4086, 4098, 4130] avg=4096.0000 max=4130 max_rank=7 min=4055 min_rank=2 imbalance_ratio=1.008301 max_load_reduction=28638 max_load_reduction_pct=0.873962 moved_to_echo=16440 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[492, 531, 514, 474, 491, 501, 503, 488, 7, 5, 0, 0, 0, 0, 0, 0, 519, 498, 531, 553, 519, 525, 524, 509, 0, 0, 0, 0, 0, 0, 0, 0, 507, 500, 459, 485, 540, 497, 503, 505, 0, 0, 0, 0, 0, 0, 0, 0, 528, 516, 521, 523, 491, 472, 524, 529, 0, 0, 0, 0, 0, 0, 0, 0, 526, 509, 523, 483, 497, 486, 522, 481, 20, 0, 0, 0, 0, 0, 0, 0, 539, 548, 525, 509, 499, 491, 580, 508, 4, 2, 0, 0, 0, 0, 0, 0, 546, 500, 528, 530, 526, 540, 500, 533, 0, 0, 0, 0, 0, 0, 0, 0, 511, 507, 473, 511, 508, 491, 494, 524, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4006, 4178, 3996, 4104, 4047, 4205, 4203, 4029] avg=4096.0000 max=4205 max_rank=5 min=3996 min_rank=2 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16484 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[479, 506, 507, 535, 486, 500, 505, 509, 7, 5, 0, 0, 0, 0, 0, 0, 556, 524, 504, 504, 466, 554, 544, 497, 0, 0, 0, 0, 0, 0, 0, 0, 489, 530, 509, 493, 537, 492, 514, 476, 0, 0, 0, 0, 0, 0, 0, 0, 492, 526, 524, 502, 500, 495, 534, 470, 0, 0, 0, 0, 0, 0, 0, 0, 523, 500, 500, 503, 511, 504, 505, 526, 19, 0, 0, 0, 0, 0, 0, 0, 497, 556, 532, 505, 518, 492, 553, 502, 5, 2, 0, 0, 0, 0, 0, 0, 486, 488, 518, 546, 497, 517, 518, 511, 0, 0, 0, 0, 0, 0, 0, 0, 530, 508, 543, 481, 536, 511, 541, 503, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4039, 4149, 4040, 4043, 4091, 4162, 4081, 4163] avg=4096.0000 max=4163 max_rank=7 min=4039 min_rank=0 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16497 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[520, 516, 512, 485, 474, 480, 521, 510, 7, 6, 0, 0, 0, 0, 0, 0, 528, 519, 507, 510, 535, 521, 505, 498, 0, 0, 0, 0, 0, 0, 0, 0, 547, 528, 526, 499, 517, 493, 518, 515, 0, 0, 0, 0, 0, 0, 0, 0, 517, 509, 507, 544, 515, 553, 499, 518, 0, 0, 0, 0, 0, 0, 0, 0, 475, 517, 518, 502, 541, 496, 504, 507, 19, 0, 0, 0, 0, 0, 0, 0, 507, 508, 458, 483, 496, 504, 505, 507, 5, 2, 0, 0, 0, 0, 0, 0, 502, 529, 532, 540, 515, 474, 480, 568, 0, 0, 0, 0, 0, 0, 0, 0, 482, 494, 523, 502, 554, 532, 510, 508, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4031, 4123, 4143, 4162, 4079, 3975, 4140, 4115] avg=4096.0000 max=4162 max_rank=3 min=3975 min_rank=5 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16309 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[540, 495, 526, 547, 489, 499, 491, 511, 7, 5, 0, 0, 0, 0, 0, 0, 498, 489, 507, 514, 461, 485, 489, 513, 0, 0, 0, 0, 0, 0, 0, 0, 495, 497, 509, 516, 532, 524, 501, 536, 0, 0, 0, 0, 0, 0, 0, 0, 518, 544, 478, 524, 472, 501, 535, 495, 0, 0, 0, 0, 0, 0, 0, 0, 529, 502, 508, 497, 533, 509, 511, 525, 20, 0, 0, 0, 0, 0, 0, 0, 502, 519, 535, 546, 493, 547, 540, 512, 4, 2, 0, 0, 0, 0, 0, 0, 514, 517, 480, 496, 503, 489, 526, 506, 0, 0, 0, 0, 0, 0, 0, 0, 526, 527, 545, 493, 561, 484, 517, 497, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 3956, 4110, 4067, 4134, 4200, 4031, 4160] avg=4096.0000 max=4200 max_rank=5 min=3956 min_rank=1 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16525 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[514, 522, 561, 498, 515, 487, 528, 527, 12, 10, 0, 0, 0, 0, 0, 0, 478, 536, 499, 517, 478, 527, 530, 498, 0, 0, 0, 0, 0, 0, 0, 0, 470, 524, 492, 536, 530, 530, 508, 507, 0, 0, 0, 0, 0, 0, 0, 0, 515, 511, 501, 510, 527, 496, 494, 464, 0, 0, 0, 0, 0, 0, 0, 0, 524, 514, 508, 526, 520, 491, 491, 490, 24, 0, 0, 0, 0, 0, 0, 0, 502, 551, 541, 539, 520, 515, 526, 510, 9, 6, 0, 0, 0, 0, 0, 0, 509, 525, 521, 487, 529, 534, 493, 503, 0, 0, 0, 0, 0, 0, 0, 0, 518, 461, 473, 532, 485, 487, 525, 506, 15, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4174, 4063, 4097, 4018, 4088, 4219, 4101, 4008] avg=4096.0000 max=4219 max_rank=5 min=4008 min_rank=7 imbalance_ratio=1.030029 max_load_reduction=28549 max_load_reduction_pct=0.871246 moved_to_echo=16416 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[538, 564, 511, 458, 486, 532, 487, 532, 6, 6, 0, 0, 0, 0, 0, 0, 544, 549, 539, 448, 511, 523, 510, 449, 0, 0, 0, 0, 0, 0, 0, 0, 520, 512, 521, 504, 546, 520, 532, 534, 0, 0, 0, 0, 0, 0, 0, 0, 498, 512, 540, 451, 510, 527, 530, 503, 0, 0, 0, 0, 0, 0, 0, 0, 513, 544, 499, 524, 516, 521, 503, 504, 17, 0, 0, 0, 0, 0, 0, 0, 493, 500, 545, 470, 505, 471, 502, 530, 4, 2, 0, 0, 0, 0, 0, 0, 470, 480, 513, 529, 513, 486, 533, 486, 0, 0, 0, 0, 0, 0, 0, 0, 508, 500, 517, 518, 533, 523, 519, 514, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4073, 4189, 4071, 4141, 4022, 4010, 4142] avg=4096.0000 max=4189 max_rank=2 min=4010 min_rank=6 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16315 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[485, 513, 532, 500, 531, 531, 572, 525, 7, 6, 0, 0, 0, 0, 0, 0, 486, 543, 523, 531, 534, 535, 489, 505, 0, 0, 0, 0, 0, 0, 0, 0, 491, 551, 515, 539, 536, 540, 470, 496, 0, 0, 0, 0, 0, 0, 0, 0, 519, 507, 510, 570, 515, 509, 509, 557, 0, 0, 0, 0, 0, 0, 0, 0, 529, 507, 513, 492, 537, 522, 456, 486, 20, 0, 0, 0, 0, 0, 0, 0, 452, 485, 481, 491, 487, 465, 525, 507, 4, 2, 0, 0, 0, 0, 0, 0, 471, 506, 487, 517, 531, 526, 534, 532, 0, 0, 0, 0, 0, 0, 0, 0, 514, 520, 497, 496, 502, 509, 489, 483, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4202, 4146, 4138, 4196, 4062, 3899, 4104, 4021] avg=4096.0000 max=4202 max_rank=0 min=3899 min_rank=5 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16086 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4352, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[3968, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[494, 503, 486, 521, 497, 502, 519, 515, 473, 512, 503, 519, 501, 539, 517, 480, 529, 501, 480, 549, 514, 461, 507, 522, 520, 513, 538, 561, 493, 542, 528, 512, 512, 476, 494, 528, 508, 539, 495, 480, 503, 524, 494, 514, 499, 518, 548, 478, 541, 510, 509, 525, 497, 535, 528, 536, 521, 534, 519, 505, 532, 480, 542, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[524, 500, 491, 480, 498, 508, 469, 510, 525, 541, 505, 515, 532, 525, 509, 480, 467, 498, 542, 537, 534, 496, 488, 531, 511, 514, 492, 560, 492, 521, 507, 528, 546, 489, 480, 530, 515, 556, 512, 516, 494, 522, 497, 528, 526, 505, 483, 502, 529, 495, 522, 525, 497, 502, 519, 528, 500, 554, 512, 532, 487, 509, 526, 500] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[493, 542, 491, 522, 451, 514, 526, 493, 482, 535, 516, 535, 531, 498, 506, 514, 511, 497, 547, 528, 493, 532, 502, 542, 458, 507, 492, 476, 527, 529, 495, 512, 518, 516, 527, 478, 520, 528, 488, 488, 488, 494, 522, 555, 555, 523, 483, 514, 492, 503, 552, 496, 499, 537, 494, 527, 549, 532, 501, 531, 539, 518, 510, 494] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[516, 513, 483, 518, 525, 536, 465, 527, 497, 477, 531, 537, 503, 539, 530, 492, 511, 488, 521, 520, 520, 515, 545, 543, 498, 513, 498, 500, 497, 514, 495, 528, 505, 525, 526, 501, 496, 535, 513, 531, 509, 545, 503, 531, 502, 521, 494, 497, 535, 503, 536, 490, 511, 500, 519, 495, 508, 506, 512, 492, 507, 528, 471, 526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[501, 510, 501, 526, 498, 498, 553, 511, 504, 533, 523, 494, 565, 490, 504, 500, 519, 523, 526, 532, 458, 499, 516, 481, 496, 523, 523, 554, 500, 532, 482, 503, 522, 491, 500, 494, 514, 525, 521, 527, 492, 552, 509, 499, 527, 503, 509, 524, 542, 502, 527, 495, 480, 545, 496, 500, 524, 510, 560, 482, 516, 481, 500, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[468, 506, 510, 541, 525, 497, 484, 538, 485, 508, 555, 519, 499, 542, 502, 535, 512, 508, 489, 506, 464, 507, 485, 523, 480, 475, 497, 542, 484, 482, 530, 490, 552, 516, 533, 529, 477, 522, 539, 483, 513, 525, 497, 504, 493, 533, 531, 494, 538, 505, 542, 525, 487, 499, 545, 515, 533, 511, 544, 552, 517, 519, 495, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[495, 479, 502, 535, 487, 513, 482, 488, 535, 503, 523, 513, 501, 515, 513, 494, 475, 498, 512, 518, 517, 519, 504, 532, 496, 568, 501, 541, 513, 484, 517, 504, 516, 516, 516, 537, 562, 536, 484, 519, 517, 486, 515, 503, 486, 490, 552, 502, 526, 527, 530, 474, 524, 529, 509, 525, 567, 516, 481, 526, 526, 490, 502, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[491, 546, 533, 512, 480, 531, 524, 534, 485, 484, 517, 526, 478, 490, 534, 504, 496, 501, 551, 532, 523, 517, 498, 514, 491, 536, 477, 509, 521, 489, 546, 494, 496, 503, 513, 509, 527, 484, 529, 535, 506, 536, 482, 493, 507, 500, 529, 542, 519, 513, 489, 519, 495, 516, 513, 478, 527, 530, 522, 521, 509, 511, 543, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[501, 510, 501, 526, 498, 498, 553, 511, 32, 9, 0, 0, 0, 0, 0, 0, 504, 533, 523, 494, 565, 490, 504, 500, 0, 0, 0, 0, 0, 0, 0, 0, 519, 523, 526, 532, 458, 499, 516, 481, 4, 0, 0, 0, 0, 0, 0, 0, 496, 523, 523, 554, 500, 532, 482, 503, 9, 5, 0, 0, 0, 0, 0, 0, 522, 491, 500, 494, 514, 511, 521, 527, 0, 0, 0, 0, 0, 0, 0, 0, 492, 552, 509, 499, 527, 503, 509, 524, 5, 0, 0, 0, 0, 0, 0, 0, 524, 502, 527, 495, 480, 545, 496, 500, 0, 0, 0, 0, 0, 0, 0, 0, 492, 510, 560, 482, 516, 481, 500, 521, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4139, 4113, 4058, 4127, 4080, 4120, 4069, 4062] avg=4096.0000 max=4139 max_rank=0 min=4058 min_rank=2 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16331 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[494, 503, 486, 521, 497, 502, 519, 515, 32, 9, 0, 0, 0, 0, 0, 0, 473, 512, 503, 519, 501, 539, 517, 480, 0, 0, 0, 0, 0, 0, 0, 0, 529, 501, 480, 549, 514, 461, 507, 522, 4, 0, 0, 0, 0, 0, 0, 0, 520, 513, 538, 561, 493, 542, 528, 512, 9, 5, 0, 0, 0, 0, 0, 0, 512, 476, 494, 528, 508, 525, 495, 480, 0, 0, 0, 0, 0, 0, 0, 0, 503, 524, 494, 514, 499, 518, 548, 478, 5, 0, 0, 0, 0, 0, 0, 0, 523, 510, 509, 525, 497, 535, 528, 536, 0, 0, 0, 0, 0, 0, 0, 0, 489, 534, 519, 505, 532, 480, 542, 493, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4044, 4067, 4221, 4018, 4083, 4163, 4094] avg=4096.0000 max=4221 max_rank=3 min=4018 min_rank=4 imbalance_ratio=1.030518 max_load_reduction=28547 max_load_reduction_pct=0.871185 moved_to_echo=16358 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[493, 542, 491, 522, 451, 514, 526, 493, 33, 8, 0, 0, 0, 0, 0, 0, 482, 535, 516, 535, 531, 498, 506, 514, 0, 0, 0, 0, 0, 0, 0, 0, 511, 497, 547, 528, 493, 532, 502, 542, 4, 0, 0, 0, 0, 0, 0, 0, 458, 507, 492, 476, 527, 529, 495, 512, 8, 5, 0, 0, 0, 0, 0, 0, 518, 516, 527, 478, 520, 514, 488, 488, 0, 0, 0, 0, 0, 0, 0, 0, 488, 494, 522, 555, 555, 523, 483, 514, 5, 0, 0, 0, 0, 0, 0, 0, 476, 503, 552, 496, 499, 537, 494, 527, 0, 0, 0, 0, 0, 0, 0, 0, 516, 532, 501, 531, 539, 518, 510, 494, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4117, 4156, 4009, 4049, 4139, 4084, 4141] avg=4096.0000 max=4156 max_rank=2 min=4009 min_rank=3 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16413 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[516, 513, 483, 518, 525, 536, 465, 527, 31, 9, 0, 0, 0, 0, 0, 0, 497, 477, 531, 537, 503, 539, 530, 492, 0, 0, 0, 0, 0, 0, 0, 0, 511, 488, 521, 520, 520, 515, 545, 543, 4, 0, 0, 0, 0, 0, 0, 0, 498, 513, 498, 500, 497, 514, 495, 528, 8, 5, 0, 0, 0, 0, 0, 0, 505, 525, 526, 501, 496, 521, 513, 531, 0, 0, 0, 0, 0, 0, 0, 0, 509, 545, 503, 531, 502, 521, 494, 497, 5, 0, 0, 0, 0, 0, 0, 0, 518, 503, 536, 490, 511, 500, 519, 495, 0, 0, 0, 0, 0, 0, 0, 0, 477, 506, 512, 492, 507, 528, 471, 526, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4123, 4106, 4167, 4056, 4118, 4107, 4072, 4019] avg=4096.0000 max=4167 max_rank=2 min=4019 min_rank=7 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16316 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[524, 500, 491, 480, 498, 508, 469, 510, 30, 9, 0, 0, 0, 0, 0, 0, 525, 541, 505, 515, 532, 525, 509, 480, 0, 0, 0, 0, 0, 0, 0, 0, 467, 498, 542, 537, 534, 496, 488, 531, 5, 0, 0, 0, 0, 0, 0, 0, 511, 514, 492, 560, 492, 521, 507, 528, 8, 6, 0, 0, 0, 0, 0, 0, 546, 489, 480, 530, 515, 539, 512, 516, 0, 0, 0, 0, 0, 0, 0, 0, 494, 522, 497, 528, 526, 505, 483, 502, 6, 0, 0, 0, 0, 0, 0, 0, 512, 495, 522, 525, 497, 502, 519, 528, 0, 0, 0, 0, 0, 0, 0, 0, 470, 554, 512, 532, 487, 509, 526, 500, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4019, 4132, 4098, 4139, 4127, 4063, 4100, 4090] avg=4096.0000 max=4139 max_rank=3 min=4019 min_rank=0 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[468, 506, 510, 541, 525, 497, 484, 538, 36, 15, 0, 0, 0, 0, 0, 0, 485, 508, 551, 519, 499, 542, 502, 535, 0, 0, 0, 0, 0, 0, 0, 0, 512, 508, 489, 506, 464, 507, 485, 523, 9, 4, 0, 0, 0, 0, 0, 0, 480, 475, 497, 542, 484, 482, 530, 490, 13, 10, 0, 0, 0, 0, 0, 0, 552, 516, 533, 529, 477, 493, 539, 483, 0, 0, 0, 0, 0, 0, 0, 0, 513, 525, 497, 504, 493, 533, 531, 494, 10, 0, 0, 0, 0, 0, 0, 0, 510, 505, 542, 525, 487, 499, 545, 515, 0, 0, 0, 0, 0, 0, 0, 0, 497, 511, 544, 552, 517, 519, 495, 512, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4141, 4007, 4003, 4122, 4100, 4128, 4147] avg=4096.0000 max=4147 max_rank=7 min=4003 min_rank=3 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16497 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[495, 479, 502, 535, 487, 513, 482, 488, 34, 9, 0, 0, 0, 0, 0, 0, 535, 503, 523, 513, 501, 515, 513, 494, 0, 0, 0, 0, 0, 0, 0, 0, 475, 498, 512, 518, 517, 519, 504, 532, 4, 0, 0, 0, 0, 0, 0, 0, 496, 568, 501, 541, 513, 484, 517, 504, 8, 5, 0, 0, 0, 0, 0, 0, 516, 516, 516, 537, 562, 522, 484, 519, 0, 0, 0, 0, 0, 0, 0, 0, 517, 486, 515, 503, 486, 490, 552, 502, 5, 0, 0, 0, 0, 0, 0, 0, 509, 527, 530, 474, 524, 529, 509, 525, 0, 0, 0, 0, 0, 0, 0, 0, 533, 516, 481, 526, 526, 490, 502, 502, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4024, 4097, 4079, 4137, 4172, 4056, 4127, 4076] avg=4096.0000 max=4172 max_rank=4 min=4024 min_rank=0 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[491, 546, 533, 512, 480, 531, 524, 534, 32, 9, 0, 0, 0, 0, 0, 0, 485, 484, 517, 526, 478, 490, 534, 504, 0, 0, 0, 0, 0, 0, 0, 0, 496, 501, 551, 532, 523, 517, 498, 514, 4, 0, 0, 0, 0, 0, 0, 0, 491, 536, 477, 509, 521, 489, 546, 494, 8, 5, 0, 0, 0, 0, 0, 0, 496, 503, 513, 509, 527, 470, 529, 535, 0, 0, 0, 0, 0, 0, 0, 0, 506, 536, 482, 493, 507, 500, 529, 542, 5, 0, 0, 0, 0, 0, 0, 0, 502, 513, 489, 519, 495, 516, 513, 478, 0, 0, 0, 0, 0, 0, 0, 0, 495, 530, 522, 521, 509, 511, 543, 508, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4192, 4018, 4136, 4076, 4082, 4100, 4025, 4139] avg=4096.0000 max=4192 max_rank=0 min=4018 min_rank=1 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16346 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4224, 3968, 4224, 4096, 4224, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3968, 4224, 4096, 4352, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[553, 517, 534, 528, 512, 530, 507, 490, 502, 555, 515, 506, 533, 527, 535, 476, 481, 510, 560, 552, 516, 521, 534, 530, 530, 502, 485, 492, 536, 498, 488, 547, 503, 533, 496, 466, 503, 514, 518, 495, 524, 491, 504, 492, 524, 525, 507, 479, 493, 505, 521, 499, 517, 477, 492, 493, 551, 487, 473, 528, 537, 506, 523, 490] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[518, 539, 551, 488, 544, 506, 492, 513, 523, 541, 490, 489, 508, 512, 502, 474, 538, 547, 498, 516, 519, 517, 553, 489, 525, 518, 495, 514, 538, 508, 490, 509, 546, 526, 498, 482, 535, 554, 513, 534, 500, 492, 505, 500, 501, 485, 482, 521, 494, 525, 486, 503, 532, 492, 529, 519, 481, 522, 531, 507, 494, 524, 471, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[509, 489, 537, 532, 524, 526, 514, 520, 509, 497, 531, 501, 530, 568, 516, 497, 533, 519, 520, 504, 481, 495, 512, 507, 497, 493, 510, 522, 530, 503, 475, 534, 499, 483, 535, 493, 522, 478, 543, 523, 509, 506, 487, 507, 528, 511, 516, 516, 499, 525, 500, 492, 507, 538, 526, 519, 549, 513, 506, 482, 513, 504, 498, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[571, 522, 540, 531, 533, 492, 495, 546, 537, 538, 536, 529, 503, 509, 518, 501, 537, 516, 515, 497, 491, 514, 491, 513, 496, 519, 488, 482, 518, 500, 501, 500, 486, 504, 512, 531, 499, 475, 502, 528, 514, 502, 514, 514, 497, 534, 513, 487, 513, 511, 501, 472, 540, 522, 511, 487, 518, 508, 514, 502, 517, 534, 487, 540] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[544, 500, 500, 490, 494, 515, 516, 481, 497, 515, 524, 512, 551, 506, 516, 503, 540, 545, 483, 492, 508, 472, 552, 499, 503, 532, 522, 521, 508, 537, 488, 544, 529, 556, 528, 482, 517, 474, 519, 530, 513, 487, 488, 539, 484, 495, 512, 537, 499, 483, 508, 497, 533, 523, 495, 525, 480, 494, 510, 539, 515, 527, 533, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[519, 470, 542, 537, 531, 500, 485, 480, 487, 510, 500, 505, 511, 484, 495, 509, 497, 483, 517, 505, 523, 502, 518, 513, 517, 561, 504, 519, 480, 502, 540, 512, 494, 534, 490, 501, 530, 550, 527, 514, 522, 544, 497, 512, 446, 526, 521, 539, 517, 505, 523, 504, 505, 544, 505, 533, 532, 499, 544, 478, 503, 485, 545, 541] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[514, 516, 552, 506, 511, 508, 532, 485, 517, 514, 514, 512, 507, 507, 519, 536, 537, 537, 524, 506, 500, 502, 516, 485, 503, 513, 513, 512, 496, 482, 507, 521, 484, 539, 474, 522, 506, 496, 519, 500, 508, 517, 545, 486, 514, 536, 519, 517, 507, 517, 484, 530, 470, 506, 488, 549, 521, 504, 547, 485, 528, 547, 496, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[517, 527, 512, 550, 510, 521, 505, 494, 543, 492, 490, 503, 475, 529, 500, 505, 481, 525, 564, 508, 517, 489, 530, 475, 490, 496, 517, 546, 505, 539, 545, 527, 488, 510, 541, 529, 516, 498, 516, 503, 504, 502, 511, 481, 489, 508, 494, 524, 480, 502, 532, 513, 546, 535, 485, 529, 489, 496, 491, 542, 541, 533, 490, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[553, 517, 497, 528, 512, 530, 507, 490, 0, 0, 0, 0, 0, 0, 0, 0, 502, 552, 515, 506, 533, 527, 535, 476, 0, 0, 0, 0, 0, 0, 0, 0, 481, 510, 560, 552, 516, 521, 521, 530, 0, 0, 0, 0, 0, 0, 0, 0, 530, 502, 485, 492, 536, 498, 488, 547, 3, 0, 0, 0, 0, 0, 0, 0, 503, 533, 496, 466, 503, 514, 518, 495, 2, 0, 0, 0, 0, 0, 0, 0, 524, 491, 504, 492, 524, 525, 507, 479, 33, 0, 0, 0, 0, 0, 0, 0, 493, 505, 521, 499, 517, 477, 492, 493, 13, 4, 2, 0, 0, 0, 0, 0, 551, 487, 473, 528, 537, 502, 523, 490, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4134, 4146, 4191, 4081, 4030, 4079, 4016, 4091] avg=4096.0000 max=4191 max_rank=2 min=4016 min_rank=6 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16216 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[518, 539, 513, 488, 544, 506, 492, 513, 0, 0, 0, 0, 0, 0, 0, 0, 523, 539, 490, 489, 508, 512, 502, 474, 0, 0, 0, 0, 0, 0, 0, 0, 538, 547, 498, 516, 519, 517, 540, 489, 0, 0, 0, 0, 0, 0, 0, 0, 525, 518, 495, 514, 538, 508, 490, 509, 2, 0, 0, 0, 0, 0, 0, 0, 546, 526, 498, 482, 535, 554, 513, 534, 2, 0, 0, 0, 0, 0, 0, 0, 500, 492, 505, 500, 501, 485, 482, 521, 34, 0, 0, 0, 0, 0, 0, 0, 494, 525, 486, 503, 532, 492, 529, 519, 13, 4, 2, 0, 0, 0, 0, 0, 481, 522, 531, 507, 494, 520, 471, 520, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4037, 4164, 4099, 4190, 4020, 4099, 4046] avg=4096.0000 max=4190 max_rank=4 min=4020 min_rank=5 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16355 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[544, 500, 466, 490, 494, 515, 516, 481, 0, 0, 0, 0, 0, 0, 0, 0, 497, 513, 524, 512, 551, 506, 516, 503, 0, 0, 0, 0, 0, 0, 0, 0, 540, 545, 483, 492, 508, 472, 539, 499, 0, 0, 0, 0, 0, 0, 0, 0, 503, 532, 522, 521, 508, 537, 488, 544, 2, 0, 0, 0, 0, 0, 0, 0, 529, 556, 528, 482, 517, 474, 519, 530, 2, 0, 0, 0, 0, 0, 0, 0, 513, 487, 488, 539, 484, 495, 512, 537, 30, 0, 0, 0, 0, 0, 0, 0, 499, 483, 508, 497, 533, 523, 495, 525, 13, 4, 2, 0, 0, 0, 0, 0, 480, 494, 510, 539, 515, 523, 533, 507, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4006, 4122, 4078, 4157, 4137, 4085, 4082, 4101] avg=4096.0000 max=4157 max_rank=3 min=4006 min_rank=0 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16405 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[509, 489, 500, 532, 524, 526, 514, 520, 0, 0, 0, 0, 0, 0, 0, 0, 509, 495, 531, 501, 530, 568, 516, 497, 0, 0, 0, 0, 0, 0, 0, 0, 533, 519, 520, 504, 481, 495, 500, 507, 0, 0, 0, 0, 0, 0, 0, 0, 497, 493, 510, 522, 530, 503, 475, 534, 2, 0, 0, 0, 0, 0, 0, 0, 499, 483, 535, 493, 522, 478, 543, 523, 2, 0, 0, 0, 0, 0, 0, 0, 509, 506, 487, 507, 528, 511, 516, 516, 33, 0, 0, 0, 0, 0, 0, 0, 499, 525, 500, 492, 507, 538, 526, 519, 12, 4, 2, 0, 0, 0, 0, 0, 549, 513, 506, 482, 513, 500, 498, 506, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4147, 4059, 4066, 4078, 4113, 4124, 4067] avg=4096.0000 max=4147 max_rank=1 min=4059 min_rank=2 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[571, 522, 497, 531, 533, 492, 495, 546, 0, 0, 0, 0, 0, 0, 0, 0, 537, 525, 536, 529, 503, 509, 518, 501, 0, 0, 0, 0, 0, 0, 0, 0, 537, 516, 515, 497, 491, 514, 475, 513, 0, 0, 0, 0, 0, 0, 0, 0, 496, 519, 488, 482, 518, 500, 501, 500, 8, 0, 0, 0, 0, 0, 0, 0, 486, 504, 512, 531, 499, 475, 502, 528, 4, 5, 0, 0, 0, 0, 0, 0, 514, 502, 514, 514, 497, 534, 513, 487, 36, 0, 0, 0, 0, 0, 0, 0, 513, 511, 501, 472, 540, 522, 511, 487, 16, 7, 4, 0, 0, 0, 0, 0, 518, 508, 514, 502, 517, 526, 487, 540, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4187, 4158, 4058, 4012, 4046, 4111, 4084, 4112] avg=4096.0000 max=4187 max_rank=0 min=4012 min_rank=3 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[514, 516, 514, 506, 511, 508, 532, 485, 0, 0, 0, 0, 0, 0, 0, 0, 517, 512, 514, 512, 507, 507, 519, 536, 0, 0, 0, 0, 0, 0, 0, 0, 537, 537, 524, 506, 500, 502, 504, 485, 0, 0, 0, 0, 0, 0, 0, 0, 503, 513, 513, 512, 496, 482, 507, 521, 2, 0, 0, 0, 0, 0, 0, 0, 484, 539, 474, 522, 506, 496, 519, 500, 2, 0, 0, 0, 0, 0, 0, 0, 508, 517, 545, 486, 514, 536, 519, 517, 34, 0, 0, 0, 0, 0, 0, 0, 507, 517, 484, 530, 470, 506, 488, 549, 12, 4, 2, 0, 0, 0, 0, 0, 521, 504, 547, 485, 528, 543, 496, 503, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4124, 4095, 4049, 4042, 4176, 4069, 4127] avg=4096.0000 max=4176 max_rank=5 min=4042 min_rank=4 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16414 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[519, 470, 505, 537, 531, 500, 485, 480, 0, 0, 0, 0, 0, 0, 0, 0, 487, 508, 500, 505, 511, 484, 495, 509, 0, 0, 0, 0, 0, 0, 0, 0, 497, 483, 517, 505, 523, 502, 506, 513, 0, 0, 0, 0, 0, 0, 0, 0, 517, 561, 504, 519, 480, 502, 540, 512, 2, 0, 0, 0, 0, 0, 0, 0, 494, 534, 490, 501, 530, 550, 527, 514, 2, 0, 0, 0, 0, 0, 0, 0, 522, 544, 497, 512, 446, 526, 521, 539, 33, 0, 0, 0, 0, 0, 0, 0, 517, 505, 523, 504, 505, 544, 505, 533, 12, 4, 2, 0, 0, 0, 0, 0, 532, 499, 544, 478, 503, 481, 545, 541, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4027, 3999, 4046, 4137, 4142, 4140, 4154, 4123] avg=4096.0000 max=4154 max_rank=6 min=3999 min_rank=1 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16559 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[517, 527, 477, 550, 510, 521, 505, 494, 0, 0, 0, 0, 0, 0, 0, 0, 543, 490, 490, 503, 475, 529, 500, 505, 0, 0, 0, 0, 0, 0, 0, 0, 481, 525, 564, 508, 517, 489, 518, 475, 0, 0, 0, 0, 0, 0, 0, 0, 490, 496, 517, 546, 505, 539, 545, 527, 2, 0, 0, 0, 0, 0, 0, 0, 488, 510, 541, 529, 516, 498, 516, 503, 2, 0, 0, 0, 0, 0, 0, 0, 504, 502, 511, 481, 489, 508, 494, 524, 31, 0, 0, 0, 0, 0, 0, 0, 480, 502, 532, 513, 546, 535, 485, 529, 12, 4, 2, 0, 0, 0, 0, 0, 489, 496, 491, 542, 541, 529, 490, 523, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4035, 4077, 4167, 4103, 4044, 4140, 4101] avg=4096.0000 max=4167 max_rank=3 min=4035 min_rank=1 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4352, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[544, 552, 503, 507, 514, 505, 491, 505, 539, 534, 503, 527, 513, 501, 504, 488, 569, 502, 529, 501, 488, 521, 493, 511, 513, 497, 509, 516, 534, 501, 521, 484, 505, 505, 524, 513, 491, 501, 481, 500, 531, 536, 523, 503, 524, 473, 508, 502, 524, 531, 509, 529, 482, 503, 471, 551, 492, 506, 470, 547, 541, 520, 515, 538] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[554, 527, 484, 524, 575, 522, 528, 513, 501, 511, 496, 477, 515, 508, 465, 502, 488, 488, 474, 474, 529, 523, 481, 531, 547, 512, 505, 517, 518, 491, 525, 526, 542, 553, 466, 519, 520, 502, 548, 469, 504, 524, 479, 479, 497, 514, 528, 498, 526, 508, 509, 541, 521, 535, 509, 530, 518, 524, 501, 501, 529, 502, 521, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[520, 521, 539, 522, 545, 519, 502, 498, 492, 523, 484, 519, 506, 505, 505, 547, 541, 508, 496, 535, 543, 511, 505, 554, 518, 529, 514, 511, 460, 512, 513, 511, 513, 480, 495, 467, 554, 515, 494, 547, 515, 526, 521, 511, 520, 476, 492, 515, 539, 487, 490, 525, 487, 538, 580, 455, 496, 483, 515, 531, 511, 495, 477, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[499, 541, 505, 536, 522, 505, 537, 517, 530, 508, 559, 487, 503, 505, 499, 511, 491, 485, 499, 535, 537, 520, 483, 553, 508, 470, 529, 528, 537, 482, 487, 514, 494, 496, 486, 534, 504, 504, 502, 512, 497, 540, 533, 456, 547, 496, 527, 550, 520, 487, 508, 550, 512, 523, 553, 508, 496, 479, 509, 516, 485, 499, 488, 535] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[560, 481, 502, 487, 542, 512, 519, 515, 545, 543, 526, 509, 492, 486, 539, 514, 497, 498, 537, 473, 481, 504, 507, 479, 472, 493, 506, 555, 527, 526, 503, 504, 506, 523, 527, 553, 483, 518, 567, 488, 523, 523, 507, 522, 502, 505, 494, 518, 512, 530, 483, 546, 518, 501, 517, 514, 543, 509, 499, 489, 497, 515, 499, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[499, 497, 492, 521, 488, 507, 491, 492, 539, 515, 533, 521, 503, 516, 514, 482, 479, 508, 546, 497, 512, 517, 510, 506, 532, 498, 500, 534, 517, 519, 499, 489, 533, 526, 524, 554, 486, 494, 503, 492, 510, 507, 556, 472, 514, 535, 476, 498, 527, 529, 523, 520, 539, 533, 512, 489, 536, 532, 531, 503, 520, 487, 531, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[514, 509, 545, 509, 530, 506, 541, 522, 527, 504, 525, 486, 509, 472, 516, 539, 494, 541, 517, 502, 523, 523, 520, 501, 500, 498, 486, 501, 524, 491, 506, 491, 526, 539, 527, 472, 525, 525, 496, 501, 498, 498, 518, 505, 559, 525, 501, 504, 515, 530, 534, 509, 546, 496, 516, 510, 504, 503, 513, 476, 485, 522, 507, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[549, 553, 471, 536, 485, 506, 490, 504, 472, 482, 543, 505, 562, 492, 514, 531, 501, 520, 539, 494, 516, 484, 506, 537, 554, 504, 529, 521, 481, 524, 518, 501, 502, 524, 547, 521, 535, 546, 500, 567, 467, 515, 500, 498, 489, 483, 520, 475, 507, 544, 485, 482, 540, 531, 552, 542, 505, 493, 508, 492, 485, 496, 515, 478] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[509, 552, 503, 507, 514, 505, 491, 505, 0, 0, 0, 0, 0, 0, 0, 0, 539, 534, 503, 527, 513, 501, 504, 488, 5, 0, 0, 0, 0, 0, 0, 0, 569, 502, 529, 501, 488, 521, 493, 511, 6, 6, 0, 0, 0, 0, 0, 0, 513, 497, 509, 516, 534, 501, 521, 484, 16, 0, 0, 0, 0, 0, 0, 0, 505, 494, 524, 513, 491, 501, 481, 500, 0, 0, 0, 0, 0, 0, 0, 0, 531, 536, 523, 503, 524, 473, 508, 502, 13, 10, 0, 0, 0, 0, 0, 0, 524, 531, 509, 529, 482, 503, 427, 551, 0, 0, 0, 0, 0, 0, 0, 0, 492, 506, 470, 547, 541, 520, 515, 538, 34, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4114, 4126, 4091, 4009, 4123, 4056, 4163] avg=4096.0000 max=4163 max_rank=7 min=4009 min_rank=4 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16351 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[518, 527, 484, 524, 575, 522, 528, 513, 0, 0, 0, 0, 0, 0, 0, 0, 501, 511, 496, 477, 515, 508, 465, 502, 6, 0, 0, 0, 0, 0, 0, 0, 488, 488, 474, 474, 529, 523, 481, 531, 7, 6, 0, 0, 0, 0, 0, 0, 547, 512, 505, 517, 518, 491, 525, 526, 16, 0, 0, 0, 0, 0, 0, 0, 542, 540, 466, 519, 520, 502, 548, 469, 0, 0, 0, 0, 0, 0, 0, 0, 504, 524, 479, 479, 497, 514, 528, 498, 14, 11, 0, 0, 0, 0, 0, 0, 526, 508, 509, 541, 521, 535, 461, 530, 0, 0, 0, 0, 0, 0, 0, 0, 518, 524, 501, 501, 529, 502, 521, 520, 37, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4191, 3981, 4001, 4157, 4106, 4048, 4131, 4153] avg=4096.0000 max=4191 max_rank=0 min=3981 min_rank=1 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16438 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[487, 521, 539, 522, 545, 519, 502, 498, 0, 0, 0, 0, 0, 0, 0, 0, 492, 523, 484, 519, 506, 505, 505, 547, 5, 0, 0, 0, 0, 0, 0, 0, 541, 508, 496, 535, 543, 511, 505, 554, 6, 5, 0, 0, 0, 0, 0, 0, 518, 529, 514, 511, 460, 512, 513, 511, 15, 0, 0, 0, 0, 0, 0, 0, 513, 469, 495, 467, 554, 515, 494, 547, 0, 0, 0, 0, 0, 0, 0, 0, 515, 526, 521, 511, 520, 476, 492, 515, 13, 13, 0, 0, 0, 0, 0, 0, 539, 487, 490, 525, 487, 538, 525, 455, 0, 0, 0, 0, 0, 0, 0, 0, 496, 483, 515, 531, 511, 495, 477, 510, 42, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4133, 4086, 4204, 4083, 4054, 4102, 4046, 4060] avg=4096.0000 max=4204 max_rank=2 min=4046 min_rank=6 imbalance_ratio=1.026367 max_load_reduction=28564 max_load_reduction_pct=0.871704 moved_to_echo=16262 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[468, 541, 505, 536, 522, 505, 537, 517, 0, 0, 0, 0, 0, 0, 0, 0, 530, 508, 559, 487, 503, 505, 499, 511, 5, 0, 0, 0, 0, 0, 0, 0, 491, 485, 499, 535, 537, 520, 483, 553, 6, 5, 0, 0, 0, 0, 0, 0, 508, 470, 529, 528, 537, 482, 487, 514, 14, 0, 0, 0, 0, 0, 0, 0, 494, 485, 486, 534, 504, 504, 502, 512, 0, 0, 0, 0, 0, 0, 0, 0, 497, 540, 533, 456, 547, 496, 527, 550, 12, 12, 0, 0, 0, 0, 0, 0, 520, 487, 508, 550, 512, 523, 501, 508, 0, 0, 0, 0, 0, 0, 0, 0, 496, 479, 509, 516, 485, 499, 488, 535, 40, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4131, 4107, 4114, 4069, 4021, 4170, 4109, 4047] avg=4096.0000 max=4170 max_rank=5 min=4021 min_rank=4 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[524, 481, 502, 487, 542, 512, 519, 515, 0, 0, 0, 0, 0, 0, 0, 0, 545, 543, 526, 509, 492, 486, 539, 514, 5, 0, 0, 0, 0, 0, 0, 0, 497, 498, 537, 473, 481, 504, 507, 479, 6, 6, 0, 0, 0, 0, 0, 0, 472, 493, 506, 555, 527, 526, 503, 504, 16, 0, 0, 0, 0, 0, 0, 0, 506, 512, 527, 553, 483, 518, 567, 488, 0, 0, 0, 0, 0, 0, 0, 0, 523, 523, 507, 522, 502, 505, 494, 518, 14, 11, 0, 0, 0, 0, 0, 0, 512, 530, 483, 546, 518, 501, 469, 514, 0, 0, 0, 0, 0, 0, 0, 0, 543, 509, 499, 489, 497, 515, 499, 503, 37, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4159, 3988, 4102, 4154, 4119, 4073, 4091] avg=4096.0000 max=4159 max_rank=1 min=3988 min_rank=2 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16437 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[456, 497, 492, 521, 488, 507, 491, 492, 0, 0, 0, 0, 0, 0, 0, 0, 539, 515, 533, 521, 503, 516, 514, 482, 9, 0, 0, 0, 0, 0, 0, 0, 479, 508, 546, 497, 512, 517, 510, 506, 10, 9, 0, 0, 0, 0, 0, 0, 532, 498, 500, 534, 517, 519, 499, 489, 18, 0, 0, 0, 0, 0, 0, 0, 533, 507, 524, 554, 486, 494, 503, 492, 0, 0, 0, 0, 0, 0, 0, 0, 510, 507, 556, 472, 514, 535, 476, 498, 16, 17, 0, 0, 0, 0, 0, 0, 527, 529, 523, 520, 539, 533, 454, 489, 0, 0, 0, 0, 0, 0, 0, 0, 536, 532, 531, 503, 520, 487, 531, 503, 41, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3944, 4132, 4094, 4106, 4093, 4101, 4114, 4184] avg=4096.0000 max=4184 max_rank=7 min=3944 min_rank=0 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16492 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[481, 509, 545, 509, 530, 506, 541, 522, 0, 0, 0, 0, 0, 0, 0, 0, 527, 504, 525, 486, 509, 472, 516, 539, 5, 0, 0, 0, 0, 0, 0, 0, 494, 541, 517, 502, 523, 523, 520, 501, 6, 5, 0, 0, 0, 0, 0, 0, 500, 498, 486, 501, 524, 491, 506, 491, 15, 0, 0, 0, 0, 0, 0, 0, 526, 528, 527, 472, 525, 525, 496, 501, 0, 0, 0, 0, 0, 0, 0, 0, 498, 498, 518, 505, 559, 525, 501, 504, 13, 11, 0, 0, 0, 0, 0, 0, 515, 530, 534, 509, 546, 496, 468, 510, 0, 0, 0, 0, 0, 0, 0, 0, 504, 503, 513, 476, 485, 522, 507, 511, 37, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4143, 4083, 4132, 4012, 4100, 4132, 4108, 4058] avg=4096.0000 max=4143 max_rank=0 min=4012 min_rank=3 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[513, 553, 471, 536, 485, 506, 490, 504, 0, 0, 0, 0, 0, 0, 0, 0, 472, 482, 543, 505, 562, 492, 514, 531, 5, 0, 0, 0, 0, 0, 0, 0, 501, 520, 539, 494, 516, 484, 506, 537, 6, 6, 0, 0, 0, 0, 0, 0, 554, 504, 529, 521, 481, 524, 518, 501, 16, 0, 0, 0, 0, 0, 0, 0, 502, 513, 547, 521, 535, 546, 500, 567, 0, 0, 0, 0, 0, 0, 0, 0, 467, 515, 500, 498, 489, 483, 520, 475, 14, 12, 0, 0, 0, 0, 0, 0, 507, 544, 485, 482, 540, 531, 500, 542, 0, 0, 0, 0, 0, 0, 0, 0, 505, 493, 508, 492, 485, 496, 515, 478, 40, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4058, 4106, 4109, 4148, 4231, 3973, 4131, 4012] avg=4096.0000 max=4231 max_rank=4 min=3973 min_rank=5 imbalance_ratio=1.032959 max_load_reduction=28537 max_load_reduction_pct=0.870880 moved_to_echo=16347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3968, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4224, 3840, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 3968, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[540, 511, 510, 507, 517, 541, 501, 525, 521, 540, 525, 503, 486, 454, 520, 482, 536, 538, 483, 522, 483, 537, 551, 500, 499, 501, 502, 509, 500, 532, 509, 502, 514, 474, 474, 524, 548, 502, 542, 529, 516, 487, 474, 533, 521, 520, 526, 485, 506, 484, 551, 504, 523, 518, 502, 545, 491, 512, 503, 505, 535, 514, 510, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[520, 498, 518, 527, 512, 488, 532, 489, 492, 509, 526, 549, 525, 498, 495, 523, 515, 522, 558, 481, 492, 554, 479, 529, 528, 457, 509, 529, 544, 517, 537, 556, 492, 494, 540, 499, 508, 496, 518, 498, 528, 523, 472, 524, 518, 457, 503, 482, 504, 527, 495, 512, 522, 460, 536, 516, 528, 488, 515, 502, 544, 529, 505, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[505, 497, 515, 515, 557, 488, 463, 480, 519, 509, 515, 500, 530, 519, 486, 554, 481, 528, 480, 509, 502, 518, 528, 524, 513, 530, 517, 549, 436, 512, 518, 513, 525, 503, 498, 497, 538, 481, 537, 489, 503, 533, 510, 479, 550, 476, 585, 504, 512, 488, 521, 509, 489, 547, 543, 525, 527, 507, 494, 502, 502, 532, 509, 543] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[527, 516, 502, 505, 501, 476, 506, 553, 507, 488, 531, 531, 505, 525, 501, 498, 524, 514, 481, 556, 521, 514, 517, 493, 555, 513, 516, 499, 496, 489, 528, 503, 537, 518, 524, 519, 482, 523, 503, 526, 531, 486, 482, 507, 514, 514, 507, 512, 537, 503, 526, 496, 487, 511, 513, 493, 542, 531, 528, 479, 517, 515, 515, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[475, 504, 512, 515, 514, 511, 538, 506, 512, 499, 529, 524, 484, 484, 496, 551, 499, 512, 559, 504, 515, 523, 579, 507, 501, 553, 512, 479, 534, 502, 514, 519, 525, 463, 535, 493, 520, 488, 518, 478, 493, 459, 492, 524, 516, 478, 494, 519, 501, 518, 525, 532, 526, 484, 514, 525, 530, 522, 521, 513, 534, 525, 509, 528] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[471, 543, 498, 508, 508, 522, 535, 492, 498, 520, 502, 480, 502, 542, 517, 517, 523, 507, 553, 511, 531, 485, 496, 487, 508, 490, 525, 532, 512, 536, 490, 525, 493, 520, 521, 512, 523, 514, 503, 494, 484, 545, 513, 494, 496, 505, 516, 506, 503, 525, 536, 520, 515, 582, 481, 488, 490, 536, 491, 534, 540, 514, 486, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[540, 520, 506, 538, 494, 500, 533, 500, 499, 534, 535, 511, 481, 501, 483, 506, 500, 529, 474, 490, 496, 493, 538, 526, 510, 513, 534, 497, 522, 533, 499, 525, 515, 536, 511, 521, 494, 512, 515, 509, 520, 501, 507, 536, 479, 503, 543, 493, 517, 525, 513, 501, 516, 474, 509, 532, 489, 529, 515, 521, 527, 519, 503, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[491, 475, 553, 543, 509, 502, 502, 530, 528, 552, 514, 470, 503, 536, 499, 505, 530, 513, 518, 511, 514, 517, 533, 511, 530, 518, 503, 501, 510, 519, 491, 521, 493, 478, 527, 503, 516, 528, 516, 526, 539, 467, 530, 457, 505, 496, 527, 533, 509, 507, 526, 496, 510, 479, 490, 537, 510, 474, 506, 533, 553, 501, 524, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[540, 511, 510, 507, 517, 541, 501, 525, 4, 0, 0, 0, 0, 0, 0, 0, 521, 540, 525, 503, 486, 454, 520, 482, 8, 1, 0, 0, 0, 0, 0, 0, 536, 538, 483, 522, 483, 537, 528, 500, 0, 0, 0, 0, 0, 0, 0, 0, 499, 501, 502, 509, 500, 532, 509, 490, 0, 0, 0, 0, 0, 0, 0, 0, 514, 474, 474, 524, 548, 502, 542, 529, 10, 4, 0, 0, 0, 0, 0, 0, 516, 487, 474, 533, 521, 520, 526, 485, 29, 13, 0, 0, 0, 0, 0, 0, 506, 484, 546, 504, 523, 518, 502, 545, 0, 0, 0, 0, 0, 0, 0, 0, 491, 512, 503, 505, 506, 514, 510, 509, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4156, 4040, 4127, 4042, 4121, 4104, 4128, 4050] avg=4096.0000 max=4156 max_rank=0 min=4040 min_rank=1 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16403 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[520, 498, 518, 527, 512, 488, 532, 489, 4, 0, 0, 0, 0, 0, 0, 0, 492, 509, 526, 549, 525, 498, 495, 523, 9, 1, 0, 0, 0, 0, 0, 0, 515, 522, 558, 481, 492, 554, 459, 529, 0, 0, 0, 0, 0, 0, 0, 0, 528, 457, 509, 529, 544, 517, 537, 543, 0, 0, 0, 0, 0, 0, 0, 0, 492, 494, 540, 499, 508, 496, 518, 498, 9, 4, 0, 0, 0, 0, 0, 0, 528, 523, 472, 524, 518, 457, 503, 482, 29, 11, 0, 0, 0, 0, 0, 0, 504, 527, 490, 512, 522, 460, 536, 516, 0, 0, 0, 0, 0, 0, 0, 0, 528, 488, 515, 502, 515, 529, 505, 525, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4127, 4110, 4164, 4058, 4047, 4067, 4107] avg=4096.0000 max=4164 max_rank=3 min=4047 min_rank=5 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16279 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[527, 516, 502, 505, 501, 476, 506, 553, 4, 0, 0, 0, 0, 0, 0, 0, 507, 488, 531, 531, 505, 525, 501, 498, 8, 1, 0, 0, 0, 0, 0, 0, 524, 514, 481, 556, 521, 514, 496, 493, 0, 0, 0, 0, 0, 0, 0, 0, 555, 513, 516, 499, 496, 489, 528, 491, 0, 0, 0, 0, 0, 0, 0, 0, 537, 518, 524, 519, 482, 523, 503, 526, 9, 4, 0, 0, 0, 0, 0, 0, 531, 486, 482, 507, 514, 514, 507, 512, 28, 12, 0, 0, 0, 0, 0, 0, 537, 503, 521, 496, 487, 511, 513, 493, 0, 0, 0, 0, 0, 0, 0, 0, 542, 531, 528, 479, 489, 515, 515, 499, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4095, 4099, 4087, 4145, 4093, 4061, 4098] avg=4096.0000 max=4145 max_rank=4 min=4061 min_rank=6 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[505, 497, 515, 515, 557, 488, 463, 480, 10, 0, 0, 0, 0, 0, 0, 0, 519, 509, 515, 500, 530, 519, 486, 554, 12, 8, 0, 0, 0, 0, 0, 0, 481, 528, 480, 509, 502, 518, 498, 524, 0, 0, 0, 0, 0, 0, 0, 0, 513, 530, 517, 549, 436, 512, 518, 494, 0, 0, 0, 0, 0, 0, 0, 0, 525, 503, 498, 497, 538, 481, 537, 489, 13, 7, 0, 0, 0, 0, 0, 0, 503, 533, 510, 479, 550, 476, 585, 504, 29, 17, 0, 0, 0, 0, 0, 0, 512, 488, 503, 509, 489, 547, 543, 525, 0, 0, 0, 0, 0, 0, 0, 0, 527, 507, 494, 502, 473, 532, 509, 543, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4030, 4152, 4040, 4069, 4088, 4186, 4116, 4087] avg=4096.0000 max=4186 max_rank=5 min=4030 min_rank=0 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16477 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[475, 504, 512, 515, 514, 511, 538, 506, 4, 0, 0, 0, 0, 0, 0, 0, 512, 499, 529, 524, 484, 484, 496, 551, 9, 1, 0, 0, 0, 0, 0, 0, 499, 512, 559, 504, 515, 523, 554, 507, 0, 0, 0, 0, 0, 0, 0, 0, 501, 553, 512, 479, 534, 502, 514, 506, 0, 0, 0, 0, 0, 0, 0, 0, 525, 463, 535, 493, 520, 488, 518, 478, 11, 4, 0, 0, 0, 0, 0, 0, 493, 459, 492, 524, 516, 478, 494, 519, 29, 14, 0, 0, 0, 0, 0, 0, 501, 518, 520, 532, 526, 484, 514, 525, 0, 0, 0, 0, 0, 0, 0, 0, 530, 522, 521, 513, 505, 525, 509, 528, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4089, 4173, 4101, 4035, 4018, 4120, 4153] avg=4096.0000 max=4173 max_rank=2 min=4018 min_rank=5 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16326 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[540, 520, 506, 538, 494, 500, 533, 500, 4, 0, 0, 0, 0, 0, 0, 0, 499, 534, 535, 511, 481, 501, 483, 506, 9, 1, 0, 0, 0, 0, 0, 0, 500, 529, 474, 490, 496, 493, 515, 526, 0, 0, 0, 0, 0, 0, 0, 0, 510, 513, 534, 497, 522, 533, 499, 512, 0, 0, 0, 0, 0, 0, 0, 0, 515, 536, 511, 521, 494, 512, 515, 509, 10, 4, 0, 0, 0, 0, 0, 0, 520, 501, 507, 536, 479, 503, 543, 493, 28, 13, 0, 0, 0, 0, 0, 0, 517, 525, 508, 501, 516, 474, 509, 532, 0, 0, 0, 0, 0, 0, 0, 0, 489, 529, 515, 521, 499, 519, 503, 523, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4135, 4060, 4023, 4120, 4127, 4123, 4082, 4098] avg=4096.0000 max=4135 max_rank=0 min=4023 min_rank=2 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16430 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[471, 543, 498, 508, 508, 522, 535, 492, 4, 0, 0, 0, 0, 0, 0, 0, 498, 520, 502, 480, 502, 542, 517, 517, 9, 1, 0, 0, 0, 0, 0, 0, 523, 507, 553, 511, 531, 485, 475, 487, 0, 0, 0, 0, 0, 0, 0, 0, 508, 490, 525, 532, 512, 536, 490, 512, 0, 0, 0, 0, 0, 0, 0, 0, 493, 520, 521, 512, 523, 514, 503, 494, 9, 4, 0, 0, 0, 0, 0, 0, 484, 545, 513, 494, 496, 505, 516, 506, 29, 12, 0, 0, 0, 0, 0, 0, 503, 525, 531, 520, 515, 582, 481, 488, 0, 0, 0, 0, 0, 0, 0, 0, 490, 536, 491, 534, 511, 514, 486, 522, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4088, 4072, 4105, 4093, 4100, 4145, 4084] avg=4096.0000 max=4145 max_rank=6 min=4072 min_rank=2 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16422 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[491, 475, 553, 543, 509, 502, 502, 530, 4, 0, 0, 0, 0, 0, 0, 0, 528, 552, 514, 470, 503, 536, 499, 505, 9, 1, 0, 0, 0, 0, 0, 0, 530, 513, 518, 511, 514, 517, 510, 511, 0, 0, 0, 0, 0, 0, 0, 0, 530, 518, 503, 501, 510, 519, 491, 508, 0, 0, 0, 0, 0, 0, 0, 0, 493, 478, 527, 503, 516, 528, 516, 526, 10, 4, 0, 0, 0, 0, 0, 0, 539, 467, 530, 457, 505, 496, 527, 533, 30, 13, 0, 0, 0, 0, 0, 0, 509, 507, 521, 496, 510, 479, 490, 537, 0, 0, 0, 0, 0, 0, 0, 0, 510, 474, 506, 533, 523, 501, 524, 520, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4117, 4124, 4080, 4101, 4097, 4049, 4091] avg=4096.0000 max=4124 max_rank=2 min=4049 min_rank=6 imbalance_ratio=1.006836 max_load_reduction=28644 max_load_reduction_pct=0.874146 moved_to_echo=16338 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 3968, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[534, 509, 511, 516, 500, 476, 495, 514, 568, 510, 556, 538, 546, 491, 509, 504, 543, 488, 506, 487, 518, 543, 488, 493, 519, 514, 546, 520, 517, 521, 515, 501, 511, 491, 489, 496, 517, 489, 512, 526, 525, 496, 516, 527, 474, 510, 494, 519, 540, 531, 538, 500, 486, 558, 491, 504, 497, 503, 512, 511, 520, 522, 460, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[489, 529, 466, 519, 515, 509, 524, 510, 524, 469, 537, 569, 528, 559, 484, 519, 482, 533, 509, 510, 505, 489, 464, 549, 501, 536, 521, 546, 507, 492, 524, 527, 490, 503, 481, 534, 508, 506, 533, 532, 518, 503, 517, 517, 510, 507, 490, 511, 470, 491, 465, 495, 516, 500, 538, 533, 543, 487, 537, 522, 498, 522, 551, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[463, 479, 518, 492, 531, 513, 525, 538, 509, 482, 494, 540, 508, 514, 472, 505, 488, 515, 551, 508, 501, 513, 502, 539, 486, 507, 516, 477, 495, 508, 509, 507, 509, 504, 552, 518, 551, 498, 517, 522, 516, 490, 497, 518, 502, 524, 526, 517, 483, 490, 532, 501, 534, 517, 526, 553, 495, 495, 525, 494, 538, 577, 493, 549] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[497, 474, 529, 483, 541, 543, 494, 522, 505, 504, 521, 500, 463, 503, 471, 566, 515, 522, 491, 527, 500, 511, 485, 502, 513, 531, 511, 541, 506, 512, 560, 504, 502, 558, 543, 509, 488, 517, 477, 514, 513, 521, 501, 536, 493, 506, 474, 506, 542, 510, 511, 521, 530, 504, 542, 522, 513, 508, 505, 523, 504, 543, 478, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[528, 538, 517, 505, 529, 524, 505, 482, 531, 553, 540, 503, 491, 496, 470, 510, 534, 539, 469, 518, 529, 503, 511, 489, 498, 495, 512, 508, 486, 523, 483, 544, 539, 526, 536, 513, 522, 495, 531, 468, 534, 473, 507, 497, 534, 524, 501, 547, 532, 473, 557, 476, 494, 537, 507, 492, 512, 534, 513, 525, 495, 500, 516, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[474, 497, 513, 524, 528, 504, 523, 546, 487, 503, 481, 560, 508, 562, 505, 542, 547, 520, 484, 523, 545, 545, 507, 524, 500, 506, 520, 501, 507, 500, 559, 503, 538, 489, 517, 510, 507, 492, 504, 544, 522, 528, 500, 506, 502, 519, 514, 505, 520, 493, 524, 474, 468, 484, 529, 522, 516, 546, 468, 498, 491, 496, 486, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[514, 477, 510, 531, 503, 500, 487, 496, 460, 514, 520, 558, 512, 549, 508, 475, 487, 514, 495, 508, 586, 489, 494, 573, 532, 507, 517, 522, 490, 515, 527, 515, 509, 517, 510, 519, 506, 507, 475, 518, 510, 507, 517, 497, 521, 513, 514, 527, 522, 515, 523, 485, 528, 532, 518, 526, 528, 526, 536, 525, 461, 473, 486, 532] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[492, 496, 519, 528, 487, 520, 525, 499, 479, 487, 503, 533, 523, 487, 540, 521, 506, 508, 508, 520, 553, 517, 509, 535, 572, 508, 502, 530, 482, 520, 478, 518, 503, 524, 479, 504, 511, 516, 470, 526, 521, 535, 489, 542, 501, 496, 510, 506, 506, 516, 504, 503, 525, 489, 483, 533, 555, 499, 522, 521, 551, 503, 491, 529] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[534, 509, 511, 516, 500, 476, 495, 514, 13, 9, 0, 0, 0, 0, 0, 0, 568, 510, 556, 525, 546, 491, 509, 504, 0, 0, 0, 0, 0, 0, 0, 0, 543, 488, 506, 487, 507, 543, 488, 493, 0, 0, 0, 0, 0, 0, 0, 0, 519, 514, 546, 520, 517, 521, 504, 501, 0, 0, 0, 0, 0, 0, 0, 0, 511, 491, 489, 496, 517, 489, 512, 526, 5, 0, 0, 0, 0, 0, 0, 0, 525, 496, 516, 527, 474, 510, 494, 519, 6, 2, 0, 0, 0, 0, 0, 0, 540, 531, 538, 500, 486, 558, 491, 504, 0, 0, 0, 0, 0, 0, 0, 0, 497, 503, 512, 511, 520, 522, 460, 507, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4209, 4055, 4142, 4036, 4069, 4148, 4032] avg=4096.0000 max=4209 max_rank=1 min=4032 min_rank=7 imbalance_ratio=1.027588 max_load_reduction=28559 max_load_reduction_pct=0.871552 moved_to_echo=16285 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[489, 529, 466, 519, 515, 509, 524, 510, 14, 9, 0, 0, 0, 0, 0, 0, 524, 469, 537, 555, 528, 559, 484, 519, 0, 0, 0, 0, 0, 0, 0, 0, 482, 533, 509, 510, 494, 489, 464, 549, 0, 0, 0, 0, 0, 0, 0, 0, 501, 536, 521, 546, 507, 492, 512, 527, 0, 0, 0, 0, 0, 0, 0, 0, 490, 503, 481, 534, 508, 506, 533, 532, 5, 0, 0, 0, 0, 0, 0, 0, 518, 503, 517, 517, 510, 507, 490, 511, 6, 3, 0, 0, 0, 0, 0, 0, 470, 491, 465, 495, 516, 500, 538, 533, 0, 0, 0, 0, 0, 0, 0, 0, 543, 487, 537, 522, 498, 522, 551, 495, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4175, 4030, 4142, 4092, 4082, 4008, 4155] avg=4096.0000 max=4175 max_rank=1 min=4008 min_rank=6 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16337 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[497, 474, 529, 483, 541, 543, 494, 522, 12, 10, 0, 0, 0, 0, 0, 0, 505, 504, 521, 488, 463, 503, 471, 566, 0, 0, 0, 0, 0, 0, 0, 0, 515, 522, 491, 527, 489, 511, 485, 502, 0, 0, 0, 0, 0, 0, 0, 0, 513, 531, 511, 541, 506, 512, 547, 504, 0, 0, 0, 0, 0, 0, 0, 0, 502, 558, 543, 509, 488, 517, 477, 514, 5, 0, 0, 0, 0, 0, 0, 0, 513, 521, 501, 536, 493, 506, 474, 506, 6, 3, 0, 0, 0, 0, 0, 0, 542, 510, 511, 521, 530, 504, 542, 522, 0, 0, 0, 0, 0, 0, 0, 0, 513, 508, 505, 523, 504, 543, 478, 507, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4021, 4042, 4165, 4113, 4059, 4182, 4081] avg=4096.0000 max=4182 max_rank=6 min=4021 min_rank=1 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16435 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[463, 479, 518, 492, 531, 513, 525, 538, 13, 9, 0, 0, 0, 0, 0, 0, 509, 482, 494, 527, 508, 514, 472, 505, 0, 0, 0, 0, 0, 0, 0, 0, 488, 515, 551, 508, 490, 513, 502, 539, 0, 0, 0, 0, 0, 0, 0, 0, 486, 507, 516, 477, 495, 508, 498, 507, 0, 0, 0, 0, 0, 0, 0, 0, 509, 504, 552, 518, 551, 498, 517, 522, 5, 0, 0, 0, 0, 0, 0, 0, 516, 490, 497, 518, 502, 524, 526, 517, 6, 2, 0, 0, 0, 0, 0, 0, 483, 490, 532, 501, 534, 517, 526, 553, 0, 0, 0, 0, 0, 0, 0, 0, 495, 495, 525, 494, 538, 577, 493, 549, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4011, 4106, 3994, 4176, 4098, 4136, 4166] avg=4096.0000 max=4176 max_rank=4 min=3994 min_rank=3 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16576 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[528, 538, 517, 505, 529, 524, 505, 482, 18, 14, 0, 0, 0, 0, 0, 0, 531, 553, 540, 485, 491, 496, 470, 510, 0, 0, 0, 0, 0, 0, 0, 0, 534, 539, 469, 518, 511, 503, 511, 489, 0, 0, 0, 0, 0, 0, 0, 0, 498, 495, 512, 508, 486, 523, 463, 544, 0, 0, 0, 0, 0, 0, 0, 0, 539, 526, 536, 513, 522, 495, 531, 468, 9, 0, 0, 0, 0, 0, 0, 0, 534, 473, 507, 497, 534, 524, 501, 547, 8, 6, 0, 0, 0, 0, 0, 0, 532, 473, 557, 476, 494, 537, 507, 492, 3, 1, 0, 0, 0, 0, 0, 0, 509, 534, 513, 525, 495, 500, 516, 495, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4160, 4076, 4074, 4029, 4139, 4131, 4072, 4087] avg=4096.0000 max=4160 max_rank=0 min=4029 min_rank=3 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[474, 497, 513, 524, 528, 504, 523, 546, 14, 10, 0, 0, 0, 0, 0, 0, 487, 503, 481, 546, 508, 562, 505, 542, 0, 0, 0, 0, 0, 0, 0, 0, 547, 520, 484, 523, 534, 545, 507, 524, 0, 0, 0, 0, 0, 0, 0, 0, 500, 506, 520, 501, 507, 500, 546, 503, 0, 0, 0, 0, 0, 0, 0, 0, 538, 489, 517, 510, 507, 492, 504, 544, 5, 0, 0, 0, 0, 0, 0, 0, 522, 528, 500, 506, 502, 519, 514, 505, 6, 3, 0, 0, 0, 0, 0, 0, 520, 493, 524, 474, 468, 484, 529, 522, 0, 0, 0, 0, 0, 0, 0, 0, 516, 546, 468, 498, 491, 496, 486, 508, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4133, 4134, 4184, 4083, 4106, 4105, 4014, 4009] avg=4096.0000 max=4184 max_rank=2 min=4009 min_rank=7 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16234 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[514, 477, 510, 531, 503, 500, 487, 496, 14, 9, 0, 0, 0, 0, 0, 0, 460, 514, 520, 544, 512, 549, 508, 475, 0, 0, 0, 0, 0, 0, 0, 0, 487, 514, 495, 508, 573, 489, 494, 573, 0, 0, 0, 0, 0, 0, 0, 0, 532, 507, 517, 522, 490, 515, 515, 515, 0, 0, 0, 0, 0, 0, 0, 0, 509, 517, 510, 519, 506, 507, 475, 518, 6, 0, 0, 0, 0, 0, 0, 0, 510, 507, 517, 497, 521, 513, 514, 527, 7, 3, 0, 0, 0, 0, 0, 0, 522, 515, 523, 485, 528, 532, 518, 526, 0, 0, 0, 0, 0, 0, 0, 0, 528, 526, 536, 525, 461, 473, 486, 532, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4041, 4082, 4133, 4113, 4067, 4116, 4149, 4067] avg=4096.0000 max=4149 max_rank=6 min=4041 min_rank=0 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[492, 496, 519, 528, 487, 520, 525, 499, 13, 8, 0, 0, 0, 0, 0, 0, 479, 487, 503, 520, 523, 487, 540, 521, 0, 0, 0, 0, 0, 0, 0, 0, 506, 508, 508, 520, 541, 517, 509, 535, 0, 0, 0, 0, 0, 0, 0, 0, 572, 508, 502, 530, 482, 520, 468, 518, 0, 0, 0, 0, 0, 0, 0, 0, 503, 524, 479, 504, 511, 516, 470, 526, 6, 0, 0, 0, 0, 0, 0, 0, 521, 535, 489, 542, 501, 496, 510, 506, 6, 2, 0, 0, 0, 0, 0, 0, 506, 516, 504, 503, 525, 489, 483, 533, 0, 0, 0, 0, 0, 0, 0, 0, 555, 499, 522, 521, 551, 503, 491, 529, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4060, 4144, 4100, 4039, 4108, 4059, 4171] avg=4096.0000 max=4171 max_rank=7 min=4039 min_rank=4 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4224, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4224, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 3968, 4096, 4224, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[514, 514, 537, 515, 520, 499, 482, 523, 554, 531, 479, 488, 487, 504, 542, 520, 536, 505, 507, 512, 550, 493, 512, 517, 506, 493, 479, 509, 525, 492, 526, 502, 516, 508, 523, 460, 529, 568, 561, 495, 526, 508, 509, 488, 533, 497, 521, 475, 483, 519, 512, 518, 522, 522, 495, 519, 479, 506, 541, 539, 499, 504, 491, 529] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[534, 489, 501, 521, 497, 505, 498, 533, 506, 484, 533, 551, 552, 537, 519, 506, 507, 510, 496, 478, 537, 545, 541, 515, 508, 530, 551, 547, 487, 537, 498, 498, 538, 505, 512, 476, 500, 516, 496, 520, 465, 504, 504, 531, 514, 494, 541, 508, 508, 495, 492, 529, 534, 460, 500, 515, 536, 499, 517, 470, 509, 527, 480, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[544, 466, 502, 530, 502, 491, 517, 521, 508, 560, 530, 494, 551, 536, 496, 531, 490, 505, 489, 522, 496, 503, 534, 531, 492, 526, 501, 539, 504, 526, 536, 512, 511, 513, 483, 488, 563, 497, 489, 513, 491, 512, 550, 519, 509, 510, 542, 523, 528, 480, 494, 496, 508, 505, 516, 497, 493, 482, 514, 542, 475, 518, 532, 490] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[494, 500, 495, 501, 535, 509, 516, 512, 525, 495, 511, 525, 495, 494, 527, 461, 530, 519, 547, 511, 524, 511, 513, 546, 507, 512, 492, 481, 518, 505, 554, 534, 511, 497, 520, 469, 512, 506, 513, 548, 517, 508, 511, 523, 539, 493, 536, 513, 538, 514, 500, 544, 502, 520, 495, 471, 558, 453, 506, 534, 527, 500, 486, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[503, 497, 514, 528, 507, 505, 476, 501, 519, 509, 527, 493, 471, 490, 537, 548, 539, 480, 531, 520, 524, 489, 537, 507, 509, 514, 510, 523, 548, 491, 499, 511, 546, 490, 543, 489, 504, 501, 503, 503, 496, 526, 507, 508, 533, 496, 506, 519, 499, 528, 520, 516, 511, 517, 507, 530, 495, 491, 484, 520, 544, 528, 520, 531] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[495, 541, 476, 507, 486, 502, 535, 530, 516, 525, 498, 511, 531, 520, 513, 538, 488, 514, 520, 505, 572, 486, 526, 540, 515, 497, 522, 500, 488, 485, 538, 512, 523, 507, 496, 492, 504, 518, 523, 525, 518, 476, 528, 540, 484, 521, 484, 490, 491, 513, 540, 527, 525, 542, 510, 492, 493, 518, 523, 511, 475, 523, 508, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[510, 503, 529, 483, 520, 522, 492, 483, 514, 519, 546, 472, 518, 523, 491, 497, 481, 519, 467, 477, 485, 503, 521, 519, 498, 493, 476, 507, 547, 472, 525, 525, 541, 492, 551, 511, 524, 536, 520, 528, 535, 514, 503, 493, 547, 532, 491, 515, 531, 523, 481, 503, 500, 497, 531, 568, 538, 512, 543, 494, 514, 503, 529, 531] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[504, 543, 509, 512, 530, 543, 508, 492, 533, 525, 535, 514, 523, 498, 512, 524, 481, 523, 506, 499, 530, 518, 515, 518, 478, 531, 521, 515, 525, 520, 518, 530, 530, 499, 491, 513, 481, 541, 531, 539, 502, 515, 494, 514, 489, 497, 501, 522, 511, 468, 519, 487, 507, 524, 527, 502, 509, 507, 509, 502, 507, 516, 497, 484] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[514, 514, 537, 515, 520, 499, 482, 523, 21, 0, 0, 0, 0, 0, 0, 0, 522, 531, 479, 488, 487, 504, 542, 520, 0, 0, 0, 0, 0, 0, 0, 0, 536, 505, 507, 512, 537, 493, 512, 517, 0, 0, 0, 0, 0, 0, 0, 0, 506, 493, 479, 509, 525, 492, 526, 502, 0, 0, 0, 0, 0, 0, 0, 0, 507, 508, 523, 460, 529, 568, 561, 495, 0, 0, 0, 0, 0, 0, 0, 0, 526, 508, 509, 488, 533, 497, 521, 475, 6, 0, 0, 0, 0, 0, 0, 0, 483, 519, 512, 518, 522, 522, 495, 519, 8, 3, 0, 0, 0, 0, 0, 0, 479, 506, 541, 539, 499, 504, 491, 529, 11, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4073, 4119, 4032, 4151, 4063, 4101, 4104] avg=4096.0000 max=4151 max_rank=4 min=4032 min_rank=3 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16419 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[534, 489, 501, 521, 497, 505, 498, 533, 19, 0, 0, 0, 0, 0, 0, 0, 477, 484, 533, 551, 552, 537, 519, 506, 0, 0, 0, 0, 0, 0, 0, 0, 507, 510, 496, 478, 524, 545, 541, 515, 0, 0, 0, 0, 0, 0, 0, 0, 508, 530, 551, 547, 487, 537, 498, 498, 0, 0, 0, 0, 0, 0, 0, 0, 528, 505, 512, 476, 500, 516, 496, 520, 0, 0, 0, 0, 0, 0, 0, 0, 465, 504, 504, 531, 514, 494, 541, 508, 7, 0, 0, 0, 0, 0, 0, 0, 508, 495, 492, 529, 534, 460, 500, 515, 8, 3, 0, 0, 0, 0, 0, 0, 536, 499, 517, 470, 509, 527, 480, 522, 10, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4159, 4116, 4156, 4053, 4068, 4044, 4075] avg=4096.0000 max=4159 max_rank=1 min=4044 min_rank=6 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16240 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[494, 500, 495, 501, 535, 509, 516, 512, 20, 0, 0, 0, 0, 0, 0, 0, 494, 495, 511, 525, 495, 494, 527, 461, 0, 0, 0, 0, 0, 0, 0, 0, 530, 519, 547, 511, 512, 511, 513, 546, 0, 0, 0, 0, 0, 0, 0, 0, 507, 512, 492, 481, 518, 505, 554, 534, 0, 0, 0, 0, 0, 0, 0, 0, 502, 497, 520, 469, 512, 506, 513, 548, 0, 0, 0, 0, 0, 0, 0, 0, 517, 508, 511, 523, 539, 493, 536, 513, 6, 0, 0, 0, 0, 0, 0, 0, 538, 514, 500, 544, 502, 520, 495, 471, 7, 3, 0, 0, 0, 0, 0, 0, 558, 453, 506, 534, 527, 500, 486, 505, 11, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4002, 4189, 4103, 4067, 4146, 4094, 4085] avg=4096.0000 max=4189 max_rank=2 min=4002 min_rank=1 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[503, 497, 514, 528, 507, 505, 476, 501, 20, 0, 0, 0, 0, 0, 0, 0, 488, 509, 527, 493, 471, 490, 537, 548, 0, 0, 0, 0, 0, 0, 0, 0, 539, 480, 531, 520, 512, 489, 537, 507, 0, 0, 0, 0, 0, 0, 0, 0, 509, 514, 510, 523, 548, 491, 499, 511, 0, 0, 0, 0, 0, 0, 0, 0, 536, 490, 543, 489, 504, 501, 503, 503, 0, 0, 0, 0, 0, 0, 0, 0, 496, 526, 507, 508, 533, 496, 506, 519, 7, 0, 0, 0, 0, 0, 0, 0, 499, 528, 520, 516, 511, 517, 507, 530, 7, 3, 0, 0, 0, 0, 0, 0, 495, 491, 484, 520, 544, 528, 520, 531, 11, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4051, 4063, 4115, 4105, 4069, 4098, 4138, 4129] avg=4096.0000 max=4138 max_rank=6 min=4051 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16434 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[495, 541, 476, 507, 486, 502, 535, 530, 24, 0, 0, 0, 0, 0, 0, 0, 477, 525, 498, 511, 531, 520, 513, 538, 0, 0, 0, 0, 0, 0, 0, 0, 488, 514, 520, 505, 552, 486, 526, 540, 0, 0, 0, 0, 0, 0, 0, 0, 515, 497, 522, 500, 488, 485, 536, 512, 0, 0, 0, 0, 0, 0, 0, 0, 507, 507, 496, 492, 504, 518, 523, 525, 0, 0, 0, 0, 0, 0, 0, 0, 518, 476, 528, 540, 484, 521, 484, 490, 10, 2, 0, 0, 0, 0, 0, 0, 491, 513, 540, 527, 525, 542, 510, 492, 12, 6, 0, 0, 0, 0, 0, 0, 493, 518, 523, 511, 475, 523, 508, 516, 15, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4113, 4131, 4055, 4072, 4053, 4158, 4090] avg=4096.0000 max=4158 max_rank=6 min=4053 min_rank=5 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[544, 466, 502, 530, 502, 491, 517, 521, 20, 0, 0, 0, 0, 0, 0, 0, 478, 560, 530, 494, 551, 536, 496, 531, 0, 0, 0, 0, 0, 0, 0, 0, 490, 505, 489, 522, 485, 503, 534, 531, 0, 0, 0, 0, 0, 0, 0, 0, 492, 526, 501, 539, 504, 526, 536, 512, 0, 0, 0, 0, 0, 0, 0, 0, 502, 513, 483, 488, 563, 497, 489, 513, 0, 0, 0, 0, 0, 0, 0, 0, 491, 512, 550, 519, 509, 510, 542, 523, 6, 0, 0, 0, 0, 0, 0, 0, 528, 480, 494, 496, 508, 505, 516, 497, 7, 3, 0, 0, 0, 0, 0, 0, 493, 482, 514, 542, 475, 518, 532, 490, 10, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4176, 4059, 4136, 4048, 4162, 4034, 4060] avg=4096.0000 max=4176 max_rank=1 min=4034 min_rank=6 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16304 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[510, 503, 529, 483, 520, 522, 492, 483, 20, 0, 0, 0, 0, 0, 0, 0, 484, 519, 546, 472, 518, 523, 491, 497, 0, 0, 0, 0, 0, 0, 0, 0, 481, 519, 467, 477, 474, 503, 521, 519, 0, 0, 0, 0, 0, 0, 0, 0, 498, 493, 476, 507, 547, 472, 525, 525, 0, 0, 0, 0, 0, 0, 0, 0, 531, 492, 551, 511, 524, 536, 520, 528, 0, 0, 0, 0, 0, 0, 0, 0, 535, 514, 503, 493, 547, 532, 491, 515, 7, 0, 0, 0, 0, 0, 0, 0, 531, 523, 481, 503, 500, 497, 531, 568, 7, 3, 0, 0, 0, 0, 0, 0, 538, 512, 543, 494, 514, 503, 529, 531, 10, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4062, 4050, 3961, 4043, 4193, 4137, 4144, 4178] avg=4096.0000 max=4193 max_rank=4 min=3961 min_rank=2 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16652 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[504, 543, 509, 512, 530, 543, 508, 492, 21, 0, 0, 0, 0, 0, 0, 0, 501, 525, 535, 514, 523, 498, 512, 524, 0, 0, 0, 0, 0, 0, 0, 0, 481, 523, 506, 499, 518, 518, 515, 518, 0, 0, 0, 0, 0, 0, 0, 0, 478, 531, 521, 515, 525, 520, 518, 530, 0, 0, 0, 0, 0, 0, 0, 0, 520, 499, 491, 513, 481, 541, 531, 539, 0, 0, 0, 0, 0, 0, 0, 0, 502, 515, 494, 514, 489, 497, 501, 522, 7, 0, 0, 0, 0, 0, 0, 0, 511, 468, 519, 487, 507, 524, 527, 502, 7, 3, 0, 0, 0, 0, 0, 0, 509, 507, 509, 502, 507, 516, 497, 484, 11, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4162, 4132, 4078, 4138, 4115, 4041, 4055, 4047] avg=4096.0000 max=4162 max_rank=0 min=4041 min_rank=5 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16258 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3968, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 3968, 4224, 4224, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 3968, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[541, 567, 494, 544, 510, 497, 487, 510, 515, 549, 529, 558, 513, 501, 487, 500, 496, 524, 499, 493, 542, 488, 556, 474, 501, 525, 529, 516, 485, 505, 491, 521, 512, 499, 514, 490, 550, 518, 524, 466, 514, 512, 512, 516, 527, 511, 511, 523, 524, 508, 512, 514, 484, 467, 517, 509, 467, 506, 508, 556, 486, 523, 524, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[545, 522, 497, 540, 477, 531, 489, 512, 513, 519, 494, 504, 507, 489, 541, 519, 524, 494, 506, 517, 529, 487, 509, 545, 524, 506, 541, 538, 496, 498, 485, 515, 500, 462, 483, 547, 532, 528, 475, 541, 536, 495, 528, 533, 522, 478, 486, 475, 485, 494, 539, 514, 522, 485, 531, 515, 514, 536, 511, 521, 555, 478, 528, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[504, 532, 534, 515, 488, 541, 519, 546, 505, 520, 497, 523, 514, 526, 544, 515, 513, 529, 479, 522, 506, 514, 451, 520, 518, 484, 484, 487, 519, 514, 492, 562, 508, 527, 491, 524, 496, 518, 523, 528, 498, 505, 498, 492, 546, 537, 495, 542, 517, 514, 506, 527, 482, 524, 500, 506, 512, 498, 487, 495, 493, 516, 511, 535] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[545, 503, 488, 523, 553, 526, 488, 536, 512, 507, 546, 503, 507, 511, 524, 467, 502, 513, 477, 520, 555, 486, 506, 512, 546, 465, 537, 481, 522, 551, 522, 528, 501, 509, 505, 515, 534, 526, 552, 502, 537, 513, 501, 509, 529, 517, 503, 471, 519, 480, 504, 516, 495, 492, 511, 502, 515, 530, 509, 491, 496, 529, 479, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[482, 541, 504, 470, 479, 508, 524, 499, 484, 499, 506, 503, 488, 541, 499, 495, 528, 513, 498, 506, 514, 515, 545, 495, 530, 499, 512, 505, 501, 513, 515, 497, 478, 486, 511, 528, 530, 491, 510, 524, 506, 508, 509, 477, 545, 517, 551, 515, 525, 511, 499, 490, 514, 517, 512, 527, 554, 527, 510, 552, 517, 560, 522, 537] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[503, 482, 516, 536, 524, 566, 507, 498, 514, 542, 530, 501, 505, 522, 478, 488, 499, 480, 539, 505, 507, 504, 567, 510, 503, 521, 509, 496, 506, 533, 522, 494, 558, 540, 492, 546, 488, 508, 484, 490, 528, 521, 498, 531, 471, 515, 485, 507, 534, 538, 490, 498, 515, 542, 496, 516, 507, 530, 519, 445, 552, 504, 489, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[489, 492, 494, 563, 500, 466, 537, 531, 514, 538, 498, 533, 515, 468, 495, 516, 521, 517, 543, 503, 482, 525, 494, 530, 522, 503, 491, 493, 499, 520, 536, 487, 462, 501, 552, 517, 505, 534, 521, 496, 497, 526, 497, 547, 492, 515, 528, 521, 511, 510, 505, 517, 504, 525, 479, 529, 533, 529, 509, 533, 524, 502, 513, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[488, 519, 517, 532, 521, 508, 500, 543, 518, 507, 544, 498, 518, 546, 542, 550, 545, 519, 503, 502, 492, 506, 502, 507, 475, 500, 499, 506, 507, 563, 546, 509, 502, 549, 550, 535, 533, 492, 455, 516, 482, 537, 529, 537, 536, 534, 512, 499, 526, 481, 487, 526, 488, 481, 494, 503, 503, 483, 506, 511, 495, 485, 481, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[541, 567, 494, 522, 510, 497, 487, 510, 0, 0, 0, 0, 0, 0, 0, 0, 515, 538, 529, 558, 513, 501, 487, 500, 0, 0, 0, 0, 0, 0, 0, 0, 496, 524, 499, 493, 542, 488, 556, 474, 5, 1, 0, 0, 0, 0, 0, 0, 501, 525, 529, 516, 485, 505, 491, 521, 8, 0, 0, 0, 0, 0, 0, 0, 512, 499, 514, 489, 550, 518, 524, 466, 0, 0, 0, 0, 0, 0, 0, 0, 514, 512, 512, 516, 519, 511, 511, 523, 0, 0, 0, 0, 0, 0, 0, 0, 524, 508, 512, 514, 484, 467, 517, 509, 22, 11, 0, 0, 0, 0, 0, 0, 467, 506, 508, 556, 486, 523, 524, 512, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4141, 4078, 4081, 4072, 4118, 4068, 4082] avg=4096.0000 max=4141 max_rank=1 min=4068 min_rank=6 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16340 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[545, 522, 497, 518, 477, 531, 489, 512, 0, 0, 0, 0, 0, 0, 0, 0, 513, 509, 494, 504, 507, 489, 541, 519, 0, 0, 0, 0, 0, 0, 0, 0, 524, 494, 506, 517, 529, 487, 509, 545, 5, 2, 0, 0, 0, 0, 0, 0, 524, 506, 541, 538, 496, 498, 485, 515, 8, 0, 0, 0, 0, 0, 0, 0, 500, 462, 483, 545, 532, 528, 475, 541, 0, 0, 0, 0, 0, 0, 0, 0, 536, 495, 528, 533, 514, 478, 486, 475, 0, 0, 0, 0, 0, 0, 0, 0, 485, 494, 539, 514, 522, 485, 531, 515, 22, 10, 0, 0, 0, 0, 0, 0, 514, 536, 511, 521, 555, 478, 528, 501, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4076, 4118, 4111, 4066, 4045, 4117, 4144] avg=4096.0000 max=4144 max_rank=7 min=4045 min_rank=5 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[545, 503, 488, 502, 553, 526, 488, 536, 0, 0, 0, 0, 0, 0, 0, 0, 512, 497, 546, 503, 507, 511, 524, 467, 0, 0, 0, 0, 0, 0, 0, 0, 502, 513, 477, 520, 555, 486, 506, 512, 5, 1, 0, 0, 0, 0, 0, 0, 546, 465, 537, 481, 522, 551, 522, 528, 8, 0, 0, 0, 0, 0, 0, 0, 501, 509, 505, 514, 534, 526, 552, 502, 0, 0, 0, 0, 0, 0, 0, 0, 537, 513, 501, 509, 521, 517, 503, 471, 0, 0, 0, 0, 0, 0, 0, 0, 519, 480, 504, 516, 495, 492, 511, 502, 21, 10, 0, 0, 0, 0, 0, 0, 515, 530, 509, 491, 496, 529, 479, 509, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4141, 4067, 4077, 4160, 4143, 4072, 4050, 4058] avg=4096.0000 max=4160 max_rank=3 min=4050 min_rank=6 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16323 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[489, 492, 494, 540, 500, 466, 537, 531, 0, 0, 0, 0, 0, 0, 0, 0, 514, 527, 498, 533, 515, 468, 495, 516, 0, 0, 0, 0, 0, 0, 0, 0, 521, 517, 543, 503, 482, 525, 494, 530, 5, 1, 0, 0, 0, 0, 0, 0, 522, 503, 491, 493, 499, 520, 536, 487, 8, 0, 0, 0, 0, 0, 0, 0, 462, 501, 552, 516, 505, 534, 521, 496, 0, 0, 0, 0, 0, 0, 0, 0, 497, 526, 497, 547, 484, 515, 528, 521, 0, 0, 0, 0, 0, 0, 0, 0, 511, 510, 505, 517, 504, 525, 479, 529, 23, 11, 0, 0, 0, 0, 0, 0, 533, 529, 509, 533, 524, 502, 513, 514, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4049, 4066, 4121, 4059, 4087, 4115, 4114, 4157] avg=4096.0000 max=4157 max_rank=7 min=4049 min_rank=0 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16473 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[504, 532, 534, 494, 488, 541, 519, 546, 0, 0, 0, 0, 0, 0, 0, 0, 505, 510, 497, 523, 514, 526, 544, 515, 0, 0, 0, 0, 0, 0, 0, 0, 513, 529, 479, 522, 506, 514, 451, 520, 5, 1, 0, 0, 0, 0, 0, 0, 518, 484, 484, 487, 519, 514, 492, 562, 8, 0, 0, 0, 0, 0, 0, 0, 508, 527, 491, 523, 496, 518, 523, 528, 0, 0, 0, 0, 0, 0, 0, 0, 498, 505, 498, 492, 538, 537, 495, 542, 0, 0, 0, 0, 0, 0, 0, 0, 517, 514, 506, 527, 482, 524, 500, 506, 21, 10, 0, 0, 0, 0, 0, 0, 512, 498, 487, 495, 493, 516, 511, 530, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4158, 4134, 4040, 4068, 4114, 4105, 4107, 4042] avg=4096.0000 max=4158 max_rank=0 min=4040 min_rank=2 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[482, 541, 504, 448, 479, 508, 524, 499, 0, 0, 0, 0, 0, 0, 0, 0, 484, 486, 506, 503, 488, 541, 499, 495, 0, 0, 0, 0, 0, 0, 0, 0, 528, 513, 498, 506, 514, 515, 545, 495, 7, 6, 7, 0, 0, 0, 0, 0, 530, 499, 512, 505, 501, 513, 515, 497, 13, 0, 0, 0, 0, 0, 0, 0, 478, 486, 511, 522, 530, 491, 510, 524, 0, 0, 0, 0, 0, 0, 0, 0, 506, 508, 509, 477, 523, 517, 551, 515, 0, 0, 0, 0, 0, 0, 0, 0, 525, 511, 499, 490, 514, 517, 512, 527, 22, 13, 2, 0, 0, 0, 0, 0, 554, 527, 510, 552, 517, 560, 522, 530, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3985, 4002, 4134, 4085, 4052, 4106, 4132, 4272] avg=4096.0000 max=4272 max_rank=7 min=3985 min_rank=0 imbalance_ratio=1.042969 max_load_reduction=28496 max_load_reduction_pct=0.869629 moved_to_echo=16562 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[503, 482, 516, 514, 524, 566, 507, 498, 0, 0, 0, 0, 0, 0, 0, 0, 514, 531, 530, 501, 505, 522, 478, 488, 0, 0, 0, 0, 0, 0, 0, 0, 499, 480, 539, 505, 507, 504, 567, 510, 5, 2, 0, 0, 0, 0, 0, 0, 503, 521, 509, 496, 506, 533, 522, 494, 7, 0, 0, 0, 0, 0, 0, 0, 558, 540, 492, 544, 488, 508, 484, 490, 0, 0, 0, 0, 0, 0, 0, 0, 528, 521, 498, 531, 464, 515, 485, 507, 0, 0, 0, 0, 0, 0, 0, 0, 534, 538, 490, 498, 515, 542, 496, 516, 22, 11, 0, 0, 0, 0, 0, 0, 507, 530, 519, 445, 552, 504, 489, 519, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4069, 4118, 4091, 4104, 4049, 4162, 4065] avg=4096.0000 max=4162 max_rank=6 min=4049 min_rank=5 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[488, 519, 517, 510, 521, 508, 500, 543, 0, 0, 0, 0, 0, 0, 0, 0, 518, 497, 544, 498, 518, 546, 542, 550, 0, 0, 0, 0, 0, 0, 0, 0, 545, 519, 503, 502, 492, 506, 502, 507, 4, 2, 0, 0, 0, 0, 0, 0, 475, 500, 499, 506, 507, 563, 546, 509, 8, 0, 0, 0, 0, 0, 0, 0, 502, 549, 550, 533, 533, 492, 455, 516, 0, 0, 0, 0, 0, 0, 0, 0, 482, 537, 529, 537, 528, 534, 512, 499, 0, 0, 0, 0, 0, 0, 0, 0, 526, 481, 487, 526, 488, 481, 494, 503, 22, 10, 0, 0, 0, 0, 0, 0, 503, 483, 506, 511, 495, 485, 481, 484, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4213, 4082, 4113, 4130, 4158, 4018, 3948] avg=4096.0000 max=4213 max_rank=1 min=3948 min_rank=7 imbalance_ratio=1.028564 max_load_reduction=28555 max_load_reduction_pct=0.871429 moved_to_echo=16254 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[526, 495, 536, 510, 523, 524, 513, 496, 515, 492, 510, 527, 494, 535, 532, 494, 541, 494, 493, 511, 504, 535, 548, 499, 485, 493, 502, 509, 521, 513, 508, 508, 495, 546, 563, 543, 508, 456, 503, 502, 492, 530, 518, 503, 536, 493, 535, 527, 531, 496, 496, 516, 510, 484, 509, 525, 512, 498, 507, 525, 533, 499, 493, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[480, 525, 514, 505, 537, 515, 479, 517, 526, 479, 523, 554, 498, 504, 471, 499, 528, 538, 522, 535, 473, 489, 477, 512, 514, 529, 514, 539, 524, 505, 507, 543, 509, 488, 562, 497, 518, 502, 514, 563, 509, 550, 510, 522, 521, 494, 490, 490, 501, 511, 526, 475, 552, 514, 494, 490, 523, 493, 505, 553, 515, 505, 508, 489] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[530, 533, 479, 518, 557, 506, 508, 516, 498, 520, 536, 531, 515, 469, 511, 517, 523, 483, 509, 495, 500, 506, 538, 533, 480, 508, 464, 507, 487, 525, 500, 512, 480, 516, 493, 562, 490, 523, 525, 527, 519, 544, 495, 544, 483, 522, 539, 565, 472, 473, 521, 523, 461, 522, 468, 521, 530, 522, 515, 510, 510, 527, 503, 549] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[511, 546, 526, 513, 524, 519, 508, 516, 500, 517, 490, 516, 511, 508, 521, 552, 505, 512, 504, 527, 518, 507, 492, 491, 525, 513, 507, 535, 505, 531, 556, 560, 483, 515, 520, 475, 517, 484, 458, 515, 531, 519, 486, 500, 520, 511, 518, 489, 525, 469, 552, 530, 503, 512, 487, 500, 494, 492, 561, 530, 482, 499, 529, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[479, 490, 487, 496, 515, 509, 530, 511, 538, 489, 532, 530, 524, 476, 558, 507, 538, 508, 508, 508, 507, 513, 509, 524, 486, 517, 534, 544, 504, 493, 520, 538, 540, 527, 511, 515, 526, 503, 544, 506, 523, 511, 515, 497, 512, 531, 493, 493, 495, 495, 502, 473, 488, 513, 517, 509, 489, 535, 537, 505, 489, 528, 512, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[567, 509, 495, 476, 520, 511, 508, 476, 532, 502, 485, 505, 548, 524, 514, 539, 489, 502, 537, 516, 492, 555, 497, 506, 481, 493, 516, 534, 510, 494, 514, 483, 500, 561, 527, 516, 513, 525, 512, 491, 511, 503, 540, 504, 529, 507, 530, 516, 487, 525, 517, 510, 511, 543, 486, 485, 526, 518, 469, 533, 483, 504, 532, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[514, 510, 558, 485, 561, 473, 464, 504, 501, 505, 485, 548, 477, 533, 521, 515, 517, 529, 481, 536, 524, 477, 500, 555, 487, 531, 513, 511, 493, 535, 470, 505, 513, 526, 514, 535, 499, 531, 541, 502, 527, 548, 494, 511, 487, 480, 504, 548, 504, 528, 546, 531, 524, 497, 488, 524, 483, 531, 537, 508, 524, 491, 473, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[508, 483, 501, 496, 502, 521, 530, 516, 526, 532, 510, 491, 493, 513, 548, 480, 510, 504, 513, 539, 502, 519, 520, 510, 533, 496, 489, 476, 509, 481, 514, 487, 517, 503, 517, 498, 512, 559, 498, 495, 499, 484, 527, 502, 498, 489, 513, 557, 500, 523, 558, 545, 543, 520, 513, 525, 538, 495, 509, 524, 509, 518, 505, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[526, 495, 536, 510, 523, 524, 513, 496, 4, 2, 0, 0, 0, 0, 0, 0, 515, 492, 510, 518, 494, 535, 532, 494, 0, 0, 0, 0, 0, 0, 0, 0, 541, 494, 493, 509, 504, 535, 548, 499, 0, 0, 0, 0, 0, 0, 0, 0, 485, 493, 502, 509, 521, 513, 508, 508, 13, 5, 0, 0, 0, 0, 0, 0, 495, 546, 542, 543, 508, 456, 503, 502, 0, 0, 0, 0, 0, 0, 0, 0, 492, 516, 518, 503, 536, 493, 535, 527, 0, 0, 0, 0, 0, 0, 0, 0, 531, 496, 496, 516, 510, 484, 509, 525, 21, 1, 0, 0, 0, 0, 0, 0, 512, 498, 507, 525, 533, 499, 493, 498, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4129, 4090, 4123, 4057, 4095, 4120, 4089, 4065] avg=4096.0000 max=4129 max_rank=0 min=4057 min_rank=3 imbalance_ratio=1.008057 max_load_reduction=28639 max_load_reduction_pct=0.873993 moved_to_echo=16369 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[480, 525, 514, 505, 537, 515, 479, 517, 4, 2, 0, 0, 0, 0, 0, 0, 526, 479, 523, 545, 498, 504, 471, 499, 0, 0, 0, 0, 0, 0, 0, 0, 528, 538, 522, 533, 473, 489, 477, 512, 0, 0, 0, 0, 0, 0, 0, 0, 514, 529, 514, 539, 524, 505, 507, 543, 14, 5, 0, 0, 0, 0, 0, 0, 509, 488, 541, 497, 518, 502, 514, 563, 0, 0, 0, 0, 0, 0, 0, 0, 509, 535, 510, 522, 521, 494, 490, 490, 0, 0, 0, 0, 0, 0, 0, 0, 501, 511, 526, 475, 552, 514, 494, 490, 21, 1, 0, 0, 0, 0, 0, 0, 523, 493, 505, 553, 515, 505, 508, 489, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4045, 4072, 4194, 4132, 4071, 4085, 4091] avg=4096.0000 max=4194 max_rank=3 min=4045 min_rank=1 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[530, 533, 479, 518, 557, 506, 508, 516, 4, 2, 0, 0, 0, 0, 0, 0, 498, 520, 536, 522, 515, 469, 511, 517, 0, 0, 0, 0, 0, 0, 0, 0, 523, 483, 509, 493, 500, 506, 538, 533, 0, 0, 0, 0, 0, 0, 0, 0, 480, 508, 464, 507, 487, 525, 500, 512, 13, 5, 0, 0, 0, 0, 0, 0, 480, 516, 475, 562, 490, 523, 525, 527, 0, 0, 0, 0, 0, 0, 0, 0, 519, 530, 495, 544, 483, 522, 539, 565, 0, 0, 0, 0, 0, 0, 0, 0, 472, 473, 521, 523, 461, 522, 468, 521, 18, 1, 0, 0, 0, 0, 0, 0, 530, 522, 515, 510, 510, 527, 503, 549, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4153, 4088, 4085, 4001, 4098, 4197, 3980, 4166] avg=4096.0000 max=4197 max_rank=5 min=3980 min_rank=6 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16441 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[511, 546, 526, 513, 524, 519, 508, 516, 4, 2, 0, 0, 0, 0, 0, 0, 500, 517, 490, 507, 511, 508, 521, 552, 0, 0, 0, 0, 0, 0, 0, 0, 505, 512, 504, 525, 518, 507, 492, 491, 0, 0, 0, 0, 0, 0, 0, 0, 525, 513, 507, 535, 505, 531, 556, 560, 13, 5, 0, 0, 0, 0, 0, 0, 483, 515, 501, 475, 517, 484, 458, 515, 0, 0, 0, 0, 0, 0, 0, 0, 531, 505, 486, 500, 520, 511, 518, 489, 0, 0, 0, 0, 0, 0, 0, 0, 525, 469, 552, 530, 503, 512, 487, 500, 19, 1, 0, 0, 0, 0, 0, 0, 494, 492, 561, 530, 482, 499, 529, 496, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4169, 4106, 4054, 4250, 3948, 4060, 4098, 4083] avg=4096.0000 max=4250 max_rank=3 min=3948 min_rank=4 imbalance_ratio=1.037598 max_load_reduction=28518 max_load_reduction_pct=0.870300 moved_to_echo=16189 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[567, 509, 495, 476, 520, 511, 508, 476, 9, 5, 2, 0, 0, 0, 0, 0, 532, 502, 485, 489, 548, 524, 514, 539, 0, 0, 0, 0, 0, 0, 0, 0, 489, 502, 537, 511, 492, 555, 497, 506, 0, 0, 0, 0, 0, 0, 0, 0, 481, 493, 516, 534, 510, 494, 514, 483, 16, 7, 0, 0, 0, 0, 0, 0, 500, 561, 502, 516, 513, 525, 512, 491, 0, 0, 0, 0, 0, 0, 0, 0, 511, 481, 540, 504, 529, 507, 530, 516, 0, 0, 0, 0, 0, 0, 0, 0, 487, 525, 517, 510, 511, 543, 486, 485, 25, 6, 0, 0, 0, 0, 0, 0, 526, 518, 469, 531, 483, 504, 532, 524, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4133, 4089, 4048, 4120, 4118, 4095, 4087] avg=4096.0000 max=4133 max_rank=1 min=4048 min_rank=3 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16420 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[479, 490, 487, 496, 515, 509, 530, 511, 4, 2, 0, 0, 0, 0, 0, 0, 538, 489, 532, 521, 524, 476, 558, 507, 0, 0, 0, 0, 0, 0, 0, 0, 538, 508, 508, 506, 507, 513, 509, 524, 0, 0, 0, 0, 0, 0, 0, 0, 486, 517, 534, 544, 504, 493, 520, 538, 13, 5, 0, 0, 0, 0, 0, 0, 540, 527, 492, 515, 526, 503, 544, 506, 0, 0, 0, 0, 0, 0, 0, 0, 523, 497, 515, 497, 512, 531, 493, 493, 0, 0, 0, 0, 0, 0, 0, 0, 495, 495, 502, 473, 488, 513, 517, 509, 19, 1, 0, 0, 0, 0, 0, 0, 489, 535, 537, 505, 489, 528, 512, 512, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4023, 4145, 4113, 4154, 4153, 4061, 4012, 4107] avg=4096.0000 max=4154 max_rank=3 min=4012 min_rank=6 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16333 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[514, 510, 558, 485, 561, 473, 464, 504, 4, 2, 0, 0, 0, 0, 0, 0, 501, 505, 485, 539, 477, 533, 521, 515, 0, 0, 0, 0, 0, 0, 0, 0, 517, 529, 481, 534, 524, 477, 500, 555, 0, 0, 0, 0, 0, 0, 0, 0, 487, 531, 513, 511, 493, 535, 470, 505, 13, 5, 0, 0, 0, 0, 0, 0, 513, 526, 495, 535, 499, 531, 541, 502, 0, 0, 0, 0, 0, 0, 0, 0, 527, 534, 494, 511, 487, 480, 504, 548, 0, 0, 0, 0, 0, 0, 0, 0, 504, 528, 546, 531, 524, 497, 488, 524, 19, 1, 0, 0, 0, 0, 0, 0, 483, 531, 537, 508, 524, 491, 473, 501, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4076, 4117, 4063, 4142, 4085, 4162, 4048] avg=4096.0000 max=4162 max_rank=6 min=4048 min_rank=7 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16437 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[508, 483, 501, 496, 502, 521, 530, 516, 4, 2, 0, 0, 0, 0, 0, 0, 526, 532, 510, 483, 493, 513, 548, 480, 0, 0, 0, 0, 0, 0, 0, 0, 510, 504, 513, 537, 502, 519, 520, 510, 0, 0, 0, 0, 0, 0, 0, 0, 533, 496, 489, 476, 509, 481, 514, 487, 12, 4, 0, 0, 0, 0, 0, 0, 517, 503, 498, 498, 512, 559, 498, 495, 0, 0, 0, 0, 0, 0, 0, 0, 499, 471, 527, 502, 498, 489, 513, 557, 0, 0, 0, 0, 0, 0, 0, 0, 500, 523, 558, 545, 543, 520, 513, 525, 19, 1, 0, 0, 0, 0, 0, 0, 538, 495, 509, 524, 509, 518, 505, 523, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4085, 4115, 4001, 4080, 4056, 4247, 4121] avg=4096.0000 max=4247 max_rank=6 min=4001 min_rank=3 imbalance_ratio=1.036865 max_load_reduction=28521 max_load_reduction_pct=0.870392 moved_to_echo=16504 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 4352, 4096, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 3968, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[518, 522, 521, 526, 487, 501, 518, 501, 518, 485, 478, 547, 501, 506, 540, 522, 525, 551, 473, 513, 490, 522, 540, 506, 512, 517, 509, 510, 470, 536, 516, 527, 538, 521, 474, 510, 563, 504, 494, 472, 525, 525, 505, 508, 508, 503, 530, 549, 509, 486, 520, 511, 512, 489, 531, 539, 514, 471, 524, 523, 519, 490, 498, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[495, 535, 499, 505, 506, 532, 497, 515, 505, 526, 481, 501, 515, 478, 510, 506, 509, 489, 526, 496, 499, 552, 516, 547, 522, 465, 521, 510, 501, 487, 488, 549, 529, 530, 489, 536, 485, 532, 536, 492, 509, 508, 513, 491, 512, 527, 552, 505, 504, 473, 557, 520, 501, 514, 499, 507, 502, 504, 558, 506, 539, 525, 494, 536] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[488, 481, 492, 472, 499, 529, 516, 496, 509, 543, 507, 535, 506, 482, 545, 523, 503, 504, 518, 506, 523, 540, 508, 535, 529, 550, 457, 508, 562, 491, 489, 484, 528, 507, 501, 528, 551, 521, 507, 569, 494, 500, 482, 509, 510, 484, 536, 490, 531, 487, 507, 535, 535, 546, 524, 493, 510, 508, 491, 524, 498, 496, 521, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[467, 506, 520, 518, 519, 505, 515, 557, 499, 510, 515, 472, 506, 499, 546, 486, 521, 513, 507, 547, 515, 478, 517, 498, 523, 496, 513, 487, 508, 529, 499, 486, 558, 506, 469, 557, 512, 477, 520, 548, 486, 517, 510, 518, 480, 520, 521, 495, 542, 514, 532, 463, 549, 524, 523, 522, 556, 499, 509, 527, 508, 492, 518, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[490, 517, 525, 493, 535, 523, 503, 503, 497, 566, 513, 477, 506, 537, 500, 511, 497, 494, 496, 505, 513, 510, 517, 502, 492, 500, 526, 486, 510, 475, 516, 489, 528, 492, 529, 528, 523, 563, 515, 484, 500, 545, 514, 498, 516, 505, 522, 511, 493, 585, 573, 464, 536, 518, 491, 515, 509, 513, 501, 510, 497, 534, 508, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[508, 495, 525, 534, 519, 507, 533, 492, 521, 518, 547, 570, 538, 477, 504, 484, 510, 503, 526, 494, 539, 478, 542, 465, 517, 489, 508, 490, 477, 538, 487, 469, 487, 488, 545, 524, 483, 538, 508, 501, 524, 509, 500, 540, 495, 518, 536, 499, 508, 542, 514, 509, 517, 525, 522, 515, 526, 500, 517, 502, 518, 533, 517, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[493, 516, 488, 514, 512, 512, 527, 535, 539, 500, 533, 465, 542, 499, 503, 524, 528, 495, 523, 537, 497, 555, 494, 523, 473, 522, 518, 498, 474, 507, 506, 496, 529, 493, 507, 531, 559, 517, 513, 516, 564, 490, 517, 537, 529, 498, 493, 516, 473, 482, 492, 504, 543, 519, 483, 505, 512, 523, 517, 483, 513, 521, 542, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[517, 540, 509, 495, 499, 517, 493, 504, 533, 514, 530, 525, 500, 538, 562, 545, 535, 502, 484, 502, 481, 521, 515, 540, 516, 515, 518, 514, 495, 507, 511, 500, 516, 500, 514, 471, 495, 505, 547, 528, 489, 512, 523, 519, 534, 504, 502, 562, 559, 459, 456, 484, 500, 537, 462, 503, 515, 510, 519, 530, 513, 522, 494, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[518, 522, 521, 526, 487, 501, 518, 501, 6, 5, 2, 2, 0, 0, 0, 0, 518, 485, 478, 547, 501, 506, 521, 522, 0, 0, 0, 0, 0, 0, 0, 0, 525, 551, 473, 513, 490, 517, 540, 506, 0, 0, 0, 0, 0, 0, 0, 0, 512, 517, 509, 510, 470, 536, 516, 527, 34, 19, 6, 0, 0, 0, 0, 0, 504, 521, 474, 510, 563, 504, 494, 472, 0, 0, 0, 0, 0, 0, 0, 0, 525, 525, 505, 508, 508, 503, 522, 549, 0, 0, 0, 0, 0, 0, 0, 0, 509, 486, 520, 511, 510, 489, 531, 539, 0, 0, 0, 0, 0, 0, 0, 0, 508, 471, 524, 523, 519, 490, 498, 495, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4078, 4115, 4156, 4042, 4145, 4095, 4028] avg=4096.0000 max=4156 max_rank=3 min=4028 min_rank=7 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16310 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[495, 535, 499, 505, 506, 532, 497, 515, 6, 5, 2, 2, 0, 0, 0, 0, 505, 526, 481, 501, 515, 478, 492, 506, 0, 0, 0, 0, 0, 0, 0, 0, 509, 489, 526, 496, 499, 547, 516, 547, 0, 0, 0, 0, 0, 0, 0, 0, 522, 465, 521, 510, 501, 487, 488, 549, 34, 18, 6, 0, 0, 0, 0, 0, 495, 530, 489, 536, 485, 532, 536, 492, 0, 0, 0, 0, 0, 0, 0, 0, 509, 508, 513, 491, 512, 527, 544, 505, 0, 0, 0, 0, 0, 0, 0, 0, 504, 473, 557, 520, 499, 514, 499, 507, 0, 0, 0, 0, 0, 0, 0, 0, 496, 504, 558, 506, 539, 525, 494, 536, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4004, 4129, 4101, 4095, 4109, 4073, 4158] avg=4096.0000 max=4158 max_rank=7 min=4004 min_rank=1 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16435 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[467, 506, 520, 518, 519, 505, 515, 557, 7, 4, 2, 2, 0, 0, 0, 0, 499, 510, 515, 472, 506, 499, 527, 486, 0, 0, 0, 0, 0, 0, 0, 0, 521, 513, 507, 547, 515, 474, 517, 498, 0, 0, 0, 0, 0, 0, 0, 0, 523, 496, 513, 487, 508, 529, 499, 486, 36, 19, 6, 0, 0, 0, 0, 0, 522, 506, 469, 557, 512, 477, 520, 548, 0, 0, 0, 0, 0, 0, 0, 0, 486, 517, 510, 518, 480, 520, 513, 495, 0, 0, 0, 0, 0, 0, 0, 0, 542, 514, 532, 463, 547, 524, 523, 522, 0, 0, 0, 0, 0, 0, 0, 0, 549, 499, 509, 527, 508, 492, 518, 519, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4122, 4014, 4092, 4102, 4111, 4039, 4167, 4121] avg=4096.0000 max=4167 max_rank=6 min=4014 min_rank=1 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16438 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[508, 495, 525, 534, 519, 507, 533, 492, 6, 4, 2, 2, 0, 0, 0, 0, 521, 518, 547, 570, 538, 477, 486, 484, 0, 0, 0, 0, 0, 0, 0, 0, 510, 503, 526, 494, 539, 474, 542, 465, 0, 0, 0, 0, 0, 0, 0, 0, 517, 489, 508, 490, 477, 538, 487, 469, 31, 18, 6, 0, 0, 0, 0, 0, 456, 488, 545, 524, 483, 538, 508, 501, 0, 0, 0, 0, 0, 0, 0, 0, 524, 509, 500, 540, 495, 518, 528, 499, 0, 0, 0, 0, 0, 0, 0, 0, 508, 542, 514, 509, 515, 525, 522, 515, 0, 0, 0, 0, 0, 0, 0, 0, 520, 500, 517, 502, 518, 533, 517, 504, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4127, 4141, 4053, 4030, 4043, 4113, 4150, 4111] avg=4096.0000 max=4150 max_rank=6 min=4030 min_rank=3 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16417 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[488, 481, 492, 472, 499, 529, 516, 496, 11, 9, 8, 5, 0, 0, 0, 0, 509, 543, 507, 535, 506, 482, 523, 523, 0, 0, 0, 0, 0, 0, 0, 0, 503, 504, 518, 506, 523, 531, 508, 535, 0, 0, 0, 0, 0, 0, 0, 0, 529, 550, 457, 508, 562, 491, 489, 484, 37, 22, 11, 0, 0, 0, 0, 0, 491, 507, 501, 528, 551, 521, 507, 569, 0, 0, 0, 0, 0, 0, 0, 0, 494, 500, 482, 509, 510, 484, 517, 490, 0, 0, 0, 0, 0, 0, 0, 0, 531, 487, 507, 535, 530, 546, 524, 493, 0, 0, 0, 0, 0, 0, 0, 0, 499, 508, 491, 524, 498, 496, 521, 515, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4006, 4128, 4128, 4140, 4175, 3986, 4153, 4052] avg=4096.0000 max=4175 max_rank=4 min=3986 min_rank=5 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[490, 517, 525, 493, 535, 523, 503, 503, 6, 5, 2, 2, 0, 0, 0, 0, 497, 566, 513, 477, 506, 537, 482, 511, 0, 0, 0, 0, 0, 0, 0, 0, 497, 494, 496, 505, 513, 505, 517, 502, 0, 0, 0, 0, 0, 0, 0, 0, 492, 500, 526, 486, 510, 475, 516, 489, 34, 18, 6, 0, 0, 0, 0, 0, 494, 492, 529, 528, 523, 563, 515, 484, 0, 0, 0, 0, 0, 0, 0, 0, 500, 545, 514, 498, 516, 505, 514, 511, 0, 0, 0, 0, 0, 0, 0, 0, 493, 585, 573, 464, 534, 518, 491, 515, 0, 0, 0, 0, 0, 0, 0, 0, 503, 513, 501, 510, 497, 534, 508, 524, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4089, 4029, 4052, 4128, 4103, 4173, 4090] avg=4096.0000 max=4173 max_rank=6 min=4029 min_rank=2 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16494 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[493, 516, 488, 514, 512, 512, 527, 535, 6, 5, 2, 2, 0, 0, 0, 0, 539, 500, 533, 465, 542, 499, 485, 524, 0, 0, 0, 0, 0, 0, 0, 0, 528, 495, 523, 537, 497, 550, 494, 523, 0, 0, 0, 0, 0, 0, 0, 0, 473, 522, 518, 498, 474, 507, 506, 496, 34, 18, 6, 0, 0, 0, 0, 0, 495, 493, 507, 531, 559, 517, 513, 516, 0, 0, 0, 0, 0, 0, 0, 0, 564, 490, 517, 537, 529, 498, 485, 516, 0, 0, 0, 0, 0, 0, 0, 0, 473, 482, 492, 504, 541, 519, 483, 505, 0, 0, 0, 0, 0, 0, 0, 0, 506, 523, 517, 483, 513, 521, 542, 499, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4087, 4147, 4052, 4131, 4136, 3999, 4104] avg=4096.0000 max=4147 max_rank=2 min=3999 min_rank=6 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16370 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[517, 540, 509, 495, 499, 517, 493, 504, 6, 5, 2, 2, 0, 0, 0, 0, 533, 514, 530, 525, 500, 538, 542, 545, 0, 0, 0, 0, 0, 0, 0, 0, 535, 502, 484, 502, 481, 516, 515, 540, 0, 0, 0, 0, 0, 0, 0, 0, 516, 515, 518, 514, 495, 507, 511, 500, 33, 20, 6, 0, 0, 0, 0, 0, 483, 500, 514, 471, 495, 505, 547, 528, 0, 0, 0, 0, 0, 0, 0, 0, 489, 512, 523, 519, 534, 504, 494, 562, 0, 0, 0, 0, 0, 0, 0, 0, 559, 459, 456, 484, 498, 537, 462, 503, 0, 0, 0, 0, 0, 0, 0, 0, 509, 510, 519, 530, 513, 522, 494, 507, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4227, 4075, 4135, 4043, 4137, 3958, 4104] avg=4096.0000 max=4227 max_rank=1 min=3958 min_rank=6 imbalance_ratio=1.031982 max_load_reduction=28541 max_load_reduction_pct=0.871002 moved_to_echo=16242 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3968, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 128, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 384, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33792 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True + [2026-09-21 08:26:49] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 259.5 | throughput per GPU (TFLOP/s/GPU): 124.1 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.696043E+00 | loss scale: 1.0 | grad norm: 8.336 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[508, 543, 484, 564, 496, 540, 527, 483, 503, 495, 519, 489, 512, 543, 535, 487, 501, 500, 510, 497, 524, 515, 499, 551, 507, 500, 547, 508, 547, 526, 502, 509, 489, 485, 524, 498, 509, 525, 509, 506, 493, 467, 525, 499, 488, 538, 514, 506, 536, 514, 545, 521, 513, 484, 547, 525, 482, 516, 524, 498, 513, 506, 501, 497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[523, 486, 465, 509, 505, 535, 489, 519, 503, 512, 483, 530, 490, 525, 510, 498, 521, 523, 505, 502, 536, 484, 498, 503, 499, 513, 523, 544, 500, 509, 568, 531, 520, 489, 509, 494, 508, 510, 548, 512, 547, 562, 524, 525, 514, 463, 538, 510, 517, 507, 520, 500, 493, 481, 522, 499, 527, 501, 517, 495, 511, 510, 539, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[503, 530, 509, 465, 493, 513, 509, 534, 549, 530, 547, 501, 493, 510, 503, 482, 527, 529, 535, 472, 503, 508, 501, 555, 480, 504, 528, 508, 537, 498, 503, 510, 525, 479, 537, 509, 469, 500, 482, 548, 490, 513, 502, 492, 516, 516, 523, 535, 515, 501, 515, 503, 529, 530, 495, 499, 522, 526, 508, 562, 512, 519, 504, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[502, 530, 531, 514, 537, 527, 513, 508, 540, 500, 518, 505, 537, 516, 499, 502, 488, 507, 542, 543, 508, 529, 522, 491, 495, 477, 532, 453, 525, 511, 470, 526, 520, 506, 504, 498, 534, 522, 524, 503, 497, 494, 478, 535, 527, 505, 498, 503, 530, 551, 492, 511, 501, 521, 536, 493, 563, 500, 517, 519, 490, 508, 515, 475] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[517, 484, 507, 528, 525, 536, 537, 502, 503, 478, 521, 492, 514, 516, 513, 510, 513, 544, 530, 507, 529, 514, 516, 479, 501, 508, 484, 513, 506, 508, 515, 521, 539, 561, 500, 478, 508, 520, 505, 510, 523, 498, 555, 537, 523, 535, 507, 492, 489, 527, 494, 507, 509, 492, 504, 507, 537, 518, 510, 500, 495, 503, 499, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[511, 455, 499, 520, 536, 551, 508, 528, 502, 495, 534, 496, 512, 487, 513, 528, 538, 545, 534, 500, 510, 527, 551, 512, 508, 529, 507, 461, 515, 517, 475, 498, 489, 490, 513, 542, 483, 481, 507, 533, 520, 515, 530, 488, 509, 523, 562, 513, 509, 505, 500, 522, 526, 482, 513, 525, 518, 493, 480, 532, 548, 473, 540, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[504, 529, 497, 515, 522, 505, 509, 549, 511, 542, 508, 511, 535, 518, 518, 547, 515, 521, 493, 508, 524, 516, 524, 485, 503, 522, 506, 505, 527, 524, 474, 481, 512, 531, 509, 504, 475, 556, 477, 520, 505, 507, 494, 482, 499, 464, 503, 513, 503, 515, 530, 552, 505, 520, 565, 473, 479, 510, 480, 487, 535, 539, 514, 562] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[484, 526, 519, 536, 539, 499, 522, 496, 527, 514, 511, 482, 515, 478, 507, 504, 532, 506, 538, 492, 534, 527, 531, 546, 531, 520, 518, 524, 500, 483, 515, 499, 509, 504, 488, 525, 515, 505, 543, 529, 543, 511, 507, 463, 516, 519, 531, 502, 522, 488, 500, 500, 532, 516, 512, 492, 512, 511, 491, 506, 512, 488, 539, 482] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[508, 543, 484, 564, 496, 526, 527, 483, 0, 0, 0, 0, 0, 0, 0, 0, 503, 495, 519, 489, 512, 543, 535, 487, 6, 1, 0, 0, 0, 0, 0, 0, 501, 500, 475, 497, 524, 515, 499, 551, 0, 0, 0, 0, 0, 0, 0, 0, 507, 500, 547, 508, 547, 526, 502, 509, 21, 0, 0, 0, 0, 0, 0, 0, 489, 485, 524, 498, 509, 525, 509, 506, 13, 0, 0, 0, 0, 0, 0, 0, 493, 467, 525, 499, 488, 538, 514, 506, 3, 1, 0, 0, 0, 0, 0, 0, 536, 514, 545, 521, 513, 484, 546, 525, 0, 0, 0, 0, 0, 0, 0, 0, 482, 516, 524, 498, 513, 506, 501, 497, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4131, 4090, 4062, 4167, 4058, 4034, 4184, 4042] avg=4096.0000 max=4184 max_rank=6 min=4034 min_rank=5 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16318 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[523, 486, 465, 509, 505, 521, 489, 519, 0, 0, 0, 0, 0, 0, 0, 0, 503, 512, 483, 530, 490, 525, 510, 498, 6, 1, 0, 0, 0, 0, 0, 0, 521, 523, 470, 502, 536, 484, 498, 503, 0, 0, 0, 0, 0, 0, 0, 0, 499, 513, 523, 544, 500, 509, 568, 531, 21, 0, 0, 0, 0, 0, 0, 0, 520, 489, 509, 494, 508, 510, 548, 512, 13, 0, 0, 0, 0, 0, 0, 0, 547, 562, 524, 525, 514, 463, 538, 510, 3, 1, 0, 0, 0, 0, 0, 0, 517, 507, 520, 500, 493, 481, 521, 499, 0, 0, 0, 0, 0, 0, 0, 0, 527, 501, 517, 495, 511, 510, 539, 515, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4017, 4058, 4037, 4208, 4103, 4187, 4038, 4120] avg=4096.0000 max=4208 max_rank=3 min=4017 min_rank=0 imbalance_ratio=1.027344 max_load_reduction=28560 max_load_reduction_pct=0.871582 moved_to_echo=16448 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[511, 455, 499, 520, 536, 537, 508, 528, 0, 0, 0, 0, 0, 0, 0, 0, 502, 495, 534, 496, 512, 487, 513, 528, 6, 1, 0, 0, 0, 0, 0, 0, 538, 545, 497, 500, 510, 527, 551, 512, 0, 0, 0, 0, 0, 0, 0, 0, 508, 529, 507, 461, 515, 517, 475, 498, 22, 0, 0, 0, 0, 0, 0, 0, 489, 490, 513, 542, 483, 481, 507, 533, 14, 0, 0, 0, 0, 0, 0, 0, 520, 515, 530, 488, 509, 523, 562, 513, 3, 1, 0, 0, 0, 0, 0, 0, 509, 505, 500, 522, 526, 482, 512, 525, 0, 0, 0, 0, 0, 0, 0, 0, 518, 493, 480, 532, 548, 473, 540, 502, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4074, 4180, 4032, 4052, 4164, 4081, 4091] avg=4096.0000 max=4180 max_rank=2 min=4032 min_rank=3 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[517, 484, 507, 528, 525, 512, 537, 502, 0, 0, 0, 0, 0, 0, 0, 0, 503, 478, 521, 492, 514, 516, 513, 510, 9, 3, 0, 0, 0, 0, 0, 0, 513, 544, 483, 507, 529, 514, 516, 479, 0, 0, 0, 0, 0, 0, 0, 0, 501, 508, 484, 513, 506, 508, 515, 521, 27, 0, 0, 0, 0, 0, 0, 0, 539, 561, 500, 478, 508, 520, 505, 510, 17, 0, 0, 0, 0, 0, 0, 0, 523, 498, 555, 537, 523, 535, 507, 492, 7, 7, 0, 0, 0, 0, 0, 0, 489, 527, 494, 507, 509, 492, 497, 507, 0, 0, 0, 0, 0, 0, 0, 0, 537, 518, 510, 500, 495, 503, 499, 515, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4059, 4085, 4083, 4138, 4184, 4022, 4085] avg=4096.0000 max=4184 max_rank=5 min=4022 min_rank=6 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[502, 530, 531, 514, 537, 513, 513, 508, 0, 0, 0, 0, 0, 0, 0, 0, 540, 500, 518, 505, 537, 516, 499, 502, 6, 1, 0, 0, 0, 0, 0, 0, 488, 507, 504, 543, 508, 529, 522, 491, 0, 0, 0, 0, 0, 0, 0, 0, 495, 477, 532, 453, 525, 511, 470, 526, 23, 0, 0, 0, 0, 0, 0, 0, 520, 506, 504, 498, 534, 522, 524, 503, 14, 0, 0, 0, 0, 0, 0, 0, 497, 494, 478, 535, 527, 505, 498, 503, 3, 1, 0, 0, 0, 0, 0, 0, 530, 551, 492, 511, 501, 521, 535, 493, 0, 0, 0, 0, 0, 0, 0, 0, 563, 500, 517, 519, 490, 508, 515, 475, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4148, 4124, 4092, 4012, 4125, 4041, 4134, 4092] avg=4096.0000 max=4148 max_rank=0 min=4012 min_rank=3 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[504, 529, 497, 515, 522, 491, 509, 549, 0, 0, 0, 0, 0, 0, 0, 0, 511, 542, 508, 511, 535, 518, 518, 547, 6, 1, 0, 0, 0, 0, 0, 0, 515, 521, 458, 508, 524, 516, 524, 485, 0, 0, 0, 0, 0, 0, 0, 0, 503, 522, 506, 505, 527, 524, 474, 481, 21, 0, 0, 0, 0, 0, 0, 0, 512, 531, 509, 504, 475, 556, 477, 520, 13, 0, 0, 0, 0, 0, 0, 0, 505, 507, 494, 482, 499, 464, 503, 513, 3, 1, 0, 0, 0, 0, 0, 0, 503, 515, 530, 552, 505, 520, 564, 473, 0, 0, 0, 0, 0, 0, 0, 0, 479, 510, 480, 487, 535, 539, 514, 562, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4197, 4051, 4063, 4097, 3971, 4162, 4111] avg=4096.0000 max=4197 max_rank=1 min=3971 min_rank=5 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16341 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[484, 526, 519, 536, 539, 486, 522, 496, 0, 0, 0, 0, 0, 0, 0, 0, 527, 514, 511, 482, 515, 478, 507, 504, 5, 1, 0, 0, 0, 0, 0, 0, 532, 506, 500, 492, 534, 527, 531, 546, 0, 0, 0, 0, 0, 0, 0, 0, 531, 520, 518, 524, 500, 483, 515, 499, 23, 0, 0, 0, 0, 0, 0, 0, 509, 504, 488, 525, 515, 505, 543, 529, 14, 0, 0, 0, 0, 0, 0, 0, 543, 511, 507, 463, 516, 519, 531, 502, 3, 1, 0, 0, 0, 0, 0, 0, 522, 488, 500, 500, 532, 516, 511, 492, 0, 0, 0, 0, 0, 0, 0, 0, 512, 511, 491, 506, 512, 488, 539, 482, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4044, 4168, 4113, 4132, 4096, 4061, 4046] avg=4096.0000 max=4168 max_rank=2 min=4044 min_rank=1 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16335 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[503, 530, 509, 465, 493, 499, 509, 534, 0, 0, 0, 0, 0, 0, 0, 0, 549, 530, 547, 501, 493, 510, 503, 482, 6, 1, 0, 0, 0, 0, 0, 0, 527, 529, 498, 472, 503, 508, 501, 555, 0, 0, 0, 0, 0, 0, 0, 0, 480, 504, 528, 508, 537, 498, 503, 510, 22, 0, 0, 0, 0, 0, 0, 0, 525, 479, 537, 509, 469, 500, 482, 548, 14, 0, 0, 0, 0, 0, 0, 0, 490, 513, 502, 492, 516, 516, 523, 535, 3, 1, 0, 0, 0, 0, 0, 0, 515, 501, 515, 503, 529, 530, 494, 499, 0, 0, 0, 0, 0, 0, 0, 0, 522, 526, 508, 562, 512, 519, 504, 523, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4042, 4122, 4093, 4090, 4063, 4091, 4086, 4181] avg=4096.0000 max=4181 max_rank=7 min=4042 min_rank=0 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16421 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 3968, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[532, 493, 529, 498, 529, 494, 489, 495, 509, 532, 534, 480, 508, 514, 541, 520, 484, 534, 518, 505, 501, 509, 493, 497, 529, 526, 495, 511, 483, 494, 488, 514, 548, 502, 565, 510, 504, 492, 515, 504, 481, 501, 520, 520, 483, 511, 541, 527, 517, 507, 526, 536, 528, 537, 534, 481, 517, 512, 485, 518, 519, 523, 516, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[476, 480, 522, 547, 491, 518, 537, 531, 499, 532, 526, 499, 550, 467, 497, 546, 559, 518, 530, 514, 540, 524, 516, 534, 543, 527, 498, 486, 492, 483, 530, 547, 475, 501, 560, 530, 493, 481, 485, 494, 460, 503, 482, 530, 514, 517, 515, 494, 516, 479, 493, 520, 543, 551, 543, 532, 479, 508, 499, 530, 494, 508, 497, 483] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[479, 476, 515, 534, 516, 502, 517, 479, 498, 520, 505, 518, 472, 548, 500, 519, 516, 498, 527, 506, 512, 494, 506, 569, 534, 501, 513, 535, 496, 496, 506, 504, 522, 505, 484, 525, 480, 504, 545, 532, 510, 487, 525, 532, 541, 456, 512, 527, 497, 513, 479, 510, 527, 553, 539, 516, 539, 537, 517, 484, 526, 492, 514, 527] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[500, 489, 506, 527, 500, 492, 538, 533, 512, 507, 483, 494, 501, 473, 512, 498, 541, 465, 500, 527, 510, 513, 519, 508, 491, 519, 518, 490, 510, 543, 491, 541, 514, 486, 497, 515, 526, 529, 552, 515, 484, 539, 498, 545, 530, 538, 535, 527, 497, 502, 546, 512, 500, 523, 537, 483, 513, 523, 516, 498, 508, 491, 529, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[512, 476, 519, 511, 478, 527, 494, 508, 533, 509, 477, 492, 533, 513, 531, 508, 498, 508, 524, 499, 513, 492, 503, 524, 536, 499, 517, 505, 499, 510, 497, 517, 524, 512, 510, 518, 542, 504, 489, 491, 566, 521, 500, 556, 532, 525, 524, 502, 496, 527, 490, 525, 531, 466, 537, 536, 518, 478, 509, 488, 534, 501, 515, 539] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[508, 506, 489, 521, 525, 526, 541, 535, 516, 530, 509, 506, 529, 541, 532, 455, 520, 512, 513, 521, 502, 548, 511, 505, 506, 527, 499, 538, 536, 513, 535, 501, 508, 496, 489, 496, 524, 462, 496, 508, 509, 526, 513, 528, 496, 507, 509, 490, 497, 482, 484, 509, 527, 510, 531, 504, 516, 487, 515, 524, 510, 542, 483, 534] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[487, 508, 512, 525, 533, 512, 504, 505, 522, 509, 542, 524, 512, 514, 503, 530, 524, 518, 510, 515, 493, 509, 535, 485, 529, 511, 502, 513, 501, 528, 527, 500, 531, 508, 523, 501, 490, 521, 548, 513, 494, 500, 517, 498, 521, 535, 509, 491, 492, 492, 499, 509, 516, 495, 519, 491, 514, 528, 525, 487, 518, 499, 539, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[469, 477, 503, 550, 532, 511, 506, 532, 522, 479, 499, 498, 541, 523, 529, 498, 530, 471, 531, 521, 493, 541, 523, 506, 524, 516, 500, 533, 523, 534, 519, 491, 495, 539, 504, 513, 541, 460, 494, 476, 521, 496, 559, 480, 517, 512, 522, 475, 491, 508, 530, 502, 500, 550, 522, 512, 501, 496, 508, 492, 519, 558, 522, 528] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[532, 493, 529, 498, 529, 494, 489, 495, 13, 5, 0, 0, 0, 0, 0, 0, 509, 532, 534, 480, 508, 514, 541, 520, 0, 0, 0, 0, 0, 0, 0, 0, 471, 534, 518, 505, 501, 509, 493, 497, 0, 0, 0, 0, 0, 0, 0, 0, 523, 526, 495, 511, 483, 494, 488, 514, 0, 0, 0, 0, 0, 0, 0, 0, 548, 502, 565, 510, 504, 492, 515, 504, 8, 5, 5, 0, 0, 0, 0, 0, 481, 501, 520, 512, 483, 511, 541, 527, 0, 0, 0, 0, 0, 0, 0, 0, 517, 507, 526, 536, 528, 537, 524, 481, 0, 0, 0, 0, 0, 0, 0, 0, 517, 512, 485, 518, 519, 523, 516, 510, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4138, 4028, 4034, 4158, 4076, 4156, 4101] avg=4096.0000 max=4158 max_rank=4 min=4028 min_rank=2 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16491 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[479, 476, 515, 534, 516, 502, 517, 479, 14, 5, 0, 0, 0, 0, 0, 0, 498, 520, 505, 518, 472, 548, 500, 519, 0, 0, 0, 0, 0, 0, 0, 0, 502, 498, 527, 506, 512, 494, 506, 569, 0, 0, 0, 0, 0, 0, 0, 0, 528, 501, 513, 535, 496, 496, 506, 504, 0, 0, 0, 0, 0, 0, 0, 0, 522, 505, 484, 525, 480, 504, 545, 532, 8, 5, 5, 0, 0, 0, 0, 0, 510, 487, 525, 524, 541, 456, 512, 527, 0, 0, 0, 0, 0, 0, 0, 0, 497, 513, 479, 510, 527, 553, 529, 516, 0, 0, 0, 0, 0, 0, 0, 0, 539, 537, 517, 484, 526, 492, 514, 527, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4037, 4080, 4114, 4079, 4115, 4082, 4124, 4137] avg=4096.0000 max=4137 max_rank=7 min=4037 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16458 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[476, 480, 522, 547, 491, 518, 537, 531, 15, 5, 0, 0, 0, 0, 0, 0, 499, 532, 526, 499, 550, 467, 497, 546, 0, 0, 0, 0, 0, 0, 0, 0, 544, 518, 530, 514, 540, 524, 516, 534, 0, 0, 0, 0, 0, 0, 0, 0, 537, 527, 498, 486, 492, 483, 530, 547, 0, 0, 0, 0, 0, 0, 0, 0, 475, 501, 560, 530, 493, 481, 485, 494, 8, 5, 5, 0, 0, 0, 0, 0, 460, 503, 482, 522, 514, 517, 515, 494, 0, 0, 0, 0, 0, 0, 0, 0, 516, 479, 493, 520, 543, 551, 533, 532, 0, 0, 0, 0, 0, 0, 0, 0, 479, 508, 499, 530, 494, 508, 497, 483, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4122, 4116, 4220, 4100, 4037, 4007, 4167, 3999] avg=4096.0000 max=4220 max_rank=2 min=3999 min_rank=7 imbalance_ratio=1.030273 max_load_reduction=28548 max_load_reduction_pct=0.871216 moved_to_echo=16210 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[500, 489, 506, 527, 500, 492, 538, 533, 15, 5, 0, 0, 0, 0, 0, 0, 512, 507, 483, 494, 501, 473, 512, 498, 0, 0, 0, 0, 0, 0, 0, 0, 526, 465, 500, 527, 510, 513, 519, 508, 0, 0, 0, 0, 0, 0, 0, 0, 486, 519, 518, 490, 510, 543, 491, 541, 0, 0, 0, 0, 0, 0, 0, 0, 514, 486, 497, 515, 526, 529, 552, 515, 8, 5, 4, 0, 0, 0, 0, 0, 484, 539, 498, 537, 530, 538, 535, 527, 0, 0, 0, 0, 0, 0, 0, 0, 497, 502, 546, 512, 500, 523, 527, 483, 0, 0, 0, 0, 0, 0, 0, 0, 513, 523, 516, 498, 508, 491, 529, 509, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 3980, 4068, 4098, 4151, 4188, 4090, 4088] avg=4096.0000 max=4188 max_rank=5 min=3980 min_rank=1 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16517 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[508, 506, 489, 521, 525, 526, 541, 535, 14, 5, 0, 0, 0, 0, 0, 0, 516, 530, 509, 506, 529, 541, 532, 455, 0, 0, 0, 0, 0, 0, 0, 0, 506, 512, 513, 521, 502, 548, 511, 505, 0, 0, 0, 0, 0, 0, 0, 0, 501, 527, 499, 538, 536, 513, 535, 501, 0, 0, 0, 0, 0, 0, 0, 0, 508, 496, 489, 496, 524, 462, 496, 508, 8, 5, 4, 0, 0, 0, 0, 0, 509, 526, 513, 520, 496, 507, 509, 490, 0, 0, 0, 0, 0, 0, 0, 0, 497, 482, 484, 509, 527, 510, 521, 504, 0, 0, 0, 0, 0, 0, 0, 0, 516, 487, 515, 524, 510, 542, 483, 534, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4170, 4118, 4118, 4150, 3996, 4070, 4034, 4112] avg=4096.0000 max=4170 max_rank=0 min=3996 min_rank=4 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16212 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[512, 476, 519, 511, 478, 527, 494, 508, 13, 5, 0, 0, 0, 0, 0, 0, 533, 509, 477, 492, 533, 513, 531, 508, 0, 0, 0, 0, 0, 0, 0, 0, 485, 508, 524, 499, 513, 492, 503, 524, 0, 0, 0, 0, 0, 0, 0, 0, 530, 499, 517, 505, 499, 510, 497, 517, 0, 0, 0, 0, 0, 0, 0, 0, 524, 512, 510, 518, 542, 504, 489, 491, 9, 5, 5, 0, 0, 0, 0, 0, 566, 521, 500, 547, 532, 525, 524, 502, 0, 0, 0, 0, 0, 0, 0, 0, 496, 527, 490, 525, 531, 466, 527, 536, 0, 0, 0, 0, 0, 0, 0, 0, 518, 478, 509, 488, 534, 501, 515, 539, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4043, 4096, 4048, 4074, 4109, 4217, 4098, 4083] avg=4096.0000 max=4217 max_rank=5 min=4043 min_rank=0 imbalance_ratio=1.029541 max_load_reduction=28551 max_load_reduction_pct=0.871307 moved_to_echo=16507 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[487, 508, 512, 525, 533, 512, 504, 505, 19, 10, 0, 0, 0, 0, 0, 0, 522, 509, 542, 524, 507, 514, 503, 530, 0, 0, 0, 0, 0, 0, 0, 0, 505, 518, 510, 515, 493, 509, 535, 485, 0, 0, 0, 0, 0, 0, 0, 0, 517, 511, 502, 513, 501, 528, 527, 500, 0, 0, 0, 0, 0, 0, 0, 0, 531, 508, 523, 501, 490, 521, 548, 513, 12, 9, 7, 0, 0, 0, 0, 0, 494, 500, 517, 486, 521, 535, 509, 491, 0, 0, 0, 0, 0, 0, 0, 0, 492, 492, 499, 509, 516, 495, 500, 491, 0, 0, 0, 0, 0, 0, 0, 0, 514, 528, 525, 487, 518, 499, 539, 503, 5, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4151, 4070, 4099, 4163, 4053, 3994, 4123] avg=4096.0000 max=4163 max_rank=4 min=3994 min_rank=6 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16333 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[469, 477, 503, 550, 532, 511, 506, 532, 14, 5, 0, 0, 0, 0, 0, 0, 522, 479, 499, 498, 541, 523, 529, 498, 0, 0, 0, 0, 0, 0, 0, 0, 516, 471, 531, 521, 493, 541, 523, 506, 0, 0, 0, 0, 0, 0, 0, 0, 518, 516, 500, 533, 523, 534, 519, 491, 0, 0, 0, 0, 0, 0, 0, 0, 495, 539, 504, 513, 541, 460, 494, 476, 7, 5, 5, 0, 0, 0, 0, 0, 521, 496, 559, 473, 517, 512, 522, 475, 0, 0, 0, 0, 0, 0, 0, 0, 491, 508, 530, 502, 500, 550, 512, 512, 0, 0, 0, 0, 0, 0, 0, 0, 501, 496, 508, 492, 519, 558, 522, 528, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4089, 4102, 4134, 4039, 4075, 4105, 4125] avg=4096.0000 max=4134 max_rank=3 min=4039 min_rank=4 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16344 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3968, 3968, 4096, 4224, 4224, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 3968, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[485, 484, 507, 538, 545, 505, 486, 503, 523, 458, 519, 488, 493, 500, 542, 508, 477, 505, 463, 498, 499, 502, 503, 534, 503, 551, 544, 521, 520, 492, 505, 565, 513, 534, 531, 520, 522, 518, 505, 538, 526, 483, 492, 491, 503, 503, 543, 508, 556, 489, 532, 495, 504, 492, 517, 548, 481, 523, 498, 555, 508, 524, 531, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[512, 513, 572, 513, 529, 524, 514, 516, 487, 522, 510, 508, 537, 529, 487, 501, 514, 540, 526, 508, 525, 484, 536, 543, 485, 500, 513, 505, 516, 511, 499, 547, 524, 485, 496, 504, 484, 508, 515, 505, 537, 505, 484, 532, 509, 519, 501, 550, 499, 522, 520, 488, 468, 521, 498, 495, 540, 522, 490, 496, 486, 528, 495, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[529, 516, 520, 495, 512, 527, 525, 478, 496, 482, 508, 484, 470, 536, 510, 495, 491, 465, 510, 500, 535, 512, 554, 474, 549, 504, 551, 532, 536, 572, 521, 514, 514, 520, 510, 521, 504, 509, 538, 512, 506, 496, 522, 507, 536, 486, 484, 536, 570, 516, 510, 508, 482, 511, 530, 462, 479, 520, 504, 544, 524, 489, 513, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[487, 512, 512, 510, 513, 509, 511, 520, 535, 519, 521, 565, 520, 494, 504, 534, 487, 519, 510, 526, 520, 503, 467, 497, 520, 514, 498, 489, 527, 523, 501, 525, 493, 494, 495, 538, 487, 494, 524, 528, 530, 524, 533, 507, 500, 491, 465, 557, 520, 473, 549, 518, 527, 526, 536, 522, 493, 515, 544, 506, 496, 481, 492, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[498, 566, 493, 515, 503, 527, 527, 520, 550, 490, 531, 532, 501, 506, 510, 499, 490, 547, 498, 474, 494, 524, 514, 506, 497, 491, 516, 573, 514, 519, 499, 482, 534, 550, 524, 503, 464, 515, 475, 515, 533, 518, 490, 501, 500, 524, 536, 482, 524, 511, 519, 508, 464, 564, 515, 510, 500, 511, 501, 513, 510, 531, 504, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[511, 499, 515, 528, 538, 507, 518, 504, 515, 539, 526, 456, 539, 522, 536, 498, 500, 490, 502, 477, 535, 490, 538, 500, 505, 482, 511, 530, 494, 551, 539, 490, 537, 544, 509, 505, 494, 467, 508, 522, 519, 506, 539, 507, 570, 533, 501, 508, 522, 525, 472, 497, 491, 476, 542, 515, 504, 488, 518, 497, 525, 533, 516, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[532, 535, 532, 494, 512, 541, 464, 516, 502, 509, 481, 496, 525, 526, 510, 507, 520, 493, 516, 505, 518, 489, 521, 522, 515, 545, 530, 525, 563, 511, 492, 549, 508, 525, 512, 519, 516, 485, 511, 526, 482, 518, 510, 556, 520, 509, 489, 469, 509, 547, 531, 465, 533, 510, 499, 513, 486, 502, 502, 489, 529, 505, 493, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[474, 532, 493, 479, 498, 487, 507, 504, 515, 563, 510, 536, 510, 525, 455, 477, 542, 541, 470, 508, 502, 511, 544, 532, 534, 510, 540, 496, 493, 524, 514, 496, 499, 530, 490, 504, 532, 525, 538, 534, 530, 523, 532, 528, 511, 493, 485, 520, 510, 516, 516, 527, 531, 529, 476, 498, 492, 494, 489, 516, 490, 530, 536, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[485, 482, 507, 538, 545, 505, 486, 503, 0, 0, 0, 0, 0, 0, 0, 0, 523, 458, 519, 488, 493, 500, 542, 508, 4, 3, 2, 0, 0, 0, 0, 0, 477, 505, 463, 498, 499, 502, 503, 534, 26, 0, 0, 0, 0, 0, 0, 0, 503, 551, 544, 521, 520, 445, 505, 565, 0, 0, 0, 0, 0, 0, 0, 0, 513, 530, 531, 520, 522, 518, 505, 538, 0, 0, 0, 0, 0, 0, 0, 0, 522, 483, 492, 491, 503, 503, 543, 508, 0, 0, 0, 0, 0, 0, 0, 0, 556, 489, 532, 495, 504, 492, 517, 548, 0, 0, 0, 0, 0, 0, 0, 0, 481, 523, 498, 555, 508, 524, 531, 519, 21, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4051, 4040, 4007, 4154, 4177, 4045, 4133, 4161] avg=4096.0000 max=4177 max_rank=4 min=4007 min_rank=2 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16516 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[512, 511, 572, 513, 529, 524, 514, 516, 0, 0, 0, 0, 0, 0, 0, 0, 487, 522, 510, 508, 537, 529, 487, 501, 4, 3, 2, 0, 0, 0, 0, 0, 514, 540, 526, 508, 525, 484, 536, 543, 27, 0, 0, 0, 0, 0, 0, 0, 485, 500, 513, 505, 516, 462, 499, 547, 0, 0, 0, 0, 0, 0, 0, 0, 524, 481, 496, 504, 484, 508, 515, 505, 0, 0, 0, 0, 0, 0, 0, 0, 533, 505, 484, 532, 509, 519, 501, 550, 0, 0, 0, 0, 0, 0, 0, 0, 499, 522, 520, 488, 468, 521, 498, 495, 0, 0, 0, 0, 0, 0, 0, 0, 540, 522, 490, 496, 486, 528, 495, 516, 22, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4191, 4090, 4203, 4027, 4017, 4133, 4011, 4096] avg=4096.0000 max=4203 max_rank=2 min=4011 min_rank=6 imbalance_ratio=1.026123 max_load_reduction=28565 max_load_reduction_pct=0.871735 moved_to_echo=16257 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[487, 510, 512, 510, 513, 509, 511, 520, 0, 0, 0, 0, 0, 0, 0, 0, 535, 519, 521, 565, 520, 494, 504, 534, 4, 3, 2, 0, 0, 0, 0, 0, 487, 519, 510, 526, 520, 503, 467, 497, 28, 0, 0, 0, 0, 0, 0, 0, 520, 514, 498, 489, 527, 472, 501, 525, 0, 0, 0, 0, 0, 0, 0, 0, 493, 490, 495, 538, 487, 494, 524, 528, 0, 0, 0, 0, 0, 0, 0, 0, 526, 524, 533, 507, 500, 491, 465, 557, 0, 0, 0, 0, 0, 0, 0, 0, 520, 473, 549, 518, 527, 526, 536, 522, 0, 0, 0, 0, 0, 0, 0, 0, 493, 515, 544, 506, 496, 481, 492, 518, 23, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4201, 4057, 4046, 4049, 4103, 4171, 4069] avg=4096.0000 max=4201 max_rank=1 min=4046 min_rank=3 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[529, 514, 520, 495, 512, 527, 525, 478, 0, 0, 0, 0, 0, 0, 0, 0, 496, 482, 508, 484, 470, 536, 510, 495, 4, 3, 2, 0, 0, 0, 0, 0, 491, 465, 510, 500, 535, 512, 554, 474, 31, 0, 0, 0, 0, 0, 0, 0, 549, 504, 551, 532, 536, 516, 521, 514, 0, 0, 0, 0, 0, 0, 0, 0, 514, 516, 510, 521, 504, 509, 538, 512, 0, 0, 0, 0, 0, 0, 0, 0, 502, 496, 522, 507, 536, 486, 484, 536, 0, 0, 0, 0, 0, 0, 0, 0, 570, 516, 510, 508, 482, 511, 530, 462, 0, 0, 0, 0, 0, 0, 0, 0, 479, 520, 504, 544, 524, 489, 513, 502, 25, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 3990, 4072, 4223, 4124, 4069, 4089, 4101] avg=4096.0000 max=4223 max_rank=3 min=3990 min_rank=1 imbalance_ratio=1.031006 max_load_reduction=28545 max_load_reduction_pct=0.871124 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[511, 497, 515, 528, 538, 507, 518, 504, 0, 0, 0, 0, 0, 0, 0, 0, 515, 539, 526, 456, 539, 522, 536, 498, 4, 3, 2, 0, 0, 0, 0, 0, 500, 490, 502, 477, 535, 490, 538, 500, 29, 0, 0, 0, 0, 0, 0, 0, 505, 482, 511, 530, 494, 498, 539, 490, 0, 0, 0, 0, 0, 0, 0, 0, 537, 540, 509, 505, 494, 467, 508, 522, 0, 0, 0, 0, 0, 0, 0, 0, 515, 506, 539, 507, 570, 533, 501, 508, 0, 0, 0, 0, 0, 0, 0, 0, 522, 525, 472, 497, 491, 476, 542, 515, 0, 0, 0, 0, 0, 0, 0, 0, 504, 488, 518, 497, 525, 533, 516, 493, 24, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4140, 4061, 4049, 4082, 4179, 4040, 4099] avg=4096.0000 max=4179 max_rank=5 min=4040 min_rank=6 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16400 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[498, 557, 493, 515, 503, 527, 527, 520, 0, 0, 0, 0, 0, 0, 0, 0, 550, 490, 531, 532, 501, 506, 510, 499, 7, 6, 9, 1, 0, 0, 0, 0, 490, 547, 498, 474, 494, 524, 514, 506, 32, 0, 0, 0, 0, 0, 0, 0, 497, 491, 516, 573, 514, 460, 499, 482, 0, 0, 0, 0, 0, 0, 0, 0, 534, 543, 524, 503, 464, 515, 475, 515, 0, 0, 0, 0, 0, 0, 0, 0, 521, 518, 490, 501, 500, 524, 536, 482, 0, 0, 0, 0, 0, 0, 0, 0, 523, 511, 519, 508, 464, 564, 515, 510, 0, 0, 0, 0, 0, 0, 0, 0, 500, 511, 501, 513, 510, 531, 504, 513, 27, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4140, 4142, 4079, 4032, 4073, 4072, 4114, 4116] avg=4096.0000 max=4142 max_rank=1 min=4032 min_rank=3 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[532, 533, 532, 494, 512, 541, 464, 516, 0, 0, 0, 0, 0, 0, 0, 0, 502, 509, 481, 496, 525, 526, 510, 507, 4, 3, 2, 0, 0, 0, 0, 0, 520, 493, 516, 505, 518, 489, 521, 522, 27, 0, 0, 0, 0, 0, 0, 0, 515, 545, 530, 525, 563, 462, 492, 549, 0, 0, 0, 0, 0, 0, 0, 0, 508, 521, 512, 519, 516, 485, 511, 526, 0, 0, 0, 0, 0, 0, 0, 0, 478, 518, 510, 556, 520, 509, 489, 469, 0, 0, 0, 0, 0, 0, 0, 0, 509, 547, 531, 465, 533, 510, 499, 513, 0, 0, 0, 0, 0, 0, 0, 0, 486, 502, 502, 489, 529, 505, 493, 504, 22, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4124, 4065, 4111, 4181, 4098, 4049, 4107, 4033] avg=4096.0000 max=4181 max_rank=3 min=4033 min_rank=7 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16287 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[474, 530, 493, 479, 498, 487, 507, 504, 0, 0, 0, 0, 0, 0, 0, 0, 515, 563, 510, 536, 510, 525, 455, 477, 4, 3, 2, 0, 0, 0, 0, 0, 542, 541, 470, 508, 502, 511, 544, 532, 28, 0, 0, 0, 0, 0, 0, 0, 534, 510, 540, 496, 493, 473, 514, 496, 0, 0, 0, 0, 0, 0, 0, 0, 499, 526, 490, 504, 532, 525, 538, 534, 0, 0, 0, 0, 0, 0, 0, 0, 526, 523, 532, 528, 511, 493, 485, 520, 0, 0, 0, 0, 0, 0, 0, 0, 510, 516, 516, 527, 531, 529, 476, 498, 0, 0, 0, 0, 0, 0, 0, 0, 492, 494, 489, 516, 490, 530, 536, 522, 23, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3972, 4100, 4178, 4056, 4148, 4118, 4103, 4093] avg=4096.0000 max=4178 max_rank=2 min=3972 min_rank=0 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16462 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 3840, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 128, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[511, 498, 492, 519, 495, 546, 545, 552, 498, 514, 551, 526, 511, 514, 542, 508, 480, 492, 507, 503, 547, 503, 544, 514, 477, 491, 502, 526, 470, 506, 483, 512, 498, 509, 526, 524, 515, 510, 500, 512, 510, 559, 452, 564, 509, 526, 497, 507, 498, 545, 513, 518, 490, 521, 519, 527, 522, 538, 494, 499, 505, 497, 492, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[489, 491, 493, 490, 495, 495, 533, 542, 500, 538, 516, 503, 505, 507, 529, 523, 514, 504, 486, 497, 528, 528, 501, 492, 530, 514, 480, 524, 548, 494, 535, 525, 501, 483, 559, 526, 531, 502, 535, 528, 500, 513, 507, 536, 553, 531, 545, 499, 483, 500, 480, 490, 521, 525, 529, 501, 492, 518, 495, 509, 535, 476, 494, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[469, 500, 514, 560, 483, 511, 497, 534, 523, 517, 491, 532, 518, 503, 508, 513, 525, 506, 518, 497, 480, 518, 508, 550, 499, 541, 523, 561, 510, 546, 475, 495, 470, 531, 500, 507, 488, 501, 480, 521, 519, 518, 459, 502, 501, 522, 510, 550, 513, 523, 528, 492, 489, 507, 495, 581, 509, 519, 515, 513, 487, 519, 566, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[501, 517, 522, 504, 497, 522, 520, 506, 494, 518, 516, 503, 508, 507, 548, 497, 499, 527, 501, 496, 548, 490, 535, 512, 515, 488, 521, 512, 514, 532, 538, 504, 502, 518, 527, 489, 509, 531, 527, 503, 505, 538, 516, 495, 508, 480, 515, 543, 489, 523, 511, 510, 529, 538, 497, 522, 515, 486, 536, 508, 495, 517, 486, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[550, 486, 489, 528, 506, 510, 521, 519, 536, 499, 501, 520, 502, 499, 509, 482, 499, 516, 529, 510, 495, 520, 505, 540, 504, 524, 480, 500, 518, 516, 479, 520, 476, 568, 537, 527, 545, 529, 483, 522, 501, 512, 535, 514, 522, 526, 516, 528, 480, 528, 506, 539, 534, 493, 497, 514, 510, 483, 504, 499, 497, 519, 527, 485] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[491, 494, 505, 498, 519, 502, 512, 488, 498, 458, 511, 558, 486, 522, 541, 516, 513, 505, 480, 554, 521, 516, 477, 581, 517, 517, 485, 548, 523, 516, 532, 521, 540, 479, 524, 533, 498, 502, 504, 510, 538, 494, 512, 524, 526, 491, 448, 517, 537, 530, 511, 517, 478, 526, 518, 507, 493, 515, 499, 521, 510, 497, 536, 528] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[507, 472, 518, 530, 526, 493, 521, 505, 517, 561, 556, 509, 488, 517, 492, 500, 515, 513, 514, 521, 528, 480, 503, 535, 469, 536, 526, 487, 512, 532, 493, 495, 511, 518, 530, 538, 457, 507, 522, 512, 534, 499, 500, 497, 513, 504, 518, 526, 509, 518, 501, 499, 514, 486, 533, 504, 529, 523, 519, 539, 515, 513, 497, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[537, 494, 570, 495, 534, 512, 546, 532, 526, 530, 485, 495, 514, 516, 481, 476, 494, 541, 467, 506, 524, 534, 513, 507, 513, 491, 533, 465, 538, 483, 534, 530, 536, 519, 524, 525, 509, 549, 532, 510, 513, 492, 500, 488, 515, 517, 535, 480, 488, 532, 507, 499, 515, 529, 512, 506, 505, 480, 482, 514, 507, 485, 533, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[489, 491, 493, 490, 495, 495, 533, 542, 1, 0, 0, 0, 0, 0, 0, 0, 500, 538, 516, 503, 505, 507, 528, 523, 0, 0, 0, 0, 0, 0, 0, 0, 514, 504, 486, 497, 528, 528, 501, 488, 0, 0, 0, 0, 0, 0, 0, 0, 530, 514, 480, 524, 548, 494, 535, 525, 4, 0, 0, 0, 0, 0, 0, 0, 501, 483, 537, 526, 531, 502, 535, 528, 0, 0, 0, 0, 0, 0, 0, 0, 500, 513, 507, 536, 553, 531, 545, 493, 0, 0, 0, 0, 0, 0, 0, 0, 483, 500, 480, 490, 521, 525, 529, 501, 0, 0, 0, 0, 0, 0, 0, 0, 492, 518, 495, 509, 535, 476, 494, 522, 22, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4029, 4120, 4046, 4154, 4143, 4178, 4029, 4069] avg=4096.0000 max=4178 max_rank=5 min=4029 min_rank=0 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16419 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[511, 498, 492, 519, 495, 546, 545, 552, 1, 0, 0, 0, 0, 0, 0, 0, 498, 514, 551, 526, 511, 514, 541, 508, 0, 0, 0, 0, 0, 0, 0, 0, 480, 492, 507, 503, 547, 503, 544, 510, 0, 0, 0, 0, 0, 0, 0, 0, 477, 491, 502, 526, 470, 506, 483, 512, 4, 0, 0, 0, 0, 0, 0, 0, 498, 509, 505, 524, 515, 510, 500, 512, 0, 0, 0, 0, 0, 0, 0, 0, 510, 559, 452, 564, 509, 526, 497, 501, 0, 0, 0, 0, 0, 0, 0, 0, 498, 545, 513, 518, 490, 521, 519, 527, 0, 0, 0, 0, 0, 0, 0, 0, 522, 538, 494, 499, 505, 497, 492, 493, 21, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4159, 4163, 4086, 3971, 4073, 4118, 4131, 4067] avg=4096.0000 max=4163 max_rank=1 min=3971 min_rank=3 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[507, 472, 518, 530, 526, 493, 521, 505, 1, 0, 0, 0, 0, 0, 0, 0, 517, 561, 556, 509, 488, 517, 491, 500, 0, 0, 0, 0, 0, 0, 0, 0, 515, 513, 514, 521, 528, 480, 503, 531, 0, 0, 0, 0, 0, 0, 0, 0, 469, 536, 526, 487, 512, 532, 493, 495, 4, 0, 0, 0, 0, 0, 0, 0, 511, 518, 509, 538, 457, 507, 522, 512, 0, 0, 0, 0, 0, 0, 0, 0, 534, 499, 500, 497, 513, 504, 518, 519, 0, 0, 0, 0, 0, 0, 0, 0, 509, 518, 501, 499, 514, 486, 533, 504, 0, 0, 0, 0, 0, 0, 0, 0, 529, 523, 519, 539, 515, 513, 497, 512, 21, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4139, 4105, 4054, 4074, 4084, 4064, 4175] avg=4096.0000 max=4175 max_rank=7 min=4054 min_rank=3 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[550, 486, 489, 528, 506, 510, 521, 519, 1, 0, 0, 0, 0, 0, 0, 0, 536, 499, 501, 520, 502, 499, 508, 482, 0, 0, 0, 0, 0, 0, 0, 0, 499, 516, 529, 510, 495, 520, 505, 536, 0, 0, 0, 0, 0, 0, 0, 0, 504, 524, 480, 500, 518, 516, 479, 520, 4, 0, 0, 0, 0, 0, 0, 0, 476, 568, 516, 527, 545, 529, 483, 522, 0, 0, 0, 0, 0, 0, 0, 0, 501, 512, 535, 514, 522, 526, 516, 521, 0, 0, 0, 0, 0, 0, 0, 0, 480, 528, 506, 539, 534, 493, 497, 514, 0, 0, 0, 0, 0, 0, 0, 0, 510, 483, 504, 499, 497, 519, 527, 485, 21, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4047, 4110, 4045, 4166, 4147, 4091, 4052] avg=4096.0000 max=4166 max_rank=4 min=4045 min_rank=3 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16456 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[469, 500, 514, 560, 483, 511, 497, 534, 1, 0, 0, 0, 0, 0, 0, 0, 523, 517, 491, 532, 518, 503, 507, 513, 0, 0, 0, 0, 0, 0, 0, 0, 525, 506, 518, 497, 480, 518, 508, 546, 0, 0, 0, 0, 0, 0, 0, 0, 499, 541, 523, 561, 510, 546, 475, 495, 4, 0, 0, 0, 0, 0, 0, 0, 470, 531, 480, 507, 488, 501, 480, 521, 0, 0, 0, 0, 0, 0, 0, 0, 519, 518, 459, 502, 501, 522, 510, 543, 0, 0, 0, 0, 0, 0, 0, 0, 513, 523, 528, 492, 489, 507, 495, 581, 0, 0, 0, 0, 0, 0, 0, 0, 509, 519, 515, 513, 487, 519, 566, 508, 20, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4104, 4098, 4154, 3978, 4074, 4128, 4163] avg=4096.0000 max=4163 max_rank=7 min=3978 min_rank=4 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16343 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[501, 517, 522, 504, 497, 522, 520, 506, 2, 6, 0, 0, 0, 0, 0, 0, 494, 518, 516, 503, 508, 507, 541, 497, 0, 0, 0, 0, 0, 0, 0, 0, 499, 527, 501, 496, 548, 490, 535, 502, 0, 0, 0, 0, 0, 0, 0, 0, 515, 488, 521, 512, 514, 532, 538, 504, 7, 5, 0, 0, 0, 0, 0, 0, 502, 518, 503, 489, 509, 531, 527, 503, 0, 0, 0, 0, 0, 0, 0, 0, 505, 538, 516, 495, 508, 480, 515, 532, 0, 0, 0, 0, 0, 0, 0, 0, 489, 517, 511, 510, 529, 538, 497, 522, 0, 0, 0, 0, 0, 0, 0, 0, 515, 486, 536, 508, 495, 517, 486, 488, 24, 11, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4084, 4098, 4136, 4082, 4089, 4113, 4069] avg=4096.0000 max=4136 max_rank=3 min=4069 min_rank=7 imbalance_ratio=1.009766 max_load_reduction=28632 max_load_reduction_pct=0.873779 moved_to_echo=16353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[491, 494, 505, 498, 519, 502, 512, 488, 1, 0, 0, 0, 0, 0, 0, 0, 498, 458, 511, 558, 486, 522, 540, 516, 0, 0, 0, 0, 0, 0, 0, 0, 513, 505, 480, 554, 521, 516, 477, 577, 0, 0, 0, 0, 0, 0, 0, 0, 517, 517, 485, 548, 523, 516, 532, 521, 4, 0, 0, 0, 0, 0, 0, 0, 540, 479, 503, 533, 498, 502, 504, 510, 0, 0, 0, 0, 0, 0, 0, 0, 538, 494, 512, 524, 526, 491, 448, 511, 0, 0, 0, 0, 0, 0, 0, 0, 537, 530, 511, 517, 478, 526, 518, 507, 0, 0, 0, 0, 0, 0, 0, 0, 493, 515, 499, 521, 510, 497, 536, 528, 21, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4010, 4089, 4143, 4163, 4069, 4044, 4124, 4126] avg=4096.0000 max=4163 max_rank=3 min=4010 min_rank=0 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[537, 494, 570, 495, 534, 512, 546, 532, 1, 0, 0, 0, 0, 0, 0, 0, 526, 530, 485, 495, 514, 516, 480, 476, 0, 0, 0, 0, 0, 0, 0, 0, 494, 541, 467, 506, 524, 534, 513, 503, 0, 0, 0, 0, 0, 0, 0, 0, 513, 491, 533, 465, 538, 483, 534, 530, 4, 0, 0, 0, 0, 0, 0, 0, 536, 519, 503, 525, 509, 549, 532, 510, 0, 0, 0, 0, 0, 0, 0, 0, 513, 492, 500, 488, 515, 517, 535, 474, 0, 0, 0, 0, 0, 0, 0, 0, 488, 532, 507, 499, 515, 529, 512, 506, 0, 0, 0, 0, 0, 0, 0, 0, 505, 480, 482, 514, 507, 485, 533, 514, 21, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4221, 4022, 4082, 4091, 4183, 4034, 4088, 4047] avg=4096.0000 max=4221 max_rank=0 min=4022 min_rank=1 imbalance_ratio=1.030518 max_load_reduction=28547 max_load_reduction_pct=0.871185 moved_to_echo=16352 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 3968, 4224, 4224, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[487, 510, 501, 516, 517, 476, 497, 526, 488, 491, 521, 537, 513, 511, 527, 528, 585, 516, 515, 513, 543, 523, 525, 538, 507, 501, 518, 505, 524, 504, 507, 493, 540, 537, 522, 512, 520, 532, 507, 480, 514, 483, 492, 521, 503, 511, 473, 486, 528, 513, 520, 497, 470, 528, 509, 493, 505, 532, 531, 516, 501, 486, 548, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[523, 510, 535, 511, 482, 503, 493, 518, 515, 486, 530, 528, 504, 508, 475, 526, 518, 515, 520, 510, 512, 537, 495, 503, 556, 518, 522, 523, 498, 482, 545, 506, 471, 520, 497, 550, 497, 522, 486, 502, 479, 530, 517, 478, 534, 500, 512, 500, 509, 494, 554, 516, 547, 518, 514, 472, 499, 514, 543, 546, 497, 495, 534, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[493, 493, 501, 531, 559, 509, 507, 528, 538, 513, 514, 498, 477, 528, 485, 542, 534, 517, 535, 495, 528, 519, 532, 541, 487, 523, 499, 501, 524, 482, 512, 496, 514, 541, 505, 502, 526, 490, 473, 502, 486, 533, 534, 519, 504, 508, 486, 501, 547, 517, 521, 528, 537, 539, 487, 472, 504, 507, 501, 494, 528, 509, 506, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[536, 521, 515, 517, 530, 501, 507, 539, 523, 474, 476, 520, 479, 504, 516, 526, 476, 476, 509, 530, 517, 506, 514, 506, 483, 534, 499, 553, 509, 499, 519, 536, 507, 509, 528, 525, 479, 508, 490, 485, 548, 523, 485, 531, 532, 518, 560, 521, 484, 463, 567, 520, 492, 512, 503, 533, 519, 479, 468, 506, 521, 530, 533, 539] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[451, 484, 557, 506, 518, 513, 522, 531, 508, 521, 510, 546, 532, 473, 521, 508, 489, 517, 497, 500, 514, 505, 494, 519, 518, 534, 494, 528, 516, 507, 523, 473, 496, 525, 535, 500, 513, 534, 514, 501, 516, 481, 524, 515, 573, 528, 468, 557, 520, 492, 500, 478, 542, 501, 508, 494, 522, 493, 498, 514, 523, 528, 532, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[491, 513, 529, 501, 513, 512, 525, 541, 516, 499, 501, 503, 539, 530, 500, 513, 500, 491, 537, 460, 442, 506, 512, 530, 527, 488, 509, 481, 537, 490, 509, 508, 510, 517, 483, 523, 536, 501, 554, 545, 537, 518, 542, 479, 502, 478, 507, 535, 528, 566, 477, 488, 542, 516, 483, 507, 511, 528, 487, 537, 533, 516, 499, 530] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[499, 512, 539, 525, 515, 514, 536, 511, 518, 517, 522, 515, 523, 517, 525, 533, 544, 500, 515, 500, 501, 515, 524, 521, 462, 533, 512, 507, 485, 504, 521, 518, 525, 515, 541, 476, 491, 521, 510, 537, 495, 567, 516, 452, 523, 507, 539, 521, 481, 504, 544, 486, 507, 507, 501, 510, 499, 506, 476, 511, 504, 521, 509, 483] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[502, 554, 506, 531, 476, 478, 520, 546, 515, 517, 516, 522, 513, 498, 522, 507, 498, 459, 500, 507, 529, 525, 486, 494, 508, 499, 534, 503, 520, 532, 491, 494, 511, 487, 516, 490, 538, 543, 530, 529, 505, 510, 533, 504, 550, 490, 484, 545, 497, 510, 516, 491, 554, 489, 505, 511, 506, 549, 517, 523, 535, 518, 484, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[487, 510, 501, 516, 517, 476, 497, 515, 0, 0, 0, 0, 0, 0, 0, 0, 488, 491, 521, 537, 513, 511, 527, 524, 0, 0, 0, 0, 0, 0, 0, 0, 585, 516, 515, 513, 543, 523, 525, 538, 4, 0, 0, 0, 0, 0, 0, 0, 507, 501, 518, 505, 524, 504, 507, 493, 11, 5, 0, 0, 0, 0, 0, 0, 540, 537, 522, 512, 520, 526, 507, 480, 0, 0, 0, 0, 0, 0, 0, 0, 514, 483, 492, 521, 497, 511, 473, 486, 0, 0, 0, 0, 0, 0, 0, 0, 528, 513, 520, 497, 470, 528, 509, 493, 6, 4, 1, 0, 0, 0, 0, 0, 505, 532, 531, 512, 501, 486, 548, 495, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4019, 4112, 4262, 4075, 4144, 3977, 4069, 4110] avg=4096.0000 max=4262 max_rank=2 min=3977 min_rank=5 imbalance_ratio=1.040527 max_load_reduction=28506 max_load_reduction_pct=0.869934 moved_to_echo=16300 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[523, 510, 535, 511, 482, 503, 493, 507, 0, 0, 0, 0, 0, 0, 0, 0, 515, 486, 530, 528, 504, 508, 475, 522, 0, 0, 0, 0, 0, 0, 0, 0, 518, 515, 520, 510, 512, 537, 495, 503, 4, 0, 0, 0, 0, 0, 0, 0, 556, 518, 522, 523, 498, 482, 545, 506, 11, 5, 0, 0, 0, 0, 0, 0, 471, 520, 497, 550, 497, 516, 486, 502, 0, 0, 0, 0, 0, 0, 0, 0, 479, 530, 517, 478, 528, 500, 512, 500, 0, 0, 0, 0, 0, 0, 0, 0, 509, 494, 554, 516, 547, 518, 514, 472, 6, 5, 1, 0, 0, 0, 0, 0, 499, 514, 543, 541, 497, 495, 534, 514, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4064, 4068, 4114, 4166, 4039, 4044, 4136, 4137] avg=4096.0000 max=4166 max_rank=3 min=4039 min_rank=4 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16356 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[536, 521, 515, 517, 530, 501, 507, 527, 0, 0, 0, 0, 0, 0, 0, 0, 523, 474, 476, 520, 479, 504, 516, 522, 0, 0, 0, 0, 0, 0, 0, 0, 476, 476, 509, 530, 517, 506, 514, 506, 4, 0, 0, 0, 0, 0, 0, 0, 483, 534, 499, 553, 509, 499, 519, 536, 12, 5, 0, 0, 0, 0, 0, 0, 507, 509, 528, 525, 479, 502, 490, 485, 0, 0, 0, 0, 0, 0, 0, 0, 548, 523, 485, 531, 526, 518, 560, 521, 0, 0, 0, 0, 0, 0, 0, 0, 484, 463, 567, 520, 492, 512, 503, 533, 6, 4, 1, 0, 0, 0, 0, 0, 519, 479, 468, 502, 521, 530, 533, 539, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4154, 4014, 4038, 4149, 4025, 4212, 4085, 4091] avg=4096.0000 max=4212 max_rank=5 min=4014 min_rank=1 imbalance_ratio=1.028320 max_load_reduction=28556 max_load_reduction_pct=0.871460 moved_to_echo=16413 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[493, 493, 501, 531, 559, 509, 507, 517, 0, 0, 0, 0, 0, 0, 0, 0, 538, 513, 514, 498, 477, 528, 485, 538, 0, 0, 0, 0, 0, 0, 0, 0, 534, 517, 535, 495, 528, 519, 532, 541, 4, 0, 0, 0, 0, 0, 0, 0, 487, 523, 499, 501, 524, 482, 512, 496, 11, 5, 0, 0, 0, 0, 0, 0, 514, 541, 505, 502, 526, 484, 473, 502, 0, 0, 0, 0, 0, 0, 0, 0, 486, 533, 534, 519, 498, 508, 486, 501, 0, 0, 0, 0, 0, 0, 0, 0, 547, 517, 521, 528, 537, 539, 487, 472, 6, 4, 1, 0, 0, 0, 0, 0, 504, 507, 501, 490, 528, 509, 506, 506, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4091, 4205, 4040, 4047, 4065, 4159, 4051] avg=4096.0000 max=4205 max_rank=2 min=4040 min_rank=3 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16322 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[499, 512, 539, 525, 515, 514, 536, 500, 0, 0, 0, 0, 0, 0, 0, 0, 518, 517, 522, 515, 523, 517, 525, 529, 0, 0, 0, 0, 0, 0, 0, 0, 544, 500, 515, 500, 501, 515, 524, 521, 4, 0, 0, 0, 0, 0, 0, 0, 462, 533, 512, 507, 485, 504, 521, 518, 11, 5, 0, 0, 0, 0, 0, 0, 525, 515, 541, 476, 491, 515, 510, 537, 0, 0, 0, 0, 0, 0, 0, 0, 495, 567, 516, 452, 517, 507, 539, 521, 0, 0, 0, 0, 0, 0, 0, 0, 481, 504, 544, 486, 507, 507, 501, 510, 6, 4, 1, 0, 0, 0, 0, 0, 499, 506, 476, 507, 504, 521, 509, 483, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4140, 4166, 4124, 4058, 4110, 4114, 4051, 4005] avg=4096.0000 max=4166 max_rank=1 min=4005 min_rank=7 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16280 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[451, 484, 557, 506, 518, 513, 522, 520, 0, 0, 0, 0, 0, 0, 0, 0, 508, 521, 510, 546, 532, 473, 521, 504, 0, 0, 0, 0, 0, 0, 0, 0, 489, 517, 497, 500, 514, 505, 494, 519, 4, 0, 0, 0, 0, 0, 0, 0, 518, 534, 494, 528, 516, 507, 523, 473, 11, 5, 0, 0, 0, 0, 0, 0, 496, 525, 535, 500, 513, 528, 514, 501, 0, 0, 0, 0, 0, 0, 0, 0, 516, 481, 524, 515, 566, 528, 468, 557, 0, 0, 0, 0, 0, 0, 0, 0, 520, 492, 500, 478, 542, 501, 508, 494, 7, 4, 1, 0, 0, 0, 0, 0, 522, 493, 498, 510, 523, 528, 532, 514, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4071, 4115, 4039, 4109, 4112, 4155, 4047, 4120] avg=4096.0000 max=4155 max_rank=5 min=4039 min_rank=2 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16434 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[491, 513, 529, 501, 513, 512, 525, 525, 0, 0, 0, 0, 0, 0, 0, 0, 516, 499, 501, 503, 539, 530, 500, 507, 0, 0, 0, 0, 0, 0, 0, 0, 500, 491, 537, 460, 442, 506, 512, 530, 6, 1, 0, 0, 0, 0, 0, 0, 527, 488, 509, 481, 537, 490, 509, 508, 16, 8, 0, 0, 0, 0, 0, 0, 510, 517, 483, 523, 536, 485, 554, 545, 0, 0, 0, 0, 0, 0, 0, 0, 537, 518, 542, 479, 491, 478, 507, 535, 0, 0, 0, 0, 0, 0, 0, 0, 528, 566, 477, 488, 542, 516, 483, 507, 11, 10, 8, 0, 0, 0, 0, 0, 511, 528, 487, 526, 533, 516, 499, 530, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4095, 3985, 4073, 4153, 4087, 4136, 4130] avg=4096.0000 max=4153 max_rank=4 min=3985 min_rank=2 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16506 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[502, 554, 506, 531, 476, 478, 520, 534, 0, 0, 0, 0, 0, 0, 0, 0, 515, 517, 516, 522, 513, 498, 522, 504, 0, 0, 0, 0, 0, 0, 0, 0, 498, 459, 500, 507, 529, 525, 486, 494, 3, 0, 0, 0, 0, 0, 0, 0, 508, 499, 534, 503, 520, 532, 491, 494, 12, 5, 0, 0, 0, 0, 0, 0, 511, 487, 516, 490, 538, 537, 530, 529, 0, 0, 0, 0, 0, 0, 0, 0, 505, 510, 533, 504, 543, 490, 484, 545, 0, 0, 0, 0, 0, 0, 0, 0, 497, 510, 516, 491, 554, 489, 505, 511, 7, 4, 1, 0, 0, 0, 0, 0, 506, 549, 517, 519, 535, 518, 484, 496, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4107, 4001, 4098, 4138, 4114, 4085, 4124] avg=4096.0000 max=4138 max_rank=4 min=4001 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16461 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[511, 503, 491, 538, 530, 514, 491, 524, 497, 505, 548, 508, 520, 537, 532, 511, 507, 502, 505, 507, 505, 499, 497, 484, 508, 497, 535, 533, 518, 526, 539, 512, 514, 518, 539, 520, 494, 518, 550, 489, 504, 520, 496, 512, 510, 513, 465, 526, 486, 499, 528, 511, 523, 486, 511, 498, 511, 518, 509, 500, 499, 497, 551, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[490, 536, 478, 544, 522, 525, 537, 511, 515, 520, 469, 513, 497, 486, 489, 512, 491, 522, 512, 510, 513, 558, 546, 519, 517, 520, 534, 521, 501, 511, 531, 523, 535, 526, 496, 473, 543, 543, 521, 502, 479, 496, 474, 510, 503, 496, 519, 516, 488, 514, 529, 545, 527, 461, 531, 527, 496, 449, 545, 499, 541, 532, 499, 480] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[510, 504, 503, 530, 514, 522, 513, 489, 561, 484, 482, 520, 484, 527, 532, 487, 492, 478, 477, 519, 525, 492, 512, 498, 508, 526, 480, 524, 502, 525, 505, 531, 505, 496, 503, 538, 492, 544, 517, 519, 520, 508, 530, 487, 502, 526, 549, 531, 511, 533, 475, 512, 499, 539, 529, 508, 493, 551, 502, 499, 585, 516, 468, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[484, 505, 480, 529, 473, 518, 519, 530, 519, 529, 507, 540, 531, 494, 503, 538, 496, 497, 534, 496, 511, 518, 580, 548, 486, 500, 511, 513, 504, 495, 490, 506, 498, 497, 522, 519, 534, 517, 537, 515, 543, 489, 533, 489, 485, 494, 520, 518, 502, 526, 495, 506, 504, 504, 512, 511, 495, 519, 529, 529, 522, 491, 532, 497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[502, 489, 491, 503, 501, 515, 494, 488, 524, 525, 535, 540, 509, 457, 507, 514, 476, 470, 505, 527, 527, 514, 503, 505, 526, 530, 502, 516, 549, 486, 497, 509, 511, 453, 526, 533, 547, 512, 514, 501, 528, 520, 502, 536, 488, 508, 504, 499, 527, 505, 505, 526, 558, 501, 538, 546, 517, 496, 518, 502, 522, 527, 528, 534] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[484, 511, 489, 515, 492, 545, 494, 539, 539, 494, 501, 515, 539, 500, 535, 488, 519, 522, 499, 561, 485, 524, 464, 491, 530, 514, 490, 481, 520, 537, 484, 506, 505, 514, 524, 519, 513, 497, 520, 521, 499, 533, 521, 552, 524, 505, 550, 483, 484, 522, 528, 550, 518, 484, 479, 511, 542, 519, 496, 460, 519, 540, 547, 482] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[480, 497, 479, 545, 526, 514, 508, 466, 557, 526, 503, 573, 548, 543, 514, 513, 539, 537, 518, 473, 500, 553, 529, 514, 535, 509, 488, 545, 526, 477, 474, 480, 504, 496, 492, 487, 453, 490, 505, 483, 528, 497, 500, 555, 543, 511, 487, 539, 537, 562, 493, 505, 485, 540, 513, 510, 508, 498, 523, 498, 517, 506, 523, 491] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[550, 492, 491, 536, 497, 508, 549, 517, 522, 504, 498, 501, 490, 492, 498, 516, 522, 514, 493, 546, 518, 500, 560, 517, 512, 543, 517, 498, 511, 486, 483, 474, 527, 540, 533, 514, 537, 523, 519, 520, 479, 512, 499, 494, 529, 492, 450, 486, 495, 487, 564, 496, 515, 520, 504, 507, 542, 492, 544, 509, 545, 529, 486, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[484, 505, 480, 529, 473, 518, 519, 530, 18, 5, 0, 0, 0, 0, 0, 0, 501, 529, 507, 540, 531, 494, 503, 538, 0, 0, 0, 0, 0, 0, 0, 0, 496, 497, 534, 496, 511, 518, 580, 548, 0, 0, 0, 0, 0, 0, 0, 0, 486, 500, 511, 513, 504, 495, 490, 506, 9, 2, 0, 0, 0, 0, 0, 0, 498, 497, 522, 519, 534, 517, 525, 515, 0, 0, 0, 0, 0, 0, 0, 0, 543, 489, 533, 489, 485, 494, 520, 518, 10, 8, 0, 0, 0, 0, 0, 0, 502, 526, 495, 497, 504, 504, 512, 511, 0, 0, 0, 0, 0, 0, 0, 0, 495, 519, 529, 529, 509, 491, 532, 497, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4143, 4180, 4016, 4127, 4089, 4051, 4101] avg=4096.0000 max=4180 max_rank=2 min=4016 min_rank=3 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[510, 504, 503, 530, 514, 522, 513, 489, 19, 6, 0, 0, 0, 0, 0, 0, 542, 484, 482, 520, 484, 527, 532, 487, 0, 0, 0, 0, 0, 0, 0, 0, 492, 478, 477, 519, 525, 492, 512, 498, 0, 0, 0, 0, 0, 0, 0, 0, 508, 526, 480, 524, 502, 525, 505, 531, 9, 2, 0, 0, 0, 0, 0, 0, 505, 496, 503, 538, 492, 544, 505, 519, 0, 0, 0, 0, 0, 0, 0, 0, 520, 508, 530, 487, 502, 526, 549, 531, 10, 9, 0, 0, 0, 0, 0, 0, 511, 533, 475, 503, 499, 539, 529, 508, 0, 0, 0, 0, 0, 0, 0, 0, 493, 551, 502, 499, 570, 516, 468, 525, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4058, 3993, 4112, 4102, 4172, 4097, 4124] avg=4096.0000 max=4172 max_rank=5 min=3993 min_rank=2 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16495 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[490, 536, 478, 544, 522, 525, 537, 511, 18, 5, 0, 0, 0, 0, 0, 0, 497, 520, 469, 513, 497, 486, 489, 512, 0, 0, 0, 0, 0, 0, 0, 0, 491, 522, 512, 510, 513, 558, 546, 519, 0, 0, 0, 0, 0, 0, 0, 0, 517, 520, 534, 521, 501, 511, 531, 523, 10, 2, 0, 0, 0, 0, 0, 0, 535, 526, 496, 473, 543, 543, 509, 502, 0, 0, 0, 0, 0, 0, 0, 0, 479, 496, 474, 510, 503, 496, 519, 516, 10, 8, 0, 0, 0, 0, 0, 0, 488, 514, 529, 535, 527, 461, 531, 527, 0, 0, 0, 0, 0, 0, 0, 0, 496, 449, 545, 499, 528, 532, 499, 480, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4166, 3983, 4171, 4170, 4127, 4011, 4112, 4028] avg=4096.0000 max=4171 max_rank=2 min=3983 min_rank=1 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16278 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[484, 511, 489, 515, 492, 545, 494, 539, 18, 5, 0, 0, 0, 0, 0, 0, 521, 494, 501, 515, 539, 500, 535, 488, 0, 0, 0, 0, 0, 0, 0, 0, 519, 522, 499, 561, 485, 524, 464, 491, 0, 0, 0, 0, 0, 0, 0, 0, 530, 514, 490, 481, 520, 537, 484, 506, 10, 2, 0, 0, 0, 0, 0, 0, 505, 514, 524, 519, 513, 497, 508, 521, 0, 0, 0, 0, 0, 0, 0, 0, 499, 533, 521, 552, 524, 505, 550, 483, 10, 8, 0, 0, 0, 0, 0, 0, 484, 522, 528, 540, 518, 484, 479, 511, 0, 0, 0, 0, 0, 0, 0, 0, 542, 519, 496, 460, 506, 540, 547, 482, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4093, 4065, 4074, 4101, 4185, 4066, 4092] avg=4096.0000 max=4185 max_rank=5 min=4065 min_rank=2 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16444 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[511, 503, 491, 538, 530, 514, 491, 524, 17, 5, 0, 0, 0, 0, 0, 0, 480, 505, 548, 508, 520, 537, 532, 511, 0, 0, 0, 0, 0, 0, 0, 0, 507, 502, 505, 507, 505, 499, 497, 484, 0, 0, 0, 0, 0, 0, 0, 0, 508, 497, 535, 533, 518, 526, 539, 512, 9, 2, 0, 0, 0, 0, 0, 0, 514, 518, 539, 520, 494, 518, 538, 489, 0, 0, 0, 0, 0, 0, 0, 0, 504, 520, 496, 512, 510, 513, 465, 526, 10, 8, 0, 0, 0, 0, 0, 0, 486, 499, 528, 502, 523, 486, 511, 498, 0, 0, 0, 0, 0, 0, 0, 0, 511, 518, 509, 500, 486, 497, 551, 519, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4124, 4141, 4006, 4179, 4130, 4064, 4033, 4091] avg=4096.0000 max=4179 max_rank=3 min=4006 min_rank=2 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16318 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[502, 489, 491, 503, 501, 515, 494, 488, 22, 8, 0, 0, 0, 0, 0, 0, 502, 525, 535, 540, 509, 457, 507, 514, 0, 0, 0, 0, 0, 0, 0, 0, 476, 470, 505, 527, 527, 514, 496, 505, 0, 0, 0, 0, 0, 0, 0, 0, 526, 530, 502, 516, 549, 486, 497, 509, 12, 3, 7, 0, 0, 0, 0, 0, 511, 453, 526, 533, 547, 512, 498, 501, 0, 0, 0, 0, 0, 0, 0, 0, 528, 520, 502, 536, 488, 508, 504, 499, 13, 13, 0, 0, 0, 0, 0, 0, 527, 505, 505, 514, 558, 501, 538, 546, 0, 0, 0, 0, 0, 0, 0, 0, 517, 496, 518, 502, 501, 527, 528, 534, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4013, 4089, 4020, 4137, 4081, 4111, 4194, 4123] avg=4096.0000 max=4194 max_rank=6 min=4013 min_rank=0 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16509 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[480, 497, 479, 545, 526, 514, 508, 466, 19, 5, 0, 0, 0, 0, 0, 0, 538, 526, 503, 573, 548, 543, 514, 513, 0, 0, 0, 0, 0, 0, 0, 0, 539, 537, 518, 473, 500, 553, 529, 514, 0, 0, 0, 0, 0, 0, 0, 0, 535, 509, 488, 545, 526, 477, 474, 480, 9, 2, 0, 0, 0, 0, 0, 0, 504, 496, 492, 487, 453, 490, 494, 483, 0, 0, 0, 0, 0, 0, 0, 0, 528, 497, 500, 555, 543, 511, 487, 539, 9, 8, 0, 0, 0, 0, 0, 0, 537, 562, 493, 496, 485, 540, 513, 510, 0, 0, 0, 0, 0, 0, 0, 0, 508, 498, 523, 498, 504, 506, 523, 491, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4039, 4258, 4163, 4045, 3899, 4177, 4136, 4051] avg=4096.0000 max=4258 max_rank=1 min=3899 min_rank=4 imbalance_ratio=1.039551 max_load_reduction=28510 max_load_reduction_pct=0.870056 moved_to_echo=16263 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[550, 492, 491, 536, 497, 508, 549, 517, 18, 5, 0, 0, 0, 0, 0, 0, 504, 504, 498, 501, 490, 492, 498, 516, 0, 0, 0, 0, 0, 0, 0, 0, 522, 514, 493, 546, 518, 500, 560, 517, 0, 0, 0, 0, 0, 0, 0, 0, 512, 543, 517, 498, 511, 486, 483, 474, 9, 2, 0, 0, 0, 0, 0, 0, 527, 540, 533, 514, 537, 523, 507, 520, 0, 0, 0, 0, 0, 0, 0, 0, 479, 512, 499, 494, 529, 492, 450, 486, 10, 8, 0, 0, 0, 0, 0, 0, 495, 487, 564, 487, 515, 520, 504, 507, 0, 0, 0, 0, 0, 0, 0, 0, 542, 492, 544, 509, 532, 529, 486, 524, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4163, 4003, 4170, 4035, 4201, 3959, 4079, 4158] avg=4096.0000 max=4201 max_rank=4 min=3959 min_rank=5 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 3968, 4352, 4096, 4224, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[474, 528, 488, 480, 517, 534, 527, 496, 481, 546, 509, 500, 547, 513, 486, 467, 514, 536, 515, 509, 507, 499, 538, 510, 476, 509, 519, 529, 500, 502, 493, 505, 519, 509, 505, 529, 517, 503, 486, 486, 476, 495, 475, 518, 540, 547, 526, 545, 549, 532, 497, 527, 575, 496, 501, 498, 505, 510, 520, 542, 516, 500, 569, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[507, 515, 492, 515, 520, 485, 482, 509, 525, 514, 546, 528, 486, 483, 519, 503, 517, 493, 501, 484, 522, 535, 525, 520, 517, 510, 484, 519, 505, 538, 536, 499, 534, 529, 505, 466, 499, 541, 547, 544, 513, 493, 531, 521, 494, 505, 530, 544, 533, 480, 488, 513, 550, 475, 493, 506, 510, 524, 484, 513, 499, 510, 520, 540] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[533, 491, 498, 469, 496, 505, 508, 515, 562, 500, 526, 512, 548, 532, 520, 478, 495, 534, 500, 522, 498, 513, 518, 532, 493, 500, 533, 512, 518, 550, 494, 531, 510, 501, 524, 510, 495, 530, 495, 507, 503, 523, 548, 508, 532, 504, 505, 537, 528, 501, 507, 486, 526, 503, 501, 480, 511, 500, 461, 513, 519, 540, 514, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[477, 505, 547, 479, 526, 502, 470, 530, 557, 503, 520, 480, 525, 474, 564, 513, 527, 529, 501, 530, 509, 504, 512, 499, 490, 516, 508, 503, 510, 529, 498, 522, 523, 526, 490, 537, 546, 510, 533, 474, 528, 490, 516, 498, 510, 513, 505, 517, 501, 526, 517, 499, 489, 479, 525, 532, 533, 523, 488, 492, 549, 536, 508, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[511, 519, 542, 507, 514, 494, 495, 520, 505, 500, 514, 495, 501, 519, 518, 518, 487, 506, 519, 468, 500, 517, 466, 511, 531, 536, 485, 512, 516, 513, 498, 541, 546, 515, 508, 522, 499, 544, 514, 491, 480, 539, 518, 488, 495, 523, 499, 535, 505, 499, 526, 500, 512, 528, 555, 532, 518, 530, 534, 504, 508, 520, 490, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[507, 531, 488, 498, 492, 474, 497, 517, 528, 534, 488, 488, 502, 532, 522, 517, 492, 489, 505, 518, 478, 497, 514, 488, 524, 546, 481, 529, 560, 528, 514, 511, 529, 503, 514, 525, 515, 517, 543, 507, 485, 502, 500, 519, 512, 494, 493, 522, 532, 499, 546, 524, 494, 510, 512, 529, 533, 493, 511, 510, 532, 517, 520, 537] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[494, 501, 488, 493, 531, 506, 521, 539, 521, 475, 533, 487, 479, 516, 476, 496, 530, 527, 494, 502, 503, 488, 501, 504, 542, 504, 531, 496, 533, 480, 537, 516, 517, 549, 508, 491, 526, 496, 515, 501, 546, 541, 507, 546, 506, 509, 534, 494, 498, 483, 510, 499, 505, 511, 551, 522, 513, 494, 530, 528, 515, 545, 528, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[492, 554, 532, 518, 531, 508, 457, 501, 500, 510, 517, 518, 489, 536, 538, 513, 506, 491, 537, 513, 534, 487, 503, 517, 526, 499, 499, 493, 482, 503, 514, 490, 492, 497, 527, 529, 498, 488, 558, 497, 554, 521, 512, 462, 515, 491, 517, 545, 528, 540, 555, 546, 517, 525, 490, 487, 486, 497, 517, 524, 534, 511, 513, 487] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[474, 528, 488, 480, 517, 534, 527, 496, 22, 16, 6, 0, 0, 0, 0, 0, 481, 546, 509, 500, 547, 513, 486, 467, 1, 0, 0, 0, 0, 0, 0, 0, 514, 536, 515, 509, 507, 499, 538, 510, 15, 10, 4, 0, 0, 0, 0, 0, 476, 509, 519, 529, 500, 497, 493, 505, 0, 0, 0, 0, 0, 0, 0, 0, 519, 509, 505, 529, 517, 503, 470, 486, 0, 0, 0, 0, 0, 0, 0, 0, 476, 495, 475, 518, 540, 547, 526, 529, 0, 0, 0, 0, 0, 0, 0, 0, 534, 532, 497, 527, 575, 496, 501, 498, 0, 0, 0, 0, 0, 0, 0, 0, 505, 510, 520, 542, 516, 478, 569, 501, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4050, 4157, 4028, 4038, 4106, 4160, 4141] avg=4096.0000 max=4160 max_rank=6 min=4028 min_rank=3 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16445 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[507, 515, 492, 515, 520, 485, 482, 509, 22, 18, 6, 0, 0, 0, 0, 0, 525, 514, 546, 528, 486, 483, 519, 503, 2, 0, 0, 0, 0, 0, 0, 0, 517, 493, 501, 484, 522, 535, 525, 520, 14, 10, 4, 0, 0, 0, 0, 0, 517, 510, 484, 519, 505, 532, 536, 499, 0, 0, 0, 0, 0, 0, 0, 0, 534, 529, 505, 466, 499, 541, 529, 544, 0, 0, 0, 0, 0, 0, 0, 0, 513, 493, 531, 521, 494, 505, 530, 528, 0, 0, 0, 0, 0, 0, 0, 0, 519, 480, 488, 513, 550, 475, 493, 506, 0, 0, 0, 0, 0, 0, 0, 0, 510, 524, 484, 513, 499, 488, 520, 540, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4071, 4106, 4125, 4102, 4147, 4115, 4024, 4078] avg=4096.0000 max=4147 max_rank=4 min=4024 min_rank=6 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16364 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[533, 491, 498, 469, 496, 505, 508, 515, 24, 16, 5, 0, 0, 0, 0, 0, 562, 500, 526, 512, 548, 532, 520, 478, 2, 0, 0, 0, 0, 0, 0, 0, 495, 534, 500, 522, 498, 513, 518, 532, 14, 10, 4, 0, 0, 0, 0, 0, 493, 500, 533, 512, 518, 544, 494, 531, 0, 0, 0, 0, 0, 0, 0, 0, 510, 501, 524, 510, 495, 530, 479, 507, 0, 0, 0, 0, 0, 0, 0, 0, 503, 523, 548, 508, 532, 504, 505, 522, 0, 0, 0, 0, 0, 0, 0, 0, 514, 501, 507, 486, 526, 503, 501, 480, 0, 0, 0, 0, 0, 0, 0, 0, 511, 500, 461, 513, 519, 516, 514, 510, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4180, 4140, 4125, 4056, 4145, 4018, 4044] avg=4096.0000 max=4180 max_rank=1 min=4018 min_rank=6 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16263 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[511, 519, 542, 507, 514, 494, 495, 520, 26, 21, 9, 0, 0, 0, 0, 0, 505, 500, 514, 495, 501, 519, 518, 518, 5, 0, 0, 0, 0, 0, 0, 0, 487, 506, 519, 468, 500, 517, 466, 511, 18, 12, 7, 0, 0, 0, 0, 0, 531, 536, 485, 512, 516, 501, 498, 541, 0, 0, 0, 0, 0, 0, 0, 0, 546, 515, 508, 522, 499, 544, 493, 491, 0, 0, 0, 0, 0, 0, 0, 0, 480, 539, 518, 488, 495, 523, 499, 514, 0, 0, 0, 0, 0, 0, 0, 0, 487, 499, 526, 500, 512, 528, 555, 532, 0, 0, 0, 0, 0, 0, 0, 0, 518, 530, 534, 504, 508, 494, 490, 513, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4158, 4075, 4011, 4120, 4118, 4056, 4139, 4091] avg=4096.0000 max=4158 max_rank=0 min=4011 min_rank=2 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16404 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[507, 531, 488, 498, 492, 474, 497, 517, 23, 18, 5, 0, 0, 0, 0, 0, 528, 534, 488, 488, 502, 532, 522, 517, 2, 0, 0, 0, 0, 0, 0, 0, 492, 489, 505, 518, 478, 497, 514, 488, 14, 9, 4, 0, 0, 0, 0, 0, 524, 546, 481, 529, 560, 522, 514, 511, 0, 0, 0, 0, 0, 0, 0, 0, 529, 503, 514, 525, 515, 517, 525, 507, 0, 0, 0, 0, 0, 0, 0, 0, 485, 502, 500, 519, 512, 494, 493, 508, 0, 0, 0, 0, 0, 0, 0, 0, 518, 499, 546, 524, 494, 510, 512, 529, 0, 0, 0, 0, 0, 0, 0, 0, 533, 493, 511, 510, 532, 494, 520, 537, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4050, 4113, 4008, 4187, 4135, 4013, 4132, 4130] avg=4096.0000 max=4187 max_rank=3 min=4008 min_rank=2 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16410 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[477, 505, 547, 479, 526, 502, 470, 530, 23, 18, 5, 0, 0, 0, 0, 0, 557, 503, 520, 480, 525, 474, 564, 513, 2, 0, 0, 0, 0, 0, 0, 0, 527, 529, 501, 530, 509, 504, 512, 499, 13, 9, 4, 0, 0, 0, 0, 0, 490, 516, 508, 503, 510, 523, 498, 522, 0, 0, 0, 0, 0, 0, 0, 0, 523, 526, 490, 537, 546, 510, 515, 474, 0, 0, 0, 0, 0, 0, 0, 0, 528, 490, 516, 498, 510, 513, 505, 503, 0, 0, 0, 0, 0, 0, 0, 0, 488, 526, 517, 499, 489, 479, 525, 532, 0, 0, 0, 0, 0, 0, 0, 0, 533, 523, 488, 492, 549, 513, 508, 496, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4138, 4137, 4070, 4121, 4063, 4055, 4102] avg=4096.0000 max=4138 max_rank=1 min=4055 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16341 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[494, 501, 488, 493, 531, 506, 521, 539, 24, 17, 5, 0, 0, 0, 0, 0, 521, 475, 533, 487, 479, 516, 476, 496, 1, 0, 0, 0, 0, 0, 0, 0, 530, 527, 494, 502, 503, 488, 501, 504, 13, 9, 3, 0, 0, 0, 0, 0, 542, 504, 531, 496, 533, 476, 537, 516, 0, 0, 0, 0, 0, 0, 0, 0, 517, 549, 508, 491, 526, 496, 498, 501, 0, 0, 0, 0, 0, 0, 0, 0, 546, 541, 507, 546, 506, 509, 534, 480, 0, 0, 0, 0, 0, 0, 0, 0, 485, 483, 510, 499, 505, 511, 551, 522, 0, 0, 0, 0, 0, 0, 0, 0, 513, 494, 530, 528, 515, 521, 528, 506, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 3984, 4074, 4135, 4086, 4169, 4066, 4135] avg=4096.0000 max=4169 max_rank=5 min=3984 min_rank=1 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16456 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[492, 554, 532, 518, 531, 508, 457, 501, 22, 19, 6, 0, 0, 0, 0, 0, 500, 510, 517, 518, 489, 536, 538, 513, 1, 0, 0, 0, 0, 0, 0, 0, 506, 491, 537, 513, 534, 487, 503, 517, 14, 10, 4, 0, 0, 0, 0, 0, 526, 499, 499, 493, 482, 498, 514, 490, 0, 0, 0, 0, 0, 0, 0, 0, 492, 497, 527, 529, 498, 488, 539, 497, 0, 0, 0, 0, 0, 0, 0, 0, 554, 521, 512, 462, 515, 491, 517, 529, 0, 0, 0, 0, 0, 0, 0, 0, 514, 540, 555, 546, 517, 525, 490, 487, 0, 0, 0, 0, 0, 0, 0, 0, 486, 497, 517, 524, 534, 489, 513, 487, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4140, 4122, 4116, 4001, 4067, 4101, 4174, 4047] avg=4096.0000 max=4174 max_rank=6 min=4001 min_rank=3 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 3968, 4224, 4096, 3968, 4224, 256, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[510, 534, 529, 513, 552, 531, 497, 497, 501, 537, 481, 540, 479, 514, 521, 496, 531, 540, 499, 537, 481, 532, 500, 514, 501, 515, 502, 510, 461, 549, 526, 491, 542, 532, 521, 493, 536, 507, 481, 470, 486, 512, 505, 496, 481, 560, 534, 501, 473, 511, 519, 497, 491, 530, 523, 502, 518, 510, 515, 532, 549, 511, 503, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[503, 543, 510, 505, 521, 469, 515, 502, 498, 529, 533, 502, 524, 520, 546, 507, 531, 546, 525, 484, 530, 525, 522, 513, 494, 483, 542, 518, 538, 488, 487, 522, 522, 496, 502, 475, 509, 493, 535, 459, 494, 535, 493, 503, 515, 519, 516, 525, 496, 523, 520, 541, 511, 470, 525, 498, 520, 511, 508, 544, 492, 555, 472, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[576, 536, 481, 460, 511, 506, 489, 526, 513, 552, 499, 474, 485, 514, 535, 500, 533, 500, 479, 537, 486, 480, 517, 514, 528, 526, 510, 513, 490, 542, 507, 491, 532, 493, 527, 508, 558, 485, 507, 505, 508, 531, 506, 509, 501, 537, 521, 530, 532, 519, 515, 537, 507, 526, 511, 519, 498, 478, 517, 488, 516, 516, 529, 492] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[520, 516, 525, 507, 501, 550, 547, 504, 514, 507, 509, 529, 516, 513, 515, 532, 483, 498, 521, 487, 535, 499, 545, 508, 525, 553, 527, 515, 531, 494, 453, 515, 500, 498, 496, 527, 530, 498, 544, 509, 542, 530, 486, 446, 521, 491, 512, 508, 522, 470, 526, 484, 501, 534, 536, 542, 482, 501, 551, 493, 487, 516, 494, 497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[498, 531, 483, 507, 502, 519, 525, 486, 507, 523, 508, 490, 499, 533, 523, 493, 528, 534, 536, 483, 544, 476, 546, 536, 502, 500, 540, 517, 519, 511, 508, 495, 510, 538, 505, 494, 513, 506, 512, 531, 534, 546, 485, 478, 528, 524, 469, 499, 538, 477, 542, 546, 550, 510, 486, 513, 504, 520, 531, 556, 476, 481, 467, 497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[509, 502, 526, 494, 542, 503, 485, 492, 521, 522, 505, 483, 487, 493, 523, 499, 516, 519, 524, 510, 518, 498, 535, 508, 535, 510, 500, 523, 527, 498, 548, 516, 508, 486, 502, 494, 517, 492, 534, 506, 510, 552, 501, 530, 517, 510, 525, 488, 539, 501, 508, 488, 557, 496, 503, 495, 561, 479, 516, 521, 497, 526, 517, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[505, 515, 535, 534, 517, 532, 523, 473, 505, 494, 505, 536, 520, 513, 510, 518, 526, 543, 491, 520, 536, 510, 539, 513, 471, 527, 492, 491, 499, 540, 510, 509, 536, 525, 521, 460, 502, 519, 492, 514, 514, 496, 509, 514, 510, 478, 496, 519, 531, 530, 530, 487, 514, 511, 512, 506, 504, 510, 493, 491, 529, 503, 516, 544] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[516, 535, 495, 497, 541, 491, 503, 545, 500, 529, 532, 519, 508, 524, 506, 513, 454, 523, 516, 539, 527, 509, 484, 539, 539, 513, 485, 487, 523, 508, 505, 514, 502, 513, 505, 521, 540, 528, 540, 496, 485, 544, 507, 518, 497, 550, 493, 555, 508, 477, 513, 496, 511, 534, 487, 509, 508, 495, 522, 500, 484, 490, 538, 483] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[510, 525, 529, 513, 552, 531, 497, 497, 0, 0, 0, 0, 0, 0, 0, 0, 501, 536, 481, 540, 479, 514, 521, 496, 0, 0, 0, 0, 0, 0, 0, 0, 531, 510, 499, 537, 481, 532, 500, 514, 0, 0, 0, 0, 0, 0, 0, 0, 501, 515, 502, 510, 461, 549, 526, 491, 4, 2, 0, 0, 0, 0, 0, 0, 542, 532, 521, 493, 536, 507, 481, 470, 9, 5, 0, 0, 0, 0, 0, 0, 486, 512, 505, 496, 481, 560, 534, 501, 3, 1, 0, 0, 0, 0, 0, 0, 473, 511, 514, 497, 491, 530, 523, 502, 0, 0, 0, 0, 0, 0, 0, 0, 518, 510, 515, 532, 549, 511, 503, 506, 21, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4154, 4068, 4104, 4061, 4096, 4079, 4041, 4165] avg=4096.0000 max=4165 max_rank=7 min=4041 min_rank=6 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[503, 534, 510, 505, 521, 469, 515, 502, 0, 0, 0, 0, 0, 0, 0, 0, 498, 528, 533, 502, 524, 520, 546, 507, 0, 0, 0, 0, 0, 0, 0, 0, 531, 515, 525, 484, 530, 525, 522, 513, 0, 0, 0, 0, 0, 0, 0, 0, 494, 483, 542, 518, 538, 488, 487, 522, 4, 2, 0, 0, 0, 0, 0, 0, 522, 496, 502, 475, 509, 493, 535, 459, 9, 5, 0, 0, 0, 0, 0, 0, 494, 535, 493, 503, 515, 519, 516, 525, 3, 1, 0, 0, 0, 0, 0, 0, 496, 523, 515, 541, 511, 470, 525, 498, 0, 0, 0, 0, 0, 0, 0, 0, 520, 511, 508, 544, 492, 555, 472, 516, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4059, 4158, 4145, 4078, 4005, 4104, 4079, 4140] avg=4096.0000 max=4158 max_rank=1 min=4005 min_rank=4 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16328 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[498, 522, 483, 507, 502, 519, 525, 486, 0, 0, 0, 0, 0, 0, 0, 0, 507, 523, 508, 490, 499, 533, 523, 493, 0, 0, 0, 0, 0, 0, 0, 0, 528, 504, 536, 483, 544, 476, 546, 536, 0, 0, 0, 0, 0, 0, 0, 0, 502, 500, 540, 517, 519, 511, 508, 495, 4, 2, 0, 0, 0, 0, 0, 0, 510, 538, 505, 494, 513, 506, 512, 531, 9, 5, 0, 0, 0, 0, 0, 0, 534, 546, 485, 478, 528, 524, 469, 499, 3, 0, 0, 0, 0, 0, 0, 0, 538, 477, 537, 546, 550, 510, 486, 513, 0, 0, 0, 0, 0, 0, 0, 0, 504, 520, 531, 556, 476, 481, 467, 497, 21, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4042, 4076, 4153, 4098, 4123, 4066, 4157, 4053] avg=4096.0000 max=4157 max_rank=6 min=4042 min_rank=0 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[576, 520, 481, 460, 511, 506, 489, 526, 0, 0, 0, 0, 0, 0, 0, 0, 513, 547, 499, 474, 485, 514, 535, 500, 0, 0, 0, 0, 0, 0, 0, 0, 533, 466, 479, 537, 486, 480, 517, 514, 0, 0, 0, 0, 0, 0, 0, 0, 528, 526, 510, 513, 490, 542, 507, 491, 8, 9, 0, 0, 0, 0, 0, 0, 532, 493, 527, 508, 558, 485, 507, 505, 11, 8, 0, 0, 0, 0, 0, 0, 508, 531, 506, 509, 501, 537, 521, 530, 4, 5, 0, 0, 0, 0, 0, 0, 532, 519, 502, 537, 507, 526, 511, 519, 0, 0, 0, 0, 0, 0, 0, 0, 498, 478, 517, 488, 516, 516, 529, 492, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4067, 4012, 4124, 4134, 4152, 4153, 4057] avg=4096.0000 max=4153 max_rank=6 min=4012 min_rank=2 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16496 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[520, 507, 525, 507, 501, 550, 547, 504, 0, 0, 0, 0, 0, 0, 0, 0, 514, 507, 509, 529, 516, 513, 515, 532, 0, 0, 0, 0, 0, 0, 0, 0, 483, 470, 521, 487, 535, 499, 545, 508, 0, 0, 0, 0, 0, 0, 0, 0, 525, 553, 527, 515, 531, 494, 453, 515, 4, 2, 0, 0, 0, 0, 0, 0, 500, 498, 496, 527, 530, 498, 544, 509, 8, 5, 0, 0, 0, 0, 0, 0, 542, 530, 486, 446, 521, 491, 512, 508, 3, 0, 0, 0, 0, 0, 0, 0, 522, 470, 521, 484, 501, 534, 536, 542, 0, 0, 0, 0, 0, 0, 0, 0, 482, 501, 551, 493, 487, 516, 494, 497, 20, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4161, 4135, 4048, 4119, 4115, 4039, 4110, 4041] avg=4096.0000 max=4161 max_rank=0 min=4039 min_rank=5 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16305 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[505, 506, 535, 534, 517, 532, 523, 473, 0, 0, 0, 0, 0, 0, 0, 0, 505, 494, 505, 536, 520, 513, 510, 518, 0, 0, 0, 0, 0, 0, 0, 0, 526, 512, 491, 520, 536, 510, 539, 513, 0, 0, 0, 0, 0, 0, 0, 0, 471, 527, 492, 491, 499, 540, 510, 509, 4, 2, 0, 0, 0, 0, 0, 0, 536, 525, 521, 460, 502, 519, 492, 514, 9, 5, 0, 0, 0, 0, 0, 0, 514, 496, 509, 514, 510, 478, 496, 519, 3, 0, 0, 0, 0, 0, 0, 0, 531, 530, 525, 487, 514, 511, 512, 506, 0, 0, 0, 0, 0, 0, 0, 0, 504, 510, 493, 491, 529, 503, 516, 544, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4101, 4147, 4045, 4083, 4039, 4116, 4112] avg=4096.0000 max=4147 max_rank=2 min=4039 min_rank=5 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16350 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[509, 493, 526, 494, 542, 503, 485, 492, 0, 0, 0, 0, 0, 0, 0, 0, 521, 522, 505, 483, 487, 493, 523, 499, 0, 0, 0, 0, 0, 0, 0, 0, 516, 489, 524, 510, 518, 498, 535, 508, 0, 0, 0, 0, 0, 0, 0, 0, 535, 510, 500, 523, 527, 498, 548, 516, 4, 2, 0, 0, 0, 0, 0, 0, 508, 486, 502, 494, 517, 492, 534, 506, 9, 5, 0, 0, 0, 0, 0, 0, 510, 552, 501, 530, 517, 510, 525, 488, 3, 0, 0, 0, 0, 0, 0, 0, 539, 501, 503, 488, 557, 496, 503, 495, 0, 0, 0, 0, 0, 0, 0, 0, 561, 479, 516, 521, 497, 526, 517, 521, 21, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4044, 4033, 4098, 4163, 4053, 4136, 4082, 4159] avg=4096.0000 max=4163 max_rank=3 min=4033 min_rank=1 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16430 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[516, 526, 495, 497, 541, 491, 503, 545, 0, 0, 0, 0, 0, 0, 0, 0, 500, 528, 532, 519, 508, 524, 506, 513, 0, 0, 0, 0, 0, 0, 0, 0, 454, 493, 516, 539, 527, 509, 484, 539, 0, 0, 0, 0, 0, 0, 0, 0, 539, 513, 485, 487, 523, 508, 505, 514, 4, 2, 0, 0, 0, 0, 0, 0, 502, 513, 505, 521, 540, 528, 540, 496, 9, 5, 0, 0, 0, 0, 0, 0, 485, 544, 507, 518, 497, 550, 493, 555, 3, 1, 0, 0, 0, 0, 0, 0, 508, 477, 508, 496, 511, 534, 487, 509, 0, 0, 0, 0, 0, 0, 0, 0, 508, 495, 522, 500, 484, 490, 538, 483, 21, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4130, 4061, 4080, 4159, 4153, 4030, 4041] avg=4096.0000 max=4159 max_rank=4 min=4030 min_rank=6 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4352, 4096, 4096, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 3968, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[489, 490, 507, 501, 494, 519, 479, 514, 565, 482, 527, 498, 508, 466, 551, 565, 471, 535, 507, 506, 538, 516, 494, 530, 524, 484, 525, 503, 493, 514, 532, 529, 506, 514, 525, 499, 496, 531, 516, 532, 505, 500, 538, 549, 501, 523, 495, 515, 494, 503, 499, 479, 477, 496, 508, 539, 505, 515, 530, 510, 507, 549, 513, 543] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[489, 528, 507, 505, 522, 506, 554, 495, 529, 551, 509, 493, 524, 515, 508, 462, 542, 486, 528, 518, 495, 491, 481, 499, 560, 521, 511, 498, 511, 514, 509, 507, 506, 519, 522, 546, 473, 480, 522, 495, 508, 509, 481, 521, 533, 503, 556, 527, 513, 526, 544, 473, 524, 530, 524, 497, 512, 490, 516, 535, 500, 520, 518, 477] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[520, 526, 533, 487, 495, 491, 514, 524, 483, 519, 536, 496, 511, 517, 481, 529, 533, 488, 506, 497, 497, 544, 505, 506, 505, 516, 529, 487, 514, 518, 529, 536, 524, 477, 546, 525, 530, 468, 484, 492, 473, 504, 493, 499, 511, 502, 506, 516, 530, 515, 517, 538, 495, 538, 542, 474, 514, 542, 517, 528, 514, 518, 543, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[527, 501, 526, 489, 483, 518, 509, 571, 515, 513, 520, 480, 533, 522, 506, 487, 506, 530, 527, 529, 523, 494, 514, 510, 513, 485, 507, 517, 510, 509, 520, 467, 536, 526, 514, 502, 517, 499, 515, 490, 530, 500, 497, 521, 522, 530, 516, 500, 508, 502, 489, 499, 564, 508, 506, 527, 521, 509, 535, 502, 471, 492, 559, 500] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[540, 546, 496, 522, 463, 542, 549, 481, 512, 500, 495, 560, 532, 513, 548, 491, 515, 531, 499, 474, 525, 527, 496, 511, 546, 477, 513, 507, 524, 482, 542, 500, 528, 517, 504, 488, 573, 505, 504, 509, 489, 503, 487, 527, 509, 516, 511, 501, 523, 553, 466, 480, 523, 501, 555, 494, 505, 501, 529, 491, 506, 509, 513, 489] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[496, 514, 512, 534, 490, 518, 516, 499, 478, 503, 475, 524, 510, 536, 503, 522, 510, 491, 481, 500, 479, 536, 527, 503, 511, 532, 489, 511, 525, 520, 536, 530, 522, 513, 497, 513, 492, 525, 467, 516, 524, 552, 481, 525, 523, 508, 491, 507, 504, 512, 490, 492, 575, 536, 555, 499, 522, 556, 516, 552, 484, 507, 496, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[488, 492, 529, 527, 550, 495, 470, 495, 502, 529, 490, 522, 488, 527, 505, 488, 475, 497, 490, 504, 501, 518, 520, 493, 531, 514, 543, 511, 510, 497, 492, 512, 528, 497, 506, 584, 505, 542, 530, 517, 540, 500, 492, 494, 537, 507, 538, 518, 511, 534, 545, 515, 532, 529, 498, 485, 521, 516, 509, 482, 474, 510, 515, 552] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[513, 527, 487, 506, 482, 503, 513, 501, 531, 509, 488, 508, 543, 510, 512, 511, 500, 533, 497, 499, 527, 501, 541, 511, 529, 516, 522, 532, 531, 503, 518, 521, 489, 495, 515, 517, 474, 505, 553, 500, 499, 487, 480, 504, 512, 493, 526, 533, 531, 503, 526, 533, 513, 516, 493, 529, 515, 517, 495, 501, 544, 510, 507, 528] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[489, 490, 507, 501, 494, 519, 479, 514, 13, 6, 0, 0, 0, 0, 0, 0, 565, 482, 527, 498, 508, 466, 551, 565, 0, 0, 0, 0, 0, 0, 0, 0, 471, 535, 507, 506, 538, 516, 494, 530, 18, 6, 0, 0, 0, 0, 0, 0, 505, 484, 525, 503, 493, 514, 532, 529, 0, 0, 0, 0, 0, 0, 0, 0, 506, 514, 525, 499, 496, 531, 516, 532, 1, 0, 0, 0, 0, 0, 0, 0, 505, 500, 538, 549, 501, 523, 495, 515, 8, 0, 0, 0, 0, 0, 0, 0, 494, 503, 499, 479, 459, 496, 508, 539, 0, 0, 0, 0, 0, 0, 0, 0, 505, 515, 530, 510, 507, 549, 498, 543, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4012, 4162, 4121, 4085, 4120, 4134, 3977, 4157] avg=4096.0000 max=4162 max_rank=1 min=3977 min_rank=6 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[489, 528, 507, 505, 522, 506, 554, 495, 14, 6, 0, 0, 0, 0, 0, 0, 529, 551, 509, 493, 524, 515, 508, 462, 0, 0, 0, 0, 0, 0, 0, 0, 542, 486, 528, 518, 495, 491, 481, 499, 20, 6, 0, 0, 0, 0, 0, 0, 540, 521, 511, 498, 511, 514, 509, 507, 0, 0, 0, 0, 0, 0, 0, 0, 506, 519, 522, 546, 473, 480, 522, 495, 1, 0, 0, 0, 0, 0, 0, 0, 508, 509, 481, 521, 533, 503, 556, 527, 8, 0, 0, 0, 0, 0, 0, 0, 513, 526, 544, 473, 504, 530, 524, 497, 0, 0, 0, 0, 0, 0, 0, 0, 512, 490, 516, 535, 500, 520, 503, 477, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4091, 4066, 4111, 4064, 4146, 4111, 4053] avg=4096.0000 max=4146 max_rank=5 min=4053 min_rank=7 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16374 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[540, 546, 496, 522, 463, 542, 549, 481, 13, 6, 0, 0, 0, 0, 0, 0, 512, 500, 495, 560, 532, 513, 548, 491, 0, 0, 0, 0, 0, 0, 0, 0, 515, 531, 499, 474, 525, 527, 496, 511, 20, 6, 0, 0, 0, 0, 0, 0, 527, 477, 513, 507, 524, 482, 542, 500, 0, 0, 0, 0, 0, 0, 0, 0, 528, 517, 504, 488, 573, 505, 504, 509, 1, 0, 0, 0, 0, 0, 0, 0, 489, 503, 487, 527, 509, 516, 511, 501, 8, 0, 0, 0, 0, 0, 0, 0, 523, 553, 466, 480, 503, 501, 555, 494, 0, 0, 0, 0, 0, 0, 0, 0, 505, 501, 529, 491, 506, 509, 498, 489, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4158, 4151, 4104, 4072, 4129, 4051, 4075, 4028] avg=4096.0000 max=4158 max_rank=0 min=4028 min_rank=7 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16283 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[520, 526, 533, 487, 495, 491, 514, 524, 16, 10, 0, 0, 0, 0, 0, 0, 483, 519, 536, 496, 511, 517, 481, 529, 2, 0, 0, 0, 0, 0, 0, 0, 533, 488, 506, 497, 497, 544, 505, 506, 21, 9, 0, 0, 0, 0, 0, 0, 480, 516, 529, 487, 514, 518, 529, 536, 0, 0, 0, 0, 0, 0, 0, 0, 524, 477, 546, 525, 530, 468, 484, 492, 4, 0, 0, 0, 0, 0, 0, 0, 473, 504, 493, 499, 511, 502, 506, 516, 13, 0, 0, 0, 0, 0, 0, 0, 530, 515, 517, 538, 474, 538, 542, 474, 0, 0, 0, 0, 0, 0, 0, 0, 514, 542, 517, 528, 514, 518, 514, 521, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4074, 4106, 4109, 4050, 4017, 4128, 4168] avg=4096.0000 max=4168 max_rank=7 min=4017 min_rank=5 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[496, 514, 512, 534, 490, 518, 516, 499, 12, 6, 0, 0, 0, 0, 0, 0, 478, 503, 475, 524, 510, 536, 503, 522, 0, 0, 0, 0, 0, 0, 0, 0, 510, 491, 481, 500, 479, 536, 527, 503, 22, 5, 0, 0, 0, 0, 0, 0, 494, 532, 489, 511, 525, 520, 536, 530, 0, 0, 0, 0, 0, 0, 0, 0, 522, 513, 497, 513, 492, 525, 467, 516, 1, 0, 0, 0, 0, 0, 0, 0, 524, 552, 481, 525, 523, 508, 491, 507, 8, 0, 0, 0, 0, 0, 0, 0, 504, 512, 490, 492, 553, 536, 555, 499, 0, 0, 0, 0, 0, 0, 0, 0, 522, 556, 516, 552, 484, 507, 481, 505, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4051, 4054, 4137, 4046, 4119, 4141, 4123] avg=4096.0000 max=4141 max_rank=6 min=4046 min_rank=4 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[527, 501, 526, 489, 483, 518, 509, 571, 13, 6, 0, 0, 0, 0, 0, 0, 515, 513, 520, 480, 533, 522, 506, 487, 0, 0, 0, 0, 0, 0, 0, 0, 506, 530, 527, 529, 523, 494, 514, 510, 21, 5, 0, 0, 0, 0, 0, 0, 495, 485, 507, 517, 510, 509, 520, 467, 0, 0, 0, 0, 0, 0, 0, 0, 536, 526, 514, 502, 517, 499, 515, 490, 1, 0, 0, 0, 0, 0, 0, 0, 530, 500, 497, 521, 522, 530, 516, 500, 9, 0, 0, 0, 0, 0, 0, 0, 508, 502, 489, 499, 543, 508, 506, 527, 0, 0, 0, 0, 0, 0, 0, 0, 521, 509, 535, 502, 471, 492, 543, 500, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4143, 4076, 4159, 4010, 4100, 4125, 4082, 4073] avg=4096.0000 max=4159 max_rank=2 min=4010 min_rank=3 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[488, 492, 529, 527, 550, 495, 470, 495, 13, 6, 0, 0, 0, 0, 0, 0, 502, 529, 490, 522, 488, 527, 505, 488, 0, 0, 0, 0, 0, 0, 0, 0, 475, 497, 490, 504, 501, 518, 520, 493, 20, 6, 0, 0, 0, 0, 0, 0, 512, 514, 543, 511, 510, 497, 492, 512, 0, 0, 0, 0, 0, 0, 0, 0, 528, 497, 506, 584, 505, 542, 530, 517, 1, 0, 0, 0, 0, 0, 0, 0, 540, 500, 492, 494, 537, 507, 538, 518, 8, 0, 0, 0, 0, 0, 0, 0, 511, 534, 545, 515, 512, 529, 498, 485, 0, 0, 0, 0, 0, 0, 0, 0, 521, 516, 509, 482, 474, 510, 500, 552, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4065, 4051, 4024, 4091, 4210, 4134, 4129, 4064] avg=4096.0000 max=4210 max_rank=4 min=4024 min_rank=2 imbalance_ratio=1.027832 max_load_reduction=28558 max_load_reduction_pct=0.871521 moved_to_echo=16537 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[513, 527, 487, 506, 482, 503, 513, 501, 13, 6, 0, 0, 0, 0, 0, 0, 531, 509, 488, 508, 543, 510, 512, 511, 0, 0, 0, 0, 0, 0, 0, 0, 500, 533, 497, 499, 527, 501, 541, 511, 19, 6, 0, 0, 0, 0, 0, 0, 510, 516, 522, 532, 531, 503, 518, 521, 0, 0, 0, 0, 0, 0, 0, 0, 489, 495, 515, 517, 474, 505, 553, 500, 1, 0, 0, 0, 0, 0, 0, 0, 499, 487, 480, 504, 512, 493, 526, 533, 8, 0, 0, 0, 0, 0, 0, 0, 531, 503, 526, 533, 494, 516, 493, 529, 0, 0, 0, 0, 0, 0, 0, 0, 515, 517, 495, 501, 544, 510, 492, 528, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4051, 4112, 4134, 4153, 4049, 4042, 4125, 4102] avg=4096.0000 max=4153 max_rank=3 min=4042 min_rank=5 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16318 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 3968, 4224, 4224, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[519, 527, 522, 547, 505, 537, 456, 518, 516, 533, 505, 490, 503, 504, 519, 528, 529, 489, 503, 512, 537, 492, 525, 493, 524, 522, 506, 503, 500, 540, 504, 541, 484, 476, 544, 546, 520, 529, 532, 506, 501, 516, 528, 475, 508, 547, 497, 521, 497, 519, 485, 533, 507, 503, 499, 511, 534, 489, 538, 494, 467, 522, 497, 494] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[491, 500, 502, 498, 481, 526, 486, 543, 517, 461, 533, 538, 520, 476, 525, 537, 535, 502, 493, 523, 509, 510, 534, 508, 478, 526, 492, 471, 527, 512, 512, 520, 513, 522, 502, 498, 529, 524, 487, 507, 509, 529, 535, 512, 531, 496, 516, 492, 479, 536, 493, 533, 497, 494, 524, 521, 526, 519, 525, 533, 510, 535, 517, 538] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[492, 504, 520, 487, 511, 514, 515, 515, 521, 559, 499, 527, 503, 499, 512, 499, 474, 516, 501, 531, 491, 522, 535, 547, 531, 497, 507, 523, 504, 489, 535, 505, 541, 491, 518, 533, 495, 484, 526, 546, 548, 521, 495, 476, 489, 515, 511, 496, 539, 552, 523, 534, 478, 502, 481, 491, 522, 545, 478, 495, 525, 540, 507, 486] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[507, 533, 512, 533, 510, 495, 517, 505, 477, 467, 545, 488, 521, 508, 524, 507, 495, 514, 511, 529, 496, 493, 517, 484, 521, 508, 498, 497, 533, 550, 484, 564, 503, 502, 520, 528, 516, 530, 510, 540, 498, 484, 524, 549, 520, 501, 496, 499, 514, 529, 518, 523, 492, 482, 470, 509, 528, 498, 535, 508, 522, 536, 516, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[527, 519, 517, 482, 489, 506, 517, 552, 518, 493, 525, 513, 519, 497, 493, 518, 483, 503, 515, 521, 489, 504, 505, 474, 527, 497, 525, 547, 525, 517, 523, 515, 515, 536, 521, 510, 511, 537, 476, 522, 517, 524, 543, 491, 479, 518, 523, 514, 493, 517, 543, 516, 472, 528, 525, 519, 504, 475, 514, 489, 501, 522, 533, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[477, 562, 495, 474, 509, 519, 497, 501, 509, 472, 535, 513, 501, 546, 523, 506, 460, 536, 526, 519, 513, 501, 514, 541, 517, 445, 504, 505, 545, 470, 535, 523, 537, 511, 514, 496, 508, 519, 478, 482, 553, 492, 527, 518, 547, 527, 514, 503, 517, 496, 539, 481, 500, 492, 524, 514, 560, 544, 525, 498, 524, 509, 515, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[523, 537, 483, 480, 506, 505, 557, 497, 505, 509, 528, 504, 516, 550, 527, 501, 483, 465, 522, 530, 519, 501, 522, 503, 522, 527, 553, 509, 539, 497, 500, 543, 495, 517, 487, 476, 497, 542, 522, 497, 484, 511, 510, 515, 472, 540, 523, 516, 496, 514, 555, 521, 534, 527, 480, 500, 510, 503, 506, 537, 522, 500, 486, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[573, 479, 509, 525, 532, 478, 543, 513, 496, 498, 543, 522, 520, 491, 510, 501, 534, 498, 506, 524, 491, 535, 507, 497, 540, 545, 522, 481, 510, 488, 465, 516, 556, 512, 492, 528, 497, 538, 514, 517, 510, 539, 470, 504, 524, 517, 512, 524, 509, 491, 525, 489, 534, 537, 497, 501, 527, 517, 508, 478, 525, 499, 490, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[519, 527, 522, 547, 505, 537, 456, 518, 4, 3, 0, 0, 0, 0, 0, 0, 516, 533, 505, 490, 503, 504, 519, 528, 0, 0, 0, 0, 0, 0, 0, 0, 529, 489, 503, 512, 537, 492, 525, 493, 17, 5, 0, 0, 0, 0, 0, 0, 524, 522, 506, 503, 500, 540, 504, 524, 0, 0, 0, 0, 0, 0, 0, 0, 484, 476, 544, 546, 520, 518, 532, 506, 0, 0, 0, 0, 0, 0, 0, 0, 501, 516, 528, 475, 508, 544, 497, 521, 0, 0, 0, 0, 0, 0, 0, 0, 497, 519, 485, 533, 507, 503, 499, 511, 7, 6, 0, 0, 0, 0, 0, 0, 523, 489, 538, 494, 467, 522, 497, 494, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4138, 4098, 4102, 4123, 4126, 4090, 4067, 4024] avg=4096.0000 max=4138 max_rank=0 min=4024 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16307 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[491, 500, 502, 498, 481, 526, 486, 543, 4, 2, 0, 0, 0, 0, 0, 0, 517, 461, 533, 538, 520, 476, 525, 537, 0, 0, 0, 0, 0, 0, 0, 0, 535, 502, 493, 523, 509, 510, 534, 508, 16, 5, 0, 0, 0, 0, 0, 0, 478, 526, 492, 471, 527, 512, 512, 504, 0, 0, 0, 0, 0, 0, 0, 0, 513, 522, 502, 498, 529, 513, 487, 507, 0, 0, 0, 0, 0, 0, 0, 0, 509, 529, 535, 512, 531, 494, 516, 492, 0, 0, 0, 0, 0, 0, 0, 0, 479, 536, 493, 533, 497, 494, 524, 521, 7, 6, 0, 0, 0, 0, 0, 0, 515, 519, 525, 533, 510, 535, 517, 538, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4033, 4107, 4135, 4022, 4071, 4118, 4090, 4192] avg=4096.0000 max=4192 max_rank=7 min=4022 min_rank=3 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16471 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[507, 533, 512, 533, 510, 495, 517, 505, 4, 2, 0, 0, 0, 0, 0, 0, 477, 467, 545, 488, 521, 508, 524, 507, 0, 0, 0, 0, 0, 0, 0, 0, 495, 514, 511, 529, 496, 493, 517, 484, 17, 5, 0, 0, 0, 0, 0, 0, 521, 508, 498, 497, 533, 550, 484, 547, 0, 0, 0, 0, 0, 0, 0, 0, 503, 502, 520, 528, 516, 519, 510, 540, 0, 0, 0, 0, 0, 0, 0, 0, 498, 484, 524, 549, 520, 499, 496, 499, 0, 0, 0, 0, 0, 0, 0, 0, 514, 529, 518, 523, 492, 482, 470, 509, 7, 6, 0, 0, 0, 0, 0, 0, 517, 498, 535, 508, 522, 536, 516, 525, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4037, 4061, 4138, 4138, 4069, 4050, 4157] avg=4096.0000 max=4157 max_rank=7 min=4037 min_rank=1 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16414 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[477, 562, 495, 474, 509, 519, 497, 501, 4, 2, 0, 0, 0, 0, 0, 0, 509, 472, 535, 513, 501, 546, 523, 506, 0, 0, 0, 0, 0, 0, 0, 0, 460, 536, 526, 519, 513, 501, 514, 541, 16, 5, 0, 0, 0, 0, 0, 0, 517, 445, 504, 505, 545, 470, 535, 507, 0, 0, 0, 0, 0, 0, 0, 0, 537, 511, 514, 496, 508, 508, 478, 482, 0, 0, 0, 0, 0, 0, 0, 0, 553, 492, 527, 518, 547, 525, 514, 503, 0, 0, 0, 0, 0, 0, 0, 0, 517, 496, 539, 481, 500, 492, 524, 514, 7, 7, 0, 0, 0, 0, 0, 0, 548, 544, 525, 498, 524, 509, 515, 511, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4040, 4105, 4131, 4028, 4034, 4179, 4077, 4174] avg=4096.0000 max=4179 max_rank=5 min=4028 min_rank=3 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16464 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[527, 519, 517, 482, 489, 506, 517, 552, 7, 8, 0, 0, 0, 0, 0, 0, 518, 493, 525, 513, 519, 497, 493, 518, 5, 0, 0, 0, 0, 0, 0, 0, 483, 503, 515, 521, 489, 504, 505, 474, 19, 9, 0, 0, 0, 0, 0, 0, 527, 497, 525, 547, 525, 517, 523, 496, 0, 0, 0, 0, 0, 0, 0, 0, 515, 536, 521, 510, 511, 518, 476, 522, 0, 0, 0, 0, 0, 0, 0, 0, 517, 524, 543, 491, 479, 505, 523, 514, 0, 0, 0, 0, 0, 0, 0, 0, 493, 517, 543, 516, 472, 528, 525, 519, 12, 11, 0, 0, 0, 0, 0, 0, 484, 475, 514, 489, 501, 522, 533, 525, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4124, 4081, 4022, 4157, 4109, 4096, 4136, 4043] avg=4096.0000 max=4157 max_rank=3 min=4022 min_rank=2 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[492, 504, 520, 487, 511, 514, 515, 515, 3, 2, 0, 0, 0, 0, 0, 0, 521, 559, 499, 527, 503, 499, 512, 499, 0, 0, 0, 0, 0, 0, 0, 0, 474, 516, 501, 531, 491, 522, 535, 547, 15, 5, 0, 0, 0, 0, 0, 0, 531, 497, 507, 523, 504, 489, 535, 490, 0, 0, 0, 0, 0, 0, 0, 0, 541, 491, 518, 533, 495, 475, 526, 546, 0, 0, 0, 0, 0, 0, 0, 0, 548, 521, 495, 476, 489, 513, 511, 496, 0, 0, 0, 0, 0, 0, 0, 0, 539, 552, 523, 534, 478, 502, 481, 491, 6, 6, 0, 0, 0, 0, 0, 0, 511, 545, 478, 495, 525, 540, 507, 486, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4119, 4137, 4076, 4125, 4049, 4112, 4087] avg=4096.0000 max=4137 max_rank=2 min=4049 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[523, 537, 483, 480, 506, 505, 557, 497, 4, 2, 0, 0, 0, 0, 0, 0, 505, 509, 528, 504, 516, 550, 527, 501, 0, 0, 0, 0, 0, 0, 0, 0, 483, 465, 522, 530, 519, 501, 522, 503, 17, 5, 0, 0, 0, 0, 0, 0, 522, 527, 553, 509, 539, 497, 500, 526, 0, 0, 0, 0, 0, 0, 0, 0, 495, 517, 487, 476, 497, 531, 522, 497, 0, 0, 0, 0, 0, 0, 0, 0, 484, 511, 510, 515, 472, 538, 523, 516, 0, 0, 0, 0, 0, 0, 0, 0, 496, 514, 555, 521, 534, 527, 480, 500, 7, 6, 0, 0, 0, 0, 0, 0, 499, 503, 506, 537, 522, 500, 486, 510, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4140, 4067, 4173, 4022, 4069, 4140, 4063] avg=4096.0000 max=4173 max_rank=3 min=4022 min_rank=4 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16294 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[573, 479, 509, 525, 532, 478, 543, 513, 4, 2, 0, 0, 0, 0, 0, 0, 496, 498, 543, 522, 520, 491, 510, 501, 0, 0, 0, 0, 0, 0, 0, 0, 534, 498, 506, 524, 491, 535, 507, 497, 16, 5, 0, 0, 0, 0, 0, 0, 540, 545, 522, 481, 510, 488, 465, 500, 0, 0, 0, 0, 0, 0, 0, 0, 556, 512, 492, 528, 497, 527, 514, 517, 0, 0, 0, 0, 0, 0, 0, 0, 510, 539, 470, 504, 524, 515, 512, 524, 0, 0, 0, 0, 0, 0, 0, 0, 509, 491, 525, 489, 534, 537, 497, 501, 7, 6, 0, 0, 0, 0, 0, 0, 516, 517, 508, 478, 525, 499, 490, 495, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4158, 4081, 4113, 4051, 4143, 4098, 4096, 4028] avg=4096.0000 max=4158 max_rank=0 min=4028 min_rank=7 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16365 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[531, 505, 502, 501, 481, 512, 493, 514, 479, 489, 489, 507, 501, 551, 521, 533, 520, 483, 491, 465, 516, 533, 522, 503, 506, 510, 527, 532, 527, 512, 527, 503, 534, 529, 520, 472, 479, 512, 511, 494, 533, 477, 521, 520, 517, 473, 545, 522, 503, 516, 567, 518, 529, 536, 516, 505, 512, 481, 536, 499, 550, 533, 504, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[512, 523, 527, 498, 527, 505, 515, 502, 495, 509, 511, 514, 499, 516, 532, 526, 537, 498, 512, 493, 527, 478, 485, 508, 487, 539, 526, 477, 487, 554, 520, 478, 536, 511, 467, 468, 525, 503, 517, 510, 528, 522, 513, 544, 469, 545, 523, 532, 490, 557, 544, 522, 503, 528, 518, 503, 530, 500, 521, 479, 487, 551, 506, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[552, 521, 540, 552, 506, 498, 512, 497, 537, 472, 518, 487, 497, 526, 522, 519, 527, 524, 501, 524, 516, 501, 515, 513, 523, 470, 484, 532, 516, 524, 546, 545, 502, 511, 498, 519, 519, 494, 518, 492, 505, 493, 531, 531, 490, 503, 520, 499, 516, 500, 528, 527, 512, 537, 447, 484, 524, 520, 498, 470, 483, 572, 501, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[531, 510, 527, 502, 527, 472, 528, 519, 528, 541, 518, 523, 512, 491, 496, 473, 539, 498, 478, 562, 494, 508, 499, 522, 477, 540, 485, 520, 522, 501, 515, 494, 525, 525, 488, 493, 495, 507, 512, 523, 507, 504, 528, 507, 519, 501, 531, 513, 538, 542, 520, 524, 493, 485, 525, 527, 481, 496, 535, 538, 521, 488, 504, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[482, 498, 513, 516, 524, 491, 510, 507, 474, 542, 519, 502, 525, 515, 501, 547, 514, 499, 540, 482, 522, 528, 496, 501, 491, 528, 535, 507, 524, 486, 508, 518, 521, 545, 513, 517, 547, 533, 519, 514, 481, 497, 523, 527, 504, 505, 499, 487, 505, 456, 530, 503, 514, 562, 509, 497, 507, 518, 541, 514, 472, 505, 529, 529] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[493, 520, 542, 510, 495, 524, 469, 497, 495, 537, 506, 493, 503, 526, 512, 493, 508, 470, 539, 529, 526, 531, 536, 494, 526, 519, 494, 497, 540, 502, 543, 503, 508, 512, 527, 521, 541, 499, 526, 509, 540, 524, 521, 525, 454, 472, 506, 485, 520, 524, 510, 518, 501, 511, 531, 521, 509, 478, 545, 503, 501, 522, 499, 533] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[515, 514, 530, 481, 541, 530, 528, 525, 527, 509, 499, 497, 546, 489, 481, 527, 508, 500, 481, 512, 521, 489, 498, 490, 494, 477, 536, 516, 507, 532, 462, 523, 498, 534, 506, 539, 470, 482, 507, 494, 511, 545, 531, 509, 483, 531, 480, 498, 547, 487, 525, 529, 503, 508, 542, 505, 523, 529, 526, 500, 535, 533, 520, 553] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[506, 494, 492, 518, 540, 537, 488, 541, 515, 524, 524, 486, 504, 514, 512, 523, 544, 513, 532, 478, 495, 522, 503, 501, 514, 507, 537, 526, 496, 514, 499, 513, 509, 520, 545, 513, 544, 519, 511, 510, 468, 543, 505, 484, 505, 496, 488, 514, 517, 495, 513, 503, 522, 483, 513, 503, 522, 491, 485, 519, 538, 519, 541, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[531, 510, 522, 502, 527, 472, 528, 519, 0, 0, 0, 0, 0, 0, 0, 0, 528, 541, 518, 523, 512, 491, 496, 473, 5, 2, 0, 0, 0, 0, 0, 0, 539, 498, 478, 562, 494, 508, 499, 522, 21, 0, 0, 0, 0, 0, 0, 0, 477, 540, 485, 520, 522, 501, 515, 494, 0, 0, 0, 0, 0, 0, 0, 0, 525, 525, 488, 493, 495, 507, 512, 523, 0, 0, 0, 0, 0, 0, 0, 0, 507, 504, 528, 507, 519, 501, 531, 513, 15, 3, 0, 0, 0, 0, 0, 0, 538, 542, 496, 524, 493, 485, 525, 527, 0, 0, 0, 0, 0, 0, 0, 0, 481, 496, 535, 538, 521, 471, 504, 521, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4089, 4121, 4054, 4068, 4128, 4130, 4067] avg=4096.0000 max=4130 max_rank=6 min=4054 min_rank=3 imbalance_ratio=1.008301 max_load_reduction=28638 max_load_reduction_pct=0.873962 moved_to_echo=16393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[552, 521, 535, 552, 506, 498, 512, 497, 0, 0, 0, 0, 0, 0, 0, 0, 537, 472, 518, 487, 497, 526, 522, 519, 5, 2, 0, 0, 0, 0, 0, 0, 527, 524, 501, 524, 516, 501, 515, 513, 21, 0, 0, 0, 0, 0, 0, 0, 523, 470, 484, 532, 516, 524, 546, 545, 0, 0, 0, 0, 0, 0, 0, 0, 502, 511, 498, 519, 519, 494, 518, 492, 0, 0, 0, 0, 0, 0, 0, 0, 505, 493, 531, 531, 490, 503, 520, 499, 18, 3, 0, 0, 0, 0, 0, 0, 516, 500, 504, 527, 512, 537, 447, 484, 0, 0, 0, 0, 0, 0, 0, 0, 524, 520, 498, 470, 483, 552, 501, 507, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4173, 4085, 4142, 4140, 4053, 4093, 4027, 4055] avg=4096.0000 max=4173 max_rank=0 min=4027 min_rank=6 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16228 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[493, 520, 537, 510, 495, 524, 469, 497, 0, 0, 0, 0, 0, 0, 0, 0, 495, 537, 506, 493, 503, 526, 512, 493, 5, 2, 0, 0, 0, 0, 0, 0, 508, 470, 539, 529, 526, 531, 536, 494, 20, 0, 0, 0, 0, 0, 0, 0, 526, 519, 494, 497, 540, 502, 543, 503, 0, 0, 0, 0, 0, 0, 0, 0, 508, 512, 527, 521, 541, 499, 526, 509, 0, 0, 0, 0, 0, 0, 0, 0, 540, 524, 521, 525, 454, 472, 506, 485, 16, 3, 0, 0, 0, 0, 0, 0, 520, 524, 487, 518, 501, 511, 531, 521, 0, 0, 0, 0, 0, 0, 0, 0, 509, 478, 545, 503, 501, 504, 499, 533, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4045, 4072, 4153, 4124, 4143, 4046, 4113, 4072] avg=4096.0000 max=4153 max_rank=2 min=4045 min_rank=0 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16374 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[512, 523, 517, 498, 527, 505, 515, 502, 0, 0, 0, 0, 0, 0, 0, 0, 495, 509, 511, 514, 499, 516, 532, 526, 10, 4, 6, 0, 0, 0, 0, 0, 537, 498, 512, 493, 527, 478, 485, 508, 26, 0, 0, 0, 0, 0, 0, 0, 487, 539, 526, 477, 487, 542, 520, 478, 0, 0, 0, 0, 0, 0, 0, 0, 536, 511, 467, 468, 525, 503, 517, 510, 6, 0, 0, 0, 0, 0, 0, 0, 528, 522, 513, 544, 469, 545, 523, 532, 21, 4, 0, 0, 0, 0, 0, 0, 490, 557, 514, 522, 503, 528, 518, 503, 0, 0, 0, 0, 0, 0, 0, 0, 530, 500, 521, 479, 487, 526, 506, 499, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4122, 4064, 4056, 4043, 4201, 4135, 4048] avg=4096.0000 max=4201 max_rank=5 min=4043 min_rank=4 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16427 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[531, 505, 497, 501, 481, 512, 493, 514, 0, 0, 0, 0, 0, 0, 0, 0, 479, 489, 489, 507, 501, 551, 521, 533, 5, 2, 0, 0, 0, 0, 0, 0, 520, 483, 491, 465, 516, 533, 522, 503, 23, 0, 0, 0, 0, 0, 0, 0, 506, 510, 527, 532, 527, 512, 527, 503, 0, 0, 0, 0, 0, 0, 0, 0, 534, 529, 520, 472, 479, 512, 511, 494, 0, 0, 0, 0, 0, 0, 0, 0, 533, 477, 521, 520, 517, 473, 545, 522, 17, 3, 0, 0, 0, 0, 0, 0, 503, 516, 541, 518, 529, 536, 516, 505, 0, 0, 0, 0, 0, 0, 0, 0, 512, 481, 536, 499, 550, 514, 504, 518, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4034, 4077, 4056, 4144, 4051, 4128, 4164, 4114] avg=4096.0000 max=4164 max_rank=6 min=4034 min_rank=0 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16457 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[482, 498, 508, 516, 524, 491, 510, 507, 0, 0, 0, 0, 0, 0, 0, 0, 474, 542, 519, 502, 525, 515, 501, 547, 5, 2, 0, 0, 0, 0, 0, 0, 514, 499, 540, 482, 522, 528, 496, 501, 21, 0, 0, 0, 0, 0, 0, 0, 491, 528, 535, 507, 524, 486, 508, 518, 0, 0, 0, 0, 0, 0, 0, 0, 521, 545, 513, 517, 547, 533, 519, 514, 0, 0, 0, 0, 0, 0, 0, 0, 481, 497, 523, 527, 504, 505, 499, 487, 16, 3, 0, 0, 0, 0, 0, 0, 505, 456, 506, 503, 514, 562, 509, 497, 0, 0, 0, 0, 0, 0, 0, 0, 507, 518, 541, 514, 472, 487, 529, 529, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4036, 4132, 4103, 4097, 4209, 4042, 4052, 4097] avg=4096.0000 max=4209 max_rank=4 min=4036 min_rank=0 imbalance_ratio=1.027588 max_load_reduction=28559 max_load_reduction_pct=0.871552 moved_to_echo=16400 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[515, 514, 525, 481, 541, 530, 528, 525, 0, 0, 0, 0, 0, 0, 0, 0, 527, 509, 499, 497, 546, 489, 481, 527, 5, 2, 0, 0, 0, 0, 0, 0, 508, 500, 481, 512, 521, 489, 498, 490, 21, 0, 0, 0, 0, 0, 0, 0, 494, 477, 536, 516, 507, 532, 462, 523, 0, 0, 0, 0, 0, 0, 0, 0, 498, 534, 506, 539, 470, 482, 507, 494, 0, 0, 0, 0, 0, 0, 0, 0, 511, 545, 531, 509, 483, 531, 480, 498, 17, 3, 0, 0, 0, 0, 0, 0, 547, 487, 501, 529, 503, 508, 542, 505, 0, 0, 0, 0, 0, 0, 0, 0, 523, 529, 526, 500, 535, 514, 520, 553, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4159, 4082, 4020, 4047, 4030, 4108, 4122, 4200] avg=4096.0000 max=4200 max_rank=7 min=4020 min_rank=2 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16460 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[506, 494, 487, 518, 540, 537, 488, 541, 0, 0, 0, 0, 0, 0, 0, 0, 515, 524, 524, 486, 504, 514, 512, 523, 5, 2, 0, 0, 0, 0, 0, 0, 544, 513, 532, 478, 495, 522, 503, 501, 21, 0, 0, 0, 0, 0, 0, 0, 514, 507, 537, 526, 496, 514, 499, 513, 0, 0, 0, 0, 0, 0, 0, 0, 509, 520, 545, 513, 544, 519, 511, 510, 0, 0, 0, 0, 0, 0, 0, 0, 468, 543, 505, 484, 505, 496, 488, 514, 16, 3, 0, 0, 0, 0, 0, 0, 517, 495, 489, 503, 522, 483, 513, 503, 0, 0, 0, 0, 0, 0, 0, 0, 522, 491, 485, 519, 538, 501, 541, 518, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4109, 4109, 4106, 4171, 4022, 4025, 4115] avg=4096.0000 max=4171 max_rank=4 min=4022 min_rank=5 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16333 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 3968, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[508, 557, 533, 591, 502, 525, 511, 528, 520, 480, 547, 472, 518, 515, 492, 550, 495, 528, 510, 560, 502, 537, 500, 519, 531, 506, 480, 490, 513, 511, 515, 528, 521, 474, 537, 523, 520, 524, 504, 491, 472, 502, 500, 516, 507, 517, 513, 543, 508, 476, 476, 483, 486, 490, 507, 476, 535, 477, 495, 528, 532, 520, 543, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[525, 508, 546, 499, 529, 539, 497, 494, 544, 545, 536, 500, 490, 516, 521, 497, 492, 500, 544, 511, 506, 476, 520, 510, 531, 506, 513, 473, 512, 514, 509, 505, 546, 534, 508, 530, 506, 497, 517, 510, 521, 505, 474, 552, 522, 534, 526, 501, 468, 497, 499, 530, 474, 521, 493, 506, 514, 522, 532, 553, 529, 503, 468, 468] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[507, 508, 519, 506, 512, 489, 507, 500, 508, 471, 518, 509, 512, 483, 499, 509, 477, 479, 554, 525, 520, 521, 551, 507, 527, 549, 497, 467, 548, 518, 499, 523, 506, 531, 473, 532, 498, 502, 510, 489, 503, 521, 521, 478, 522, 535, 491, 506, 472, 510, 505, 525, 527, 531, 519, 524, 468, 510, 542, 559, 548, 551, 514, 526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[493, 501, 536, 527, 516, 505, 490, 490, 492, 537, 518, 494, 517, 505, 513, 511, 490, 511, 521, 531, 559, 544, 490, 533, 475, 524, 521, 530, 479, 507, 526, 487, 493, 499, 525, 517, 533, 535, 501, 491, 497, 506, 506, 519, 479, 509, 525, 503, 537, 545, 504, 469, 470, 504, 563, 533, 500, 531, 509, 512, 539, 510, 511, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[493, 506, 511, 505, 528, 522, 534, 510, 504, 486, 532, 484, 539, 500, 511, 510, 515, 522, 513, 497, 538, 494, 518, 492, 529, 524, 516, 566, 499, 487, 505, 484, 525, 517, 524, 540, 571, 514, 533, 512, 535, 522, 503, 454, 536, 510, 513, 486, 536, 481, 500, 485, 470, 557, 553, 483, 532, 514, 521, 496, 497, 511, 485, 478] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[539, 522, 549, 490, 515, 464, 520, 495, 469, 513, 506, 518, 488, 543, 507, 549, 517, 506, 486, 496, 504, 508, 491, 514, 534, 505, 539, 536, 522, 549, 497, 516, 513, 519, 483, 546, 497, 458, 552, 525, 534, 518, 528, 505, 517, 526, 500, 509, 504, 512, 508, 511, 521, 563, 507, 503, 514, 514, 469, 519, 498, 474, 528, 486] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[511, 525, 531, 540, 487, 527, 519, 556, 553, 525, 476, 484, 501, 500, 472, 468, 511, 527, 537, 529, 481, 508, 488, 503, 499, 547, 539, 517, 520, 524, 503, 481, 521, 494, 502, 522, 522, 477, 482, 498, 479, 535, 487, 550, 522, 521, 490, 480, 501, 546, 524, 530, 504, 481, 524, 521, 516, 542, 555, 493, 502, 498, 525, 535] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[525, 485, 514, 537, 552, 476, 544, 523, 508, 515, 504, 518, 518, 523, 503, 512, 505, 511, 519, 488, 517, 449, 520, 509, 510, 491, 487, 547, 527, 503, 527, 484, 484, 548, 518, 503, 523, 519, 560, 520, 476, 509, 516, 491, 498, 517, 529, 516, 525, 510, 524, 469, 511, 516, 489, 512, 483, 522, 525, 532, 526, 528, 496, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[508, 557, 498, 591, 502, 525, 511, 528, 0, 0, 0, 0, 0, 0, 0, 0, 520, 480, 547, 472, 518, 515, 492, 550, 17, 3, 3, 0, 0, 0, 0, 0, 495, 528, 510, 560, 502, 537, 500, 519, 3, 0, 0, 0, 0, 0, 0, 0, 531, 500, 480, 490, 513, 511, 515, 528, 0, 0, 0, 0, 0, 0, 0, 0, 521, 474, 537, 508, 520, 524, 504, 491, 0, 0, 0, 0, 0, 0, 0, 0, 472, 502, 500, 516, 507, 517, 513, 543, 12, 3, 0, 0, 0, 0, 0, 0, 508, 476, 476, 483, 486, 490, 507, 476, 32, 0, 0, 0, 0, 0, 0, 0, 535, 477, 495, 511, 532, 520, 543, 498, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4220, 4117, 4154, 4068, 4079, 4085, 3934, 4111] avg=4096.0000 max=4220 max_rank=0 min=3934 min_rank=6 imbalance_ratio=1.030273 max_load_reduction=28548 max_load_reduction_pct=0.871216 moved_to_echo=16209 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[493, 506, 477, 505, 528, 522, 534, 510, 0, 0, 0, 0, 0, 0, 0, 0, 504, 486, 532, 484, 539, 500, 511, 510, 15, 3, 3, 0, 0, 0, 0, 0, 515, 522, 513, 497, 538, 494, 518, 492, 4, 0, 0, 0, 0, 0, 0, 0, 529, 517, 516, 566, 499, 487, 505, 484, 0, 0, 0, 0, 0, 0, 0, 0, 525, 517, 524, 524, 571, 514, 533, 512, 0, 0, 0, 0, 0, 0, 0, 0, 535, 522, 503, 454, 536, 510, 513, 486, 13, 3, 0, 0, 0, 0, 0, 0, 536, 481, 500, 485, 470, 557, 553, 483, 31, 0, 0, 0, 0, 0, 0, 0, 532, 514, 521, 481, 497, 511, 485, 478, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4087, 4093, 4103, 4220, 4075, 4096, 4019] avg=4096.0000 max=4220 max_rank=4 min=4019 min_rank=7 imbalance_ratio=1.030273 max_load_reduction=28548 max_load_reduction_pct=0.871216 moved_to_echo=16410 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[525, 508, 510, 499, 529, 539, 497, 494, 0, 0, 0, 0, 0, 0, 0, 0, 544, 545, 536, 500, 490, 516, 521, 497, 17, 3, 3, 0, 0, 0, 0, 0, 492, 500, 544, 511, 506, 476, 520, 510, 3, 0, 0, 0, 0, 0, 0, 0, 531, 500, 513, 473, 512, 514, 509, 505, 0, 0, 0, 0, 0, 0, 0, 0, 546, 534, 508, 515, 506, 497, 517, 510, 0, 0, 0, 0, 0, 0, 0, 0, 521, 505, 474, 552, 522, 534, 526, 501, 12, 3, 0, 0, 0, 0, 0, 0, 468, 497, 499, 530, 474, 521, 493, 506, 33, 0, 0, 0, 0, 0, 0, 0, 514, 522, 532, 536, 529, 503, 468, 468, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4172, 4062, 4057, 4133, 4150, 4021, 4072] avg=4096.0000 max=4172 max_rank=1 min=4021 min_rank=6 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16376 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[507, 508, 485, 506, 512, 489, 507, 500, 0, 0, 0, 0, 0, 0, 0, 0, 508, 471, 518, 509, 512, 483, 499, 509, 18, 3, 3, 0, 0, 0, 0, 0, 477, 479, 554, 525, 520, 521, 551, 507, 4, 0, 0, 0, 0, 0, 0, 0, 527, 542, 497, 467, 548, 518, 499, 523, 0, 0, 0, 0, 0, 0, 0, 0, 506, 531, 473, 517, 498, 502, 510, 489, 0, 0, 0, 0, 0, 0, 0, 0, 503, 521, 521, 478, 522, 535, 491, 506, 12, 3, 0, 0, 0, 0, 0, 0, 472, 510, 505, 525, 527, 531, 519, 524, 31, 0, 0, 0, 0, 0, 0, 0, 468, 510, 542, 541, 548, 551, 514, 526, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4014, 4033, 4138, 4121, 4026, 4092, 4144, 4200] avg=4096.0000 max=4200 max_rank=7 min=4014 min_rank=0 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16462 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[493, 501, 501, 527, 516, 505, 490, 490, 0, 0, 0, 0, 0, 0, 0, 0, 492, 537, 518, 494, 517, 505, 513, 511, 16, 3, 3, 0, 0, 0, 0, 0, 490, 511, 521, 531, 559, 544, 490, 533, 4, 0, 0, 0, 0, 0, 0, 0, 475, 517, 521, 530, 479, 507, 526, 487, 0, 0, 0, 0, 0, 0, 0, 0, 493, 499, 525, 502, 533, 535, 501, 491, 0, 0, 0, 0, 0, 0, 0, 0, 497, 506, 506, 519, 479, 509, 525, 503, 12, 3, 0, 0, 0, 0, 0, 0, 537, 545, 504, 469, 470, 504, 563, 533, 32, 0, 0, 0, 0, 0, 0, 0, 500, 531, 509, 496, 539, 510, 511, 520, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4023, 4109, 4183, 4042, 4079, 4059, 4157, 4116] avg=4096.0000 max=4183 max_rank=2 min=4023 min_rank=0 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16411 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[511, 525, 490, 540, 487, 527, 519, 556, 0, 0, 0, 0, 0, 0, 0, 0, 553, 525, 476, 484, 501, 500, 472, 468, 19, 8, 5, 0, 0, 0, 0, 0, 511, 527, 537, 529, 481, 508, 488, 503, 8, 0, 0, 0, 0, 0, 0, 0, 499, 532, 539, 517, 520, 524, 503, 481, 0, 0, 0, 0, 0, 0, 0, 0, 521, 494, 502, 498, 522, 477, 482, 498, 0, 0, 0, 0, 0, 0, 0, 0, 479, 535, 487, 550, 522, 521, 490, 480, 16, 7, 0, 0, 0, 0, 0, 0, 501, 546, 524, 530, 504, 481, 524, 521, 36, 0, 0, 0, 0, 0, 0, 0, 516, 542, 555, 474, 502, 498, 525, 535, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4155, 4011, 4092, 4115, 3994, 4087, 4167, 4147] avg=4096.0000 max=4167 max_rank=6 min=3994 min_rank=4 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16395 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[539, 522, 513, 490, 515, 464, 520, 495, 0, 0, 0, 0, 0, 0, 0, 0, 469, 513, 506, 518, 488, 543, 507, 549, 16, 3, 3, 0, 0, 0, 0, 0, 517, 506, 486, 496, 504, 508, 491, 514, 3, 0, 0, 0, 0, 0, 0, 0, 534, 499, 539, 536, 522, 549, 497, 516, 0, 0, 0, 0, 0, 0, 0, 0, 513, 519, 483, 530, 497, 458, 552, 525, 0, 0, 0, 0, 0, 0, 0, 0, 534, 518, 528, 505, 517, 526, 500, 509, 13, 3, 0, 0, 0, 0, 0, 0, 504, 512, 508, 511, 521, 563, 507, 503, 33, 0, 0, 0, 0, 0, 0, 0, 514, 514, 469, 503, 498, 474, 528, 486, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4058, 4115, 4025, 4192, 4077, 4153, 4162, 3986] avg=4096.0000 max=4192 max_rank=3 min=3986 min_rank=7 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[525, 485, 480, 537, 552, 476, 544, 523, 0, 0, 0, 0, 0, 0, 0, 0, 508, 515, 504, 518, 518, 523, 503, 512, 17, 3, 3, 0, 0, 0, 0, 0, 505, 511, 519, 488, 517, 449, 520, 509, 3, 0, 0, 0, 0, 0, 0, 0, 510, 485, 487, 547, 527, 503, 527, 484, 0, 0, 0, 0, 0, 0, 0, 0, 484, 548, 518, 488, 523, 519, 560, 520, 0, 0, 0, 0, 0, 0, 0, 0, 476, 509, 516, 491, 498, 517, 529, 516, 12, 3, 0, 0, 0, 0, 0, 0, 525, 510, 524, 469, 511, 516, 489, 512, 31, 0, 0, 0, 0, 0, 0, 0, 483, 522, 525, 515, 526, 528, 496, 522, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4122, 4124, 4021, 4070, 4160, 4067, 4087, 4117] avg=4096.0000 max=4160 max_rank=4 min=4021 min_rank=2 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 3968, 4224, 4224, 4096, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 3968, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[491, 503, 535, 505, 479, 507, 483, 489, 518, 516, 519, 519, 540, 511, 528, 493, 507, 499, 534, 482, 485, 506, 553, 517, 497, 527, 529, 515, 508, 505, 517, 518, 512, 487, 542, 526, 502, 502, 493, 503, 523, 541, 523, 533, 523, 520, 520, 463, 502, 480, 498, 489, 554, 509, 480, 520, 550, 517, 502, 509, 552, 556, 497, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[483, 551, 489, 488, 508, 473, 497, 526, 479, 518, 532, 576, 493, 514, 513, 480, 584, 485, 538, 548, 555, 529, 535, 490, 518, 500, 493, 508, 537, 530, 515, 527, 537, 525, 490, 511, 545, 506, 499, 500, 489, 477, 482, 518, 529, 505, 510, 503, 519, 485, 516, 529, 500, 495, 506, 515, 524, 486, 493, 527, 534, 485, 524, 492] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[573, 515, 512, 525, 521, 524, 518, 530, 519, 492, 481, 481, 540, 515, 534, 537, 504, 513, 515, 501, 503, 507, 478, 518, 523, 476, 492, 509, 528, 493, 529, 496, 537, 482, 507, 544, 545, 525, 510, 520, 533, 497, 523, 528, 534, 513, 500, 491, 462, 510, 538, 507, 488, 481, 507, 488, 513, 536, 513, 498, 480, 507, 552, 497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[490, 539, 525, 541, 547, 488, 466, 545, 527, 507, 526, 478, 485, 503, 518, 511, 542, 515, 488, 495, 513, 525, 473, 541, 500, 484, 482, 533, 530, 539, 502, 447, 482, 500, 526, 502, 513, 512, 547, 513, 532, 535, 522, 514, 493, 512, 510, 540, 505, 529, 548, 485, 507, 551, 468, 520, 507, 529, 568, 490, 518, 499, 515, 471] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[518, 495, 493, 514, 535, 482, 514, 520, 528, 528, 510, 519, 496, 534, 513, 511, 538, 518, 495, 483, 490, 541, 521, 503, 510, 532, 521, 531, 506, 509, 516, 500, 529, 446, 549, 504, 538, 511, 503, 531, 533, 501, 512, 552, 506, 503, 513, 491, 514, 497, 497, 484, 504, 485, 465, 514, 526, 535, 557, 503, 555, 510, 469, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[506, 514, 485, 523, 513, 510, 526, 500, 529, 514, 537, 532, 481, 518, 554, 520, 517, 498, 515, 464, 505, 465, 529, 521, 514, 520, 523, 517, 498, 536, 504, 525, 514, 512, 521, 510, 519, 537, 540, 531, 522, 511, 532, 515, 523, 511, 520, 496, 500, 543, 498, 499, 490, 494, 513, 516, 497, 501, 499, 501, 480, 530, 473, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[525, 514, 494, 476, 502, 527, 488, 521, 516, 512, 507, 499, 511, 522, 478, 508, 534, 531, 479, 580, 529, 475, 516, 507, 516, 502, 541, 521, 502, 486, 518, 497, 490, 510, 529, 559, 510, 517, 494, 503, 569, 498, 539, 510, 520, 501, 494, 473, 542, 519, 518, 508, 520, 524, 496, 499, 509, 493, 503, 535, 491, 512, 509, 540] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[505, 523, 511, 503, 525, 518, 496, 536, 532, 496, 471, 494, 518, 519, 539, 531, 565, 480, 501, 504, 518, 509, 514, 507, 520, 509, 498, 484, 519, 535, 515, 532, 531, 506, 532, 505, 520, 537, 480, 500, 512, 546, 531, 514, 482, 518, 474, 512, 508, 514, 496, 537, 496, 509, 509, 526, 522, 489, 486, 508, 515, 488, 523, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[491, 503, 535, 505, 479, 507, 483, 489, 8, 2, 0, 0, 0, 0, 0, 0, 518, 516, 519, 519, 540, 511, 514, 493, 0, 0, 0, 0, 0, 0, 0, 0, 501, 499, 534, 482, 485, 506, 553, 517, 0, 0, 0, 0, 0, 0, 0, 0, 497, 527, 529, 515, 508, 505, 517, 518, 0, 0, 0, 0, 0, 0, 0, 0, 512, 487, 517, 526, 502, 502, 493, 503, 0, 0, 0, 0, 0, 0, 0, 0, 511, 541, 523, 533, 523, 520, 520, 463, 0, 0, 0, 0, 0, 0, 0, 0, 502, 480, 498, 489, 554, 509, 480, 520, 25, 14, 4, 0, 0, 0, 0, 0, 550, 517, 502, 509, 552, 556, 497, 505, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4002, 4130, 4077, 4116, 4042, 4134, 4075, 4192] avg=4096.0000 max=4192 max_rank=7 min=4002 min_rank=0 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16443 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[573, 515, 512, 525, 521, 524, 518, 530, 8, 2, 0, 0, 0, 0, 0, 0, 519, 492, 481, 481, 540, 515, 520, 537, 0, 0, 0, 0, 0, 0, 0, 0, 499, 513, 515, 501, 503, 507, 478, 518, 0, 0, 0, 0, 0, 0, 0, 0, 523, 476, 492, 509, 528, 493, 529, 496, 0, 0, 0, 0, 0, 0, 0, 0, 537, 482, 484, 544, 545, 525, 510, 520, 0, 0, 0, 0, 0, 0, 0, 0, 521, 497, 523, 528, 534, 513, 500, 491, 0, 0, 0, 0, 0, 0, 0, 0, 462, 510, 538, 507, 488, 481, 507, 488, 23, 14, 4, 0, 0, 0, 0, 0, 513, 536, 513, 498, 480, 507, 552, 497, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4228, 4085, 4034, 4046, 4147, 4107, 4022, 4099] avg=4096.0000 max=4228 max_rank=0 min=4022 min_rank=6 imbalance_ratio=1.032227 max_load_reduction=28540 max_load_reduction_pct=0.870972 moved_to_echo=16375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[490, 539, 525, 541, 547, 488, 466, 545, 8, 3, 0, 0, 0, 0, 0, 0, 527, 507, 526, 478, 485, 503, 505, 511, 0, 0, 0, 0, 0, 0, 0, 0, 535, 515, 488, 495, 513, 525, 473, 541, 0, 0, 0, 0, 0, 0, 0, 0, 500, 484, 482, 533, 530, 539, 502, 447, 0, 0, 0, 0, 0, 0, 0, 0, 482, 500, 502, 502, 513, 512, 547, 513, 0, 0, 0, 0, 0, 0, 0, 0, 520, 535, 522, 514, 493, 512, 510, 540, 0, 0, 0, 0, 0, 0, 0, 0, 505, 529, 548, 485, 507, 551, 468, 520, 24, 13, 4, 0, 0, 0, 0, 0, 507, 529, 568, 490, 518, 499, 515, 471, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4152, 4042, 4085, 4017, 4071, 4146, 4154, 4101] avg=4096.0000 max=4154 max_rank=6 min=4017 min_rank=3 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16472 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[483, 551, 489, 488, 508, 473, 497, 526, 7, 3, 0, 0, 0, 0, 0, 0, 479, 518, 532, 576, 493, 514, 500, 480, 0, 0, 0, 0, 0, 0, 0, 0, 577, 485, 538, 548, 555, 529, 535, 490, 0, 0, 0, 0, 0, 0, 0, 0, 518, 500, 493, 508, 537, 530, 515, 527, 0, 0, 0, 0, 0, 0, 0, 0, 537, 525, 468, 511, 545, 506, 499, 500, 0, 0, 0, 0, 0, 0, 0, 0, 478, 477, 482, 518, 529, 505, 510, 503, 0, 0, 0, 0, 0, 0, 0, 0, 519, 485, 516, 529, 500, 495, 506, 515, 22, 13, 4, 0, 0, 0, 0, 0, 524, 486, 493, 527, 534, 485, 524, 492, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4025, 4092, 4257, 4128, 4091, 4002, 4104, 4069] avg=4096.0000 max=4257 max_rank=2 min=4002 min_rank=5 imbalance_ratio=1.039307 max_load_reduction=28511 max_load_reduction_pct=0.870087 moved_to_echo=16266 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[525, 514, 494, 476, 502, 527, 488, 521, 8, 2, 0, 0, 0, 0, 0, 0, 516, 512, 507, 499, 511, 522, 466, 508, 0, 0, 0, 0, 0, 0, 0, 0, 528, 531, 479, 580, 529, 475, 516, 507, 0, 0, 0, 0, 0, 0, 0, 0, 516, 502, 541, 521, 502, 486, 518, 497, 0, 0, 0, 0, 0, 0, 0, 0, 490, 510, 505, 559, 510, 517, 494, 503, 0, 0, 0, 0, 0, 0, 0, 0, 557, 498, 539, 510, 520, 501, 494, 473, 0, 0, 0, 0, 0, 0, 0, 0, 542, 519, 518, 508, 520, 524, 496, 499, 24, 12, 4, 0, 0, 0, 0, 0, 509, 493, 503, 535, 491, 512, 509, 540, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4057, 4041, 4145, 4083, 4088, 4092, 4166, 4096] avg=4096.0000 max=4166 max_rank=6 min=4041 min_rank=1 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16442 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[518, 495, 493, 514, 535, 482, 514, 520, 8, 3, 0, 0, 0, 0, 0, 0, 528, 528, 510, 519, 496, 534, 500, 511, 0, 0, 0, 0, 0, 0, 0, 0, 531, 518, 495, 483, 490, 541, 521, 503, 0, 0, 0, 0, 0, 0, 0, 0, 510, 532, 521, 531, 506, 509, 516, 500, 0, 0, 0, 0, 0, 0, 0, 0, 529, 446, 524, 504, 538, 511, 503, 531, 0, 0, 0, 0, 0, 0, 0, 0, 521, 501, 512, 552, 506, 503, 513, 491, 0, 0, 0, 0, 0, 0, 0, 0, 514, 497, 497, 484, 504, 485, 465, 514, 25, 13, 4, 0, 0, 0, 0, 0, 526, 535, 557, 503, 555, 510, 469, 507, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4126, 4082, 4125, 4086, 4099, 4002, 4166] avg=4096.0000 max=4166 max_rank=7 min=4002 min_rank=6 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[506, 514, 485, 523, 513, 510, 526, 500, 11, 6, 0, 0, 0, 0, 0, 0, 529, 514, 537, 532, 481, 518, 535, 520, 0, 0, 0, 0, 0, 0, 0, 0, 500, 498, 515, 464, 505, 465, 529, 521, 0, 0, 0, 0, 0, 0, 0, 0, 514, 520, 523, 517, 498, 536, 504, 525, 4, 0, 0, 0, 0, 0, 0, 0, 514, 512, 493, 510, 519, 537, 540, 531, 0, 0, 0, 0, 0, 0, 0, 0, 503, 511, 532, 515, 523, 511, 520, 496, 0, 0, 0, 0, 0, 0, 0, 0, 500, 543, 498, 499, 490, 494, 513, 516, 28, 19, 8, 0, 0, 0, 0, 0, 497, 501, 499, 501, 480, 530, 473, 507, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4166, 3997, 4141, 4156, 4111, 4108, 3995] avg=4096.0000 max=4166 max_rank=1 min=3995 min_rank=7 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16370 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[505, 523, 511, 503, 525, 518, 496, 536, 8, 3, 0, 0, 0, 0, 0, 0, 532, 496, 471, 494, 518, 519, 525, 531, 0, 0, 0, 0, 0, 0, 0, 0, 558, 480, 501, 504, 518, 509, 514, 507, 0, 0, 0, 0, 0, 0, 0, 0, 520, 509, 498, 484, 519, 535, 515, 532, 0, 0, 0, 0, 0, 0, 0, 0, 531, 506, 508, 505, 520, 537, 480, 500, 0, 0, 0, 0, 0, 0, 0, 0, 500, 546, 531, 514, 482, 518, 474, 512, 0, 0, 0, 0, 0, 0, 0, 0, 508, 514, 496, 537, 496, 509, 509, 526, 24, 14, 4, 0, 0, 0, 0, 0, 522, 489, 486, 508, 515, 488, 523, 515, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4086, 4091, 4112, 4087, 4077, 4137, 4050] avg=4096.0000 max=4137 max_rank=6 min=4050 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16351 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4352, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 3968, 4096, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 3968, 4224, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[537, 540, 508, 488, 521, 501, 492, 512, 512, 507, 535, 533, 525, 476, 525, 517, 496, 514, 521, 505, 472, 515, 510, 508, 546, 505, 495, 530, 486, 520, 510, 528, 518, 513, 520, 532, 501, 509, 523, 491, 494, 539, 505, 532, 521, 514, 502, 518, 515, 511, 517, 526, 506, 500, 486, 513, 527, 486, 492, 471, 574, 512, 478, 532] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[528, 504, 528, 509, 482, 520, 489, 524, 503, 567, 499, 460, 517, 498, 529, 486, 511, 510, 506, 524, 555, 545, 487, 527, 496, 511, 473, 502, 532, 506, 512, 512, 482, 501, 519, 506, 529, 497, 517, 504, 498, 527, 492, 507, 553, 544, 529, 509, 500, 514, 518, 505, 486, 531, 522, 531, 513, 528, 519, 530, 509, 517, 480, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[512, 535, 538, 548, 519, 519, 503, 486, 501, 475, 552, 506, 505, 539, 519, 532, 481, 498, 491, 507, 521, 506, 481, 513, 525, 474, 527, 530, 520, 543, 547, 517, 499, 489, 533, 547, 508, 528, 515, 524, 494, 465, 497, 531, 483, 532, 521, 534, 488, 518, 496, 523, 532, 495, 514, 510, 498, 498, 500, 485, 531, 500, 513, 497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[518, 529, 509, 461, 507, 504, 486, 469, 515, 553, 491, 524, 498, 496, 490, 490, 505, 548, 499, 500, 496, 496, 528, 482, 558, 513, 503, 514, 546, 539, 523, 521, 551, 490, 526, 526, 501, 548, 530, 515, 489, 506, 559, 543, 519, 508, 483, 490, 509, 493, 523, 506, 514, 519, 529, 510, 517, 505, 510, 485, 501, 514, 548, 490] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[477, 513, 496, 518, 514, 520, 483, 511, 508, 500, 529, 531, 543, 542, 517, 504, 524, 521, 470, 530, 555, 490, 521, 461, 519, 521, 506, 509, 513, 513, 539, 475, 508, 517, 492, 507, 488, 490, 563, 542, 567, 498, 537, 516, 524, 524, 520, 531, 494, 518, 474, 524, 507, 539, 501, 512, 543, 456, 509, 504, 492, 494, 506, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[508, 532, 464, 515, 485, 509, 528, 518, 512, 528, 537, 567, 534, 523, 521, 487, 524, 515, 521, 501, 475, 488, 478, 513, 513, 511, 505, 516, 478, 550, 517, 534, 487, 484, 510, 515, 514, 517, 513, 484, 537, 514, 493, 503, 528, 478, 537, 553, 469, 539, 532, 496, 529, 517, 532, 522, 499, 534, 540, 514, 454, 473, 519, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[538, 486, 489, 539, 493, 547, 532, 508, 518, 524, 515, 540, 531, 530, 528, 497, 529, 496, 522, 503, 480, 475, 524, 485, 526, 527, 487, 539, 491, 470, 539, 518, 523, 497, 508, 506, 520, 486, 495, 496, 542, 485, 504, 467, 494, 495, 536, 495, 564, 529, 548, 527, 500, 548, 503, 513, 494, 474, 489, 547, 510, 541, 505, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[542, 489, 535, 515, 485, 490, 484, 473, 544, 505, 489, 497, 503, 480, 535, 517, 534, 508, 516, 491, 515, 536, 508, 527, 531, 487, 530, 499, 500, 547, 503, 533, 566, 475, 478, 539, 526, 497, 515, 499, 506, 502, 507, 503, 499, 500, 515, 493, 534, 530, 532, 465, 530, 524, 509, 504, 527, 542, 528, 512, 505, 526, 495, 537] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[518, 529, 509, 461, 507, 504, 486, 469, 18, 6, 0, 0, 0, 0, 0, 0, 515, 553, 491, 524, 498, 496, 462, 490, 0, 0, 0, 0, 0, 0, 0, 0, 505, 548, 499, 500, 496, 496, 528, 482, 28, 4, 0, 0, 0, 0, 0, 0, 527, 513, 503, 514, 546, 539, 523, 521, 0, 0, 0, 0, 0, 0, 0, 0, 551, 490, 526, 526, 501, 548, 530, 515, 2, 0, 0, 0, 0, 0, 0, 0, 489, 506, 559, 543, 519, 508, 475, 490, 0, 0, 0, 0, 0, 0, 0, 0, 509, 493, 523, 506, 514, 501, 529, 510, 0, 0, 0, 0, 0, 0, 0, 0, 517, 505, 510, 485, 501, 514, 548, 490, 27, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4007, 4029, 4086, 4186, 4189, 4089, 4085, 4097] avg=4096.0000 max=4189 max_rank=4 min=4007 min_rank=0 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16460 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[537, 540, 508, 488, 521, 501, 492, 512, 18, 6, 0, 0, 0, 0, 0, 0, 512, 507, 535, 533, 525, 476, 495, 517, 0, 0, 0, 0, 0, 0, 0, 0, 496, 514, 521, 505, 472, 515, 510, 508, 30, 4, 0, 0, 0, 0, 0, 0, 515, 505, 495, 530, 486, 520, 510, 528, 0, 0, 0, 0, 0, 0, 0, 0, 518, 513, 520, 532, 501, 509, 523, 491, 2, 0, 0, 0, 0, 0, 0, 0, 494, 539, 505, 532, 521, 514, 494, 518, 0, 0, 0, 0, 0, 0, 0, 0, 515, 511, 517, 526, 506, 482, 486, 513, 0, 0, 0, 0, 0, 0, 0, 0, 527, 486, 492, 471, 574, 512, 478, 532, 27, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4123, 4100, 4075, 4089, 4109, 4117, 4056, 4099] avg=4096.0000 max=4123 max_rank=0 min=4056 min_rank=6 imbalance_ratio=1.006592 max_load_reduction=28645 max_load_reduction_pct=0.874176 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[528, 504, 528, 509, 482, 520, 489, 524, 19, 6, 0, 0, 0, 0, 0, 0, 503, 567, 499, 460, 517, 498, 499, 486, 0, 0, 0, 0, 0, 0, 0, 0, 511, 510, 506, 524, 555, 545, 487, 527, 30, 3, 0, 0, 0, 0, 0, 0, 469, 511, 473, 502, 532, 506, 512, 512, 0, 0, 0, 0, 0, 0, 0, 0, 482, 501, 519, 506, 529, 497, 517, 504, 2, 0, 0, 0, 0, 0, 0, 0, 498, 527, 492, 507, 553, 544, 521, 509, 0, 0, 0, 0, 0, 0, 0, 0, 500, 514, 518, 505, 486, 512, 522, 531, 0, 0, 0, 0, 0, 0, 0, 0, 513, 528, 519, 530, 509, 517, 480, 499, 24, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4029, 4198, 4017, 4057, 4151, 4088, 4119] avg=4096.0000 max=4198 max_rank=2 min=4017 min_rank=3 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16415 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[512, 535, 538, 548, 519, 519, 503, 486, 22, 12, 0, 0, 0, 0, 0, 0, 501, 475, 552, 506, 505, 539, 485, 532, 0, 0, 0, 0, 0, 0, 0, 0, 481, 498, 491, 507, 521, 506, 481, 513, 34, 8, 0, 0, 0, 0, 0, 0, 486, 474, 527, 530, 520, 543, 547, 517, 0, 0, 0, 0, 0, 0, 0, 0, 499, 489, 533, 547, 508, 528, 515, 524, 5, 0, 0, 0, 0, 0, 0, 0, 494, 465, 497, 531, 483, 532, 504, 534, 0, 0, 0, 0, 0, 0, 0, 0, 488, 518, 496, 523, 532, 468, 514, 510, 0, 0, 0, 0, 0, 0, 0, 0, 498, 498, 500, 485, 531, 500, 513, 497, 31, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4194, 4095, 4040, 4144, 4148, 4040, 4049, 4058] avg=4096.0000 max=4194 max_rank=0 min=4040 min_rank=2 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16295 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[538, 486, 489, 539, 493, 547, 532, 508, 19, 6, 0, 0, 0, 0, 0, 0, 518, 524, 515, 540, 531, 530, 498, 497, 0, 0, 0, 0, 0, 0, 0, 0, 529, 496, 522, 503, 480, 475, 524, 485, 30, 3, 0, 0, 0, 0, 0, 0, 497, 527, 487, 539, 491, 470, 539, 518, 0, 0, 0, 0, 0, 0, 0, 0, 523, 497, 508, 506, 520, 486, 495, 496, 2, 0, 0, 0, 0, 0, 0, 0, 542, 485, 504, 467, 494, 495, 528, 495, 0, 0, 0, 0, 0, 0, 0, 0, 564, 529, 548, 527, 500, 529, 503, 513, 0, 0, 0, 0, 0, 0, 0, 0, 494, 474, 489, 547, 510, 541, 505, 501, 26, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4157, 4153, 4047, 4068, 4033, 4010, 4213, 4087] avg=4096.0000 max=4213 max_rank=6 min=4010 min_rank=5 imbalance_ratio=1.028564 max_load_reduction=28555 max_load_reduction_pct=0.871429 moved_to_echo=16343 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[477, 513, 496, 518, 514, 520, 483, 511, 19, 6, 0, 0, 0, 0, 0, 0, 508, 500, 529, 531, 543, 542, 487, 504, 0, 0, 0, 0, 0, 0, 0, 0, 524, 521, 470, 530, 555, 490, 521, 461, 30, 3, 0, 0, 0, 0, 0, 0, 491, 521, 506, 509, 513, 513, 539, 475, 0, 0, 0, 0, 0, 0, 0, 0, 508, 517, 492, 507, 488, 490, 563, 542, 2, 0, 0, 0, 0, 0, 0, 0, 567, 498, 537, 516, 524, 524, 512, 531, 0, 0, 0, 0, 0, 0, 0, 0, 494, 518, 474, 524, 507, 520, 501, 512, 0, 0, 0, 0, 0, 0, 0, 0, 543, 456, 509, 504, 492, 494, 506, 498, 25, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4057, 4144, 4105, 4067, 4109, 4209, 4050, 4027] avg=4096.0000 max=4209 max_rank=5 min=4027 min_rank=7 imbalance_ratio=1.027588 max_load_reduction=28559 max_load_reduction_pct=0.871552 moved_to_echo=16395 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[508, 532, 464, 515, 485, 509, 528, 518, 18, 6, 0, 0, 0, 0, 0, 0, 512, 528, 537, 567, 534, 523, 491, 487, 0, 0, 0, 0, 0, 0, 0, 0, 524, 515, 521, 501, 475, 488, 478, 513, 30, 3, 0, 0, 0, 0, 0, 0, 485, 511, 505, 516, 478, 550, 517, 534, 0, 0, 0, 0, 0, 0, 0, 0, 487, 484, 510, 515, 514, 517, 513, 484, 2, 0, 0, 0, 0, 0, 0, 0, 537, 514, 493, 503, 528, 478, 529, 553, 0, 0, 0, 0, 0, 0, 0, 0, 469, 539, 532, 496, 529, 499, 532, 522, 0, 0, 0, 0, 0, 0, 0, 0, 499, 534, 540, 514, 454, 473, 519, 525, 25, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4179, 4048, 4096, 4026, 4135, 4118, 4083] avg=4096.0000 max=4179 max_rank=1 min=4026 min_rank=4 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16362 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[542, 489, 535, 515, 485, 490, 484, 473, 19, 6, 0, 0, 0, 0, 0, 0, 544, 505, 489, 497, 503, 480, 504, 517, 0, 0, 0, 0, 0, 0, 0, 0, 534, 508, 516, 491, 515, 536, 508, 527, 31, 3, 0, 0, 0, 0, 0, 0, 502, 487, 530, 499, 500, 547, 503, 533, 0, 0, 0, 0, 0, 0, 0, 0, 566, 475, 478, 539, 526, 497, 515, 499, 2, 0, 0, 0, 0, 0, 0, 0, 506, 502, 507, 503, 499, 500, 507, 493, 0, 0, 0, 0, 0, 0, 0, 0, 534, 530, 532, 465, 530, 505, 509, 504, 0, 0, 0, 0, 0, 0, 0, 0, 527, 542, 528, 512, 505, 526, 495, 537, 26, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4038, 4039, 4169, 4101, 4097, 4017, 4109, 4198] avg=4096.0000 max=4198 max_rank=7 min=4017 min_rank=5 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16421 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 3968, 4096, 4224, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 3968, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[495, 524, 499, 528, 530, 496, 509, 489, 524, 510, 514, 499, 484, 534, 517, 486, 489, 514, 556, 497, 521, 513, 531, 511, 499, 533, 510, 499, 523, 521, 512, 521, 493, 501, 521, 505, 521, 529, 518, 518, 520, 530, 533, 504, 483, 504, 525, 493, 540, 529, 498, 542, 488, 473, 516, 513, 492, 497, 540, 522, 514, 502, 515, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[507, 507, 517, 538, 521, 465, 523, 515, 513, 520, 509, 484, 516, 517, 534, 527, 536, 541, 520, 507, 504, 528, 514, 514, 546, 511, 489, 527, 552, 475, 510, 488, 495, 517, 512, 511, 573, 501, 544, 496, 514, 480, 510, 512, 550, 512, 503, 537, 493, 542, 525, 489, 462, 508, 498, 493, 483, 542, 487, 494, 491, 512, 508, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[507, 479, 530, 513, 494, 503, 496, 515, 498, 497, 532, 550, 500, 501, 514, 540, 527, 493, 477, 541, 496, 501, 509, 475, 537, 486, 511, 552, 500, 493, 514, 502, 530, 526, 525, 505, 538, 559, 560, 516, 553, 512, 481, 489, 537, 528, 556, 517, 464, 495, 491, 547, 471, 497, 522, 497, 522, 494, 528, 486, 484, 533, 522, 500] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[485, 538, 502, 511, 514, 461, 506, 502, 533, 560, 504, 520, 506, 496, 506, 510, 521, 508, 446, 489, 546, 499, 501, 532, 501, 517, 495, 515, 475, 541, 509, 536, 532, 504, 498, 486, 518, 505, 533, 474, 518, 557, 560, 515, 524, 496, 533, 517, 522, 458, 493, 513, 519, 531, 507, 534, 500, 517, 539, 499, 545, 531, 519, 486] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[546, 534, 495, 506, 525, 540, 532, 508, 490, 522, 551, 533, 489, 548, 547, 481, 518, 483, 503, 474, 510, 526, 486, 530, 519, 535, 497, 480, 560, 532, 524, 538, 499, 507, 504, 513, 508, 489, 514, 511, 505, 493, 478, 452, 533, 500, 520, 518, 532, 514, 463, 549, 544, 529, 505, 511, 520, 553, 520, 487, 507, 505, 475, 448] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[516, 502, 506, 559, 564, 532, 504, 529, 505, 553, 492, 467, 498, 520, 483, 531, 524, 534, 530, 540, 521, 488, 496, 507, 505, 496, 484, 500, 513, 503, 504, 500, 535, 502, 486, 492, 495, 506, 521, 498, 524, 507, 555, 491, 542, 516, 493, 500, 481, 489, 471, 516, 502, 525, 503, 535, 504, 490, 539, 530, 498, 527, 562, 527] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[480, 516, 482, 529, 509, 545, 513, 534, 512, 444, 542, 505, 510, 489, 519, 494, 467, 498, 534, 512, 485, 531, 484, 518, 554, 533, 482, 516, 561, 519, 518, 541, 526, 521, 480, 529, 504, 509, 532, 517, 512, 554, 513, 510, 510, 516, 505, 509, 528, 495, 558, 488, 508, 516, 499, 503, 526, 516, 520, 505, 504, 504, 484, 491] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[499, 540, 527, 521, 523, 509, 487, 549, 503, 500, 519, 494, 500, 503, 513, 507, 524, 479, 589, 489, 478, 510, 518, 470, 526, 502, 515, 522, 530, 519, 487, 525, 509, 514, 531, 520, 454, 470, 529, 470, 499, 511, 519, 526, 505, 569, 491, 493, 531, 525, 523, 514, 534, 514, 537, 528, 543, 501, 512, 534, 499, 493, 478, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[495, 524, 499, 515, 530, 496, 509, 489, 0, 0, 0, 0, 0, 0, 0, 0, 524, 510, 514, 499, 484, 534, 517, 486, 6, 0, 0, 0, 0, 0, 0, 0, 489, 514, 556, 497, 521, 513, 531, 511, 17, 0, 0, 0, 0, 0, 0, 0, 499, 533, 510, 499, 502, 521, 512, 521, 0, 0, 0, 0, 0, 0, 0, 0, 493, 501, 521, 505, 521, 529, 518, 518, 1, 0, 0, 0, 0, 0, 0, 0, 520, 530, 533, 504, 461, 504, 525, 493, 0, 0, 0, 0, 0, 0, 0, 0, 540, 529, 498, 542, 488, 473, 516, 513, 9, 6, 0, 0, 0, 0, 0, 0, 492, 497, 540, 522, 514, 502, 515, 501, 12, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4057, 4074, 4149, 4097, 4107, 4070, 4114, 4100] avg=4096.0000 max=4149 max_rank=2 min=4057 min_rank=0 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[507, 507, 517, 525, 521, 465, 523, 515, 0, 0, 0, 0, 0, 0, 0, 0, 513, 520, 509, 484, 516, 517, 534, 527, 6, 0, 0, 0, 0, 0, 0, 0, 536, 541, 520, 507, 504, 528, 514, 514, 20, 0, 0, 0, 0, 0, 0, 0, 546, 511, 489, 527, 531, 475, 510, 488, 0, 0, 0, 0, 0, 0, 0, 0, 495, 517, 512, 511, 573, 501, 544, 496, 1, 0, 0, 0, 0, 0, 0, 0, 514, 480, 510, 512, 524, 512, 503, 537, 0, 0, 0, 0, 0, 0, 0, 0, 493, 542, 525, 489, 462, 508, 498, 493, 9, 6, 0, 0, 0, 0, 0, 0, 483, 542, 487, 494, 491, 512, 508, 499, 12, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4126, 4184, 4077, 4150, 4092, 4025, 4034] avg=4096.0000 max=4184 max_rank=2 min=4025 min_rank=6 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16301 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[480, 516, 482, 516, 509, 545, 513, 534, 0, 0, 0, 0, 0, 0, 0, 0, 512, 444, 542, 505, 510, 489, 519, 494, 6, 0, 0, 0, 0, 0, 0, 0, 467, 498, 534, 512, 485, 531, 484, 518, 18, 0, 0, 0, 0, 0, 0, 0, 554, 533, 482, 516, 539, 519, 518, 541, 0, 0, 0, 0, 0, 0, 0, 0, 526, 521, 480, 529, 504, 509, 532, 517, 1, 0, 0, 0, 0, 0, 0, 0, 512, 554, 513, 510, 487, 516, 505, 509, 0, 0, 0, 0, 0, 0, 0, 0, 528, 495, 558, 488, 508, 516, 499, 503, 9, 6, 0, 0, 0, 0, 0, 0, 526, 516, 520, 505, 504, 504, 484, 491, 13, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4021, 4047, 4202, 4119, 4106, 4110, 4068] avg=4096.0000 max=4202 max_rank=3 min=4021 min_rank=1 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16403 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[485, 538, 502, 489, 514, 461, 506, 502, 0, 0, 0, 0, 0, 0, 0, 0, 533, 560, 504, 520, 506, 496, 506, 510, 9, 0, 0, 0, 0, 0, 0, 0, 521, 508, 446, 489, 546, 499, 501, 532, 24, 0, 0, 0, 0, 0, 0, 0, 501, 517, 495, 515, 449, 541, 509, 536, 0, 0, 0, 0, 0, 0, 0, 0, 532, 504, 498, 486, 518, 505, 533, 474, 2, 0, 0, 0, 0, 0, 0, 0, 518, 557, 560, 515, 490, 496, 533, 517, 0, 0, 0, 0, 0, 0, 0, 0, 522, 458, 493, 513, 519, 531, 507, 534, 12, 11, 0, 0, 0, 0, 0, 0, 500, 517, 539, 499, 545, 531, 519, 486, 14, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3997, 4144, 4066, 4063, 4052, 4186, 4100, 4160] avg=4096.0000 max=4186 max_rank=5 min=3997 min_rank=0 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16498 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[507, 479, 530, 501, 494, 503, 496, 515, 0, 0, 0, 0, 0, 0, 0, 0, 498, 497, 532, 550, 500, 501, 514, 540, 5, 0, 0, 0, 0, 0, 0, 0, 527, 493, 477, 541, 496, 501, 509, 475, 19, 0, 0, 0, 0, 0, 0, 0, 537, 486, 511, 552, 481, 493, 514, 502, 0, 0, 0, 0, 0, 0, 0, 0, 530, 526, 525, 505, 538, 559, 560, 516, 1, 0, 0, 0, 0, 0, 0, 0, 553, 512, 481, 489, 512, 528, 556, 517, 0, 0, 0, 0, 0, 0, 0, 0, 464, 495, 491, 547, 471, 497, 522, 497, 8, 6, 0, 0, 0, 0, 0, 0, 522, 494, 528, 486, 484, 533, 522, 500, 11, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4025, 4137, 4038, 4076, 4260, 4148, 3998, 4086] avg=4096.0000 max=4260 max_rank=4 min=3998 min_rank=6 imbalance_ratio=1.040039 max_load_reduction=28508 max_load_reduction_pct=0.869995 moved_to_echo=16492 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[516, 502, 506, 545, 564, 532, 504, 529, 0, 0, 0, 0, 0, 0, 0, 0, 505, 553, 492, 467, 498, 520, 483, 531, 6, 0, 0, 0, 0, 0, 0, 0, 524, 534, 530, 540, 521, 488, 496, 507, 20, 0, 0, 0, 0, 0, 0, 0, 505, 496, 484, 500, 493, 503, 504, 500, 0, 0, 0, 0, 0, 0, 0, 0, 535, 502, 486, 492, 495, 506, 521, 498, 1, 0, 0, 0, 0, 0, 0, 0, 524, 507, 555, 491, 516, 516, 493, 500, 0, 0, 0, 0, 0, 0, 0, 0, 481, 489, 471, 516, 502, 525, 503, 535, 9, 7, 0, 0, 0, 0, 0, 0, 504, 490, 539, 530, 498, 527, 562, 527, 11, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4198, 4055, 4160, 3985, 4036, 4102, 4038, 4194] avg=4096.0000 max=4198 max_rank=0 min=3985 min_rank=3 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16370 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[546, 534, 495, 494, 525, 540, 532, 508, 0, 0, 0, 0, 0, 0, 0, 0, 490, 522, 551, 533, 489, 548, 547, 481, 5, 0, 0, 0, 0, 0, 0, 0, 518, 483, 503, 474, 510, 526, 486, 530, 19, 0, 0, 0, 0, 0, 0, 0, 519, 535, 497, 480, 538, 532, 524, 538, 0, 0, 0, 0, 0, 0, 0, 0, 499, 507, 504, 513, 508, 489, 514, 511, 1, 0, 0, 0, 0, 0, 0, 0, 505, 493, 478, 452, 508, 500, 520, 518, 0, 0, 0, 0, 0, 0, 0, 0, 532, 514, 463, 549, 544, 529, 505, 511, 9, 6, 0, 0, 0, 0, 0, 0, 520, 553, 520, 487, 507, 505, 475, 448, 13, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4174, 4166, 4049, 4163, 4046, 3974, 4162, 4034] avg=4096.0000 max=4174 max_rank=0 min=3974 min_rank=5 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16216 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[499, 540, 527, 508, 523, 509, 487, 549, 0, 0, 0, 0, 0, 0, 0, 0, 503, 500, 519, 494, 500, 503, 513, 507, 6, 0, 0, 0, 0, 0, 0, 0, 524, 479, 589, 489, 478, 510, 518, 470, 18, 0, 0, 0, 0, 0, 0, 0, 526, 502, 515, 522, 509, 519, 487, 525, 0, 0, 0, 0, 0, 0, 0, 0, 509, 514, 531, 520, 454, 470, 529, 470, 1, 0, 0, 0, 0, 0, 0, 0, 499, 511, 519, 526, 482, 569, 491, 493, 0, 0, 0, 0, 0, 0, 0, 0, 531, 525, 523, 514, 534, 514, 537, 528, 9, 6, 0, 0, 0, 0, 0, 0, 543, 501, 512, 534, 499, 493, 478, 515, 12, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4142, 4045, 4075, 4105, 3998, 4090, 4221, 4092] avg=4096.0000 max=4221 max_rank=6 min=3998 min_rank=4 imbalance_ratio=1.030518 max_load_reduction=28547 max_load_reduction_pct=0.871185 moved_to_echo=16401 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4352, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4096, 3968, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[512, 537, 548, 507, 519, 539, 494, 499, 490, 529, 512, 527, 512, 487, 492, 504, 509, 559, 486, 531, 510, 508, 509, 524, 514, 511, 531, 503, 554, 504, 489, 502, 497, 502, 547, 524, 530, 536, 511, 528, 479, 476, 522, 488, 510, 496, 548, 497, 484, 524, 528, 487, 509, 528, 500, 502, 544, 500, 506, 493, 490, 521, 483, 526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[507, 508, 498, 486, 499, 531, 514, 505, 500, 533, 566, 477, 489, 499, 528, 519, 546, 524, 501, 511, 535, 522, 525, 480, 522, 516, 475, 520, 513, 534, 523, 523, 465, 533, 502, 549, 547, 513, 487, 498, 496, 493, 539, 492, 552, 504, 501, 524, 488, 487, 527, 501, 527, 500, 543, 503, 543, 488, 486, 487, 520, 458, 545, 541] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[506, 523, 533, 499, 533, 530, 475, 543, 496, 553, 515, 476, 517, 505, 568, 537, 516, 524, 510, 498, 521, 513, 510, 540, 514, 527, 525, 508, 552, 514, 481, 508, 533, 475, 508, 506, 509, 513, 506, 525, 492, 533, 521, 524, 509, 546, 532, 493, 483, 515, 502, 463, 503, 527, 496, 497, 485, 474, 489, 488, 549, 505, 500, 497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[523, 483, 526, 485, 487, 538, 498, 514, 517, 487, 522, 541, 544, 514, 540, 493, 533, 514, 496, 540, 494, 550, 499, 533, 500, 487, 526, 527, 525, 521, 472, 513, 497, 494, 524, 540, 495, 506, 511, 504, 489, 497, 519, 521, 519, 504, 535, 562, 499, 499, 527, 466, 534, 527, 506, 496, 523, 529, 498, 508, 527, 522, 461, 487] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[514, 513, 497, 521, 526, 552, 523, 498, 522, 483, 505, 521, 552, 500, 522, 471, 507, 512, 486, 500, 523, 558, 518, 494, 534, 509, 481, 502, 495, 521, 499, 531, 537, 525, 511, 501, 538, 520, 527, 514, 497, 476, 473, 523, 505, 542, 518, 507, 510, 506, 515, 513, 494, 516, 508, 528, 518, 515, 503, 502, 502, 503, 513, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[525, 532, 509, 478, 542, 530, 515, 525, 538, 458, 520, 540, 494, 539, 498, 546, 480, 496, 491, 509, 473, 502, 502, 510, 511, 515, 499, 563, 536, 508, 520, 536, 484, 498, 477, 520, 507, 505, 498, 492, 537, 521, 522, 507, 555, 490, 502, 522, 531, 496, 491, 519, 504, 478, 478, 507, 513, 502, 516, 546, 502, 539, 552, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[508, 543, 532, 498, 555, 521, 532, 508, 503, 479, 517, 511, 490, 508, 491, 520, 541, 524, 514, 509, 509, 493, 510, 512, 510, 513, 526, 524, 507, 537, 517, 485, 526, 542, 520, 504, 522, 483, 482, 500, 497, 491, 528, 528, 454, 512, 514, 509, 521, 473, 510, 495, 517, 528, 504, 496, 501, 569, 532, 524, 507, 472, 539, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[484, 509, 473, 525, 507, 521, 525, 513, 502, 533, 538, 522, 522, 464, 521, 459, 498, 517, 498, 534, 514, 498, 521, 486, 527, 520, 506, 489, 517, 505, 515, 499, 510, 493, 493, 513, 554, 525, 533, 510, 535, 531, 522, 519, 522, 506, 495, 503, 504, 515, 516, 508, 519, 515, 523, 519, 521, 543, 477, 523, 480, 520, 531, 528] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[512, 537, 548, 507, 519, 516, 494, 499, 0, 0, 0, 0, 0, 0, 0, 0, 490, 529, 511, 527, 512, 487, 492, 504, 0, 0, 0, 0, 0, 0, 0, 0, 509, 554, 486, 531, 510, 508, 509, 524, 0, 0, 0, 0, 0, 0, 0, 0, 514, 511, 531, 503, 538, 504, 489, 502, 0, 0, 0, 0, 0, 0, 0, 0, 497, 502, 547, 524, 529, 536, 511, 528, 0, 0, 0, 0, 0, 0, 0, 0, 479, 476, 521, 488, 510, 496, 548, 497, 0, 0, 0, 0, 0, 0, 0, 0, 484, 524, 528, 487, 509, 528, 500, 502, 23, 16, 3, 0, 0, 0, 0, 0, 544, 500, 506, 493, 490, 521, 483, 526, 2, 1, 1, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4132, 4052, 4131, 4092, 4174, 4015, 4104, 4068] avg=4096.0000 max=4174 max_rank=4 min=4015 min_rank=5 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16361 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[507, 508, 498, 486, 499, 508, 514, 505, 0, 0, 0, 0, 0, 0, 0, 0, 500, 533, 565, 477, 489, 499, 528, 519, 0, 0, 0, 0, 0, 0, 0, 0, 546, 519, 501, 511, 535, 522, 525, 480, 0, 0, 0, 0, 0, 0, 0, 0, 522, 516, 475, 520, 498, 534, 523, 523, 0, 0, 0, 0, 0, 0, 0, 0, 465, 533, 502, 549, 546, 513, 487, 498, 0, 0, 0, 0, 0, 0, 0, 0, 496, 493, 538, 492, 552, 504, 501, 524, 0, 0, 0, 0, 0, 0, 0, 0, 488, 487, 527, 501, 527, 500, 543, 503, 23, 15, 3, 0, 0, 0, 0, 0, 543, 488, 486, 487, 520, 458, 545, 541, 2, 1, 1, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4025, 4110, 4139, 4111, 4093, 4100, 4117, 4073] avg=4096.0000 max=4139 max_rank=2 min=4025 min_rank=0 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[523, 483, 526, 485, 487, 512, 498, 514, 0, 0, 0, 0, 0, 0, 0, 0, 517, 487, 519, 541, 544, 514, 540, 493, 0, 0, 0, 0, 0, 0, 0, 0, 533, 506, 496, 540, 494, 550, 499, 533, 0, 0, 0, 0, 0, 0, 0, 0, 500, 487, 526, 527, 507, 521, 472, 513, 0, 0, 0, 0, 0, 0, 0, 0, 497, 494, 524, 540, 491, 506, 511, 504, 0, 0, 0, 0, 0, 0, 0, 0, 489, 497, 517, 521, 519, 504, 535, 562, 0, 0, 0, 0, 0, 0, 0, 0, 499, 499, 527, 466, 534, 527, 506, 496, 26, 18, 5, 0, 0, 0, 0, 0, 523, 529, 498, 508, 527, 522, 461, 487, 3, 4, 3, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4028, 4155, 4151, 4053, 4067, 4144, 4103, 4067] avg=4096.0000 max=4155 max_rank=1 min=4028 min_rank=0 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[506, 523, 533, 499, 533, 507, 475, 543, 0, 0, 0, 0, 0, 0, 0, 0, 496, 553, 514, 476, 517, 505, 568, 537, 0, 0, 0, 0, 0, 0, 0, 0, 516, 519, 510, 498, 521, 513, 510, 540, 0, 0, 0, 0, 0, 0, 0, 0, 514, 527, 525, 508, 536, 514, 481, 508, 0, 0, 0, 0, 0, 0, 0, 0, 533, 475, 508, 506, 508, 513, 506, 525, 0, 0, 0, 0, 0, 0, 0, 0, 492, 533, 520, 524, 509, 546, 532, 493, 0, 0, 0, 0, 0, 0, 0, 0, 483, 515, 502, 463, 503, 527, 496, 497, 23, 16, 3, 0, 0, 0, 0, 0, 485, 474, 489, 488, 549, 505, 500, 497, 2, 1, 1, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4166, 4127, 4113, 4074, 4149, 4028, 3992] avg=4096.0000 max=4166 max_rank=1 min=3992 min_rank=7 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16243 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[514, 513, 497, 521, 526, 529, 523, 498, 0, 0, 0, 0, 0, 0, 0, 0, 522, 483, 504, 521, 552, 500, 522, 471, 0, 0, 0, 0, 0, 0, 0, 0, 507, 507, 486, 500, 523, 558, 518, 494, 0, 0, 0, 0, 0, 0, 0, 0, 534, 509, 481, 502, 481, 521, 499, 531, 0, 0, 0, 0, 0, 0, 0, 0, 537, 525, 511, 501, 537, 520, 527, 514, 0, 0, 0, 0, 0, 0, 0, 0, 497, 476, 473, 523, 505, 542, 518, 507, 0, 0, 0, 0, 0, 0, 0, 0, 510, 506, 515, 513, 494, 516, 508, 528, 23, 14, 3, 0, 0, 0, 0, 0, 518, 515, 503, 502, 502, 503, 513, 518, 2, 1, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4075, 4093, 4058, 4172, 4041, 4130, 4078] avg=4096.0000 max=4172 max_rank=4 min=4041 min_rank=5 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16421 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[525, 532, 509, 478, 542, 507, 515, 525, 0, 0, 0, 0, 0, 0, 0, 0, 538, 458, 519, 540, 494, 539, 498, 546, 0, 0, 0, 0, 0, 0, 0, 0, 480, 492, 491, 509, 473, 502, 502, 510, 0, 0, 0, 0, 0, 0, 0, 0, 511, 515, 499, 563, 521, 508, 520, 536, 0, 0, 0, 0, 0, 0, 0, 0, 484, 498, 477, 520, 506, 505, 498, 492, 0, 0, 0, 0, 0, 0, 0, 0, 537, 521, 521, 507, 555, 490, 502, 522, 0, 0, 0, 0, 0, 0, 0, 0, 531, 496, 491, 519, 504, 478, 478, 507, 23, 15, 2, 0, 0, 0, 0, 0, 513, 502, 516, 546, 502, 539, 552, 517, 2, 1, 1, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4133, 4132, 3959, 4173, 3980, 4155, 4044, 4192] avg=4096.0000 max=4192 max_rank=7 min=3959 min_rank=2 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16371 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[508, 543, 532, 498, 555, 499, 532, 508, 0, 0, 0, 0, 0, 0, 0, 0, 503, 479, 516, 511, 490, 508, 491, 520, 0, 0, 0, 0, 0, 0, 0, 0, 541, 519, 514, 509, 509, 493, 510, 512, 0, 0, 0, 0, 0, 0, 0, 0, 510, 513, 526, 524, 493, 537, 517, 485, 0, 0, 0, 0, 0, 0, 0, 0, 526, 542, 520, 504, 521, 483, 482, 500, 0, 0, 0, 0, 0, 0, 0, 0, 497, 491, 527, 528, 454, 512, 514, 509, 0, 0, 0, 0, 0, 0, 0, 0, 521, 473, 510, 495, 517, 528, 504, 496, 22, 14, 3, 0, 0, 0, 0, 0, 501, 569, 532, 524, 507, 472, 539, 521, 2, 1, 1, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4175, 4018, 4107, 4105, 4078, 4032, 4083, 4170] avg=4096.0000 max=4175 max_rank=0 min=4018 min_rank=1 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[484, 509, 473, 525, 507, 499, 525, 513, 0, 0, 0, 0, 0, 0, 0, 0, 502, 533, 537, 522, 522, 464, 521, 459, 0, 0, 0, 0, 0, 0, 0, 0, 498, 512, 498, 534, 514, 498, 521, 486, 0, 0, 0, 0, 0, 0, 0, 0, 527, 520, 506, 489, 502, 505, 515, 499, 0, 0, 0, 0, 0, 0, 0, 0, 510, 493, 493, 513, 553, 525, 533, 510, 0, 0, 0, 0, 0, 0, 0, 0, 535, 531, 521, 519, 522, 506, 495, 503, 0, 0, 0, 0, 0, 0, 0, 0, 504, 515, 516, 508, 519, 515, 523, 519, 22, 15, 3, 0, 0, 0, 0, 0, 521, 543, 477, 523, 480, 520, 531, 528, 2, 1, 1, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4035, 4060, 4061, 4063, 4130, 4132, 4159, 4128] avg=4096.0000 max=4159 max_rank=6 min=4035 min_rank=0 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16549 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 128, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 3968, 4224, 4224, 4096, 4096, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33792 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True + [2026-09-21 08:26:49] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 259.2 | throughput per GPU (TFLOP/s/GPU): 124.3 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 9.128730E+00 | loss scale: 1.0 | grad norm: 10.668 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[513, 493, 516, 523, 543, 501, 515, 536, 493, 520, 544, 527, 498, 489, 533, 523, 541, 532, 546, 498, 534, 508, 513, 475, 515, 482, 511, 500, 573, 510, 500, 512, 539, 501, 530, 519, 495, 554, 523, 500, 488, 545, 493, 532, 505, 480, 517, 473, 500, 518, 515, 515, 522, 502, 466, 498, 468, 533, 547, 486, 472, 467, 512, 536] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[493, 468, 513, 540, 527, 523, 478, 537, 503, 551, 556, 509, 490, 580, 472, 538, 490, 530, 527, 518, 505, 523, 503, 526, 492, 499, 529, 533, 500, 537, 483, 508, 494, 457, 526, 461, 499, 535, 504, 506, 524, 489, 502, 530, 499, 494, 507, 474, 520, 525, 503, 526, 491, 519, 511, 515, 505, 535, 531, 534, 526, 505, 525, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[480, 518, 551, 500, 518, 520, 538, 496, 518, 499, 492, 506, 484, 505, 500, 521, 499, 507, 513, 544, 528, 469, 491, 453, 524, 505, 487, 565, 563, 529, 495, 535, 492, 487, 573, 497, 506, 480, 517, 504, 498, 502, 562, 538, 488, 515, 511, 488, 521, 532, 509, 516, 512, 505, 520, 536, 528, 502, 497, 531, 544, 523, 475, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[562, 478, 505, 512, 470, 497, 501, 479, 475, 546, 492, 533, 503, 509, 486, 483, 537, 517, 546, 541, 514, 521, 477, 510, 549, 531, 524, 519, 521, 514, 509, 537, 489, 554, 539, 534, 490, 508, 520, 523, 519, 530, 522, 493, 516, 553, 519, 532, 489, 504, 515, 459, 531, 470, 508, 513, 469, 505, 534, 489, 505, 502, 512, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[506, 519, 493, 505, 504, 543, 513, 484, 516, 494, 504, 509, 499, 553, 527, 524, 510, 496, 510, 507, 511, 475, 520, 529, 500, 532, 508, 509, 537, 534, 469, 512, 537, 487, 507, 506, 539, 536, 535, 477, 538, 522, 504, 478, 498, 541, 481, 524, 523, 507, 462, 535, 483, 527, 503, 500, 518, 504, 521, 532, 520, 527, 519, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[495, 486, 517, 484, 490, 523, 570, 522, 545, 544, 497, 519, 533, 487, 514, 493, 491, 542, 505, 508, 504, 486, 526, 507, 514, 501, 563, 488, 506, 516, 502, 474, 522, 542, 541, 544, 516, 505, 488, 495, 521, 520, 494, 499, 501, 487, 504, 526, 508, 537, 489, 497, 500, 570, 500, 505, 502, 485, 530, 492, 530, 541, 520, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[506, 503, 518, 545, 507, 504, 514, 495, 526, 514, 501, 492, 526, 504, 467, 495, 537, 510, 518, 491, 493, 510, 482, 509, 517, 509, 522, 513, 529, 521, 514, 536, 529, 488, 526, 517, 505, 542, 536, 516, 566, 514, 523, 457, 514, 495, 501, 524, 506, 515, 526, 520, 512, 510, 485, 514, 503, 519, 578, 477, 488, 524, 492, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[523, 535, 511, 527, 535, 521, 540, 513, 526, 465, 512, 515, 532, 478, 515, 535, 507, 486, 517, 510, 497, 519, 573, 519, 509, 477, 555, 486, 535, 552, 521, 504, 496, 480, 485, 481, 493, 488, 497, 482, 490, 510, 497, 502, 515, 571, 482, 505, 523, 479, 558, 536, 537, 536, 491, 506, 509, 512, 502, 532, 495, 503, 516, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[493, 468, 513, 540, 527, 523, 476, 537, 0, 0, 0, 0, 0, 0, 0, 0, 503, 551, 556, 509, 490, 580, 472, 538, 2, 0, 0, 0, 0, 0, 0, 0, 490, 530, 527, 518, 505, 523, 503, 526, 2, 0, 0, 0, 0, 0, 0, 0, 492, 499, 529, 533, 469, 537, 483, 508, 0, 0, 0, 0, 0, 0, 0, 0, 494, 457, 526, 461, 499, 535, 504, 506, 8, 0, 0, 0, 0, 0, 0, 0, 524, 489, 502, 530, 499, 494, 507, 474, 14, 0, 0, 0, 0, 0, 0, 0, 520, 525, 503, 526, 491, 519, 511, 515, 9, 0, 0, 0, 0, 0, 0, 0, 505, 535, 529, 534, 526, 505, 525, 515, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4201, 4124, 4050, 3990, 4033, 4119, 4174] avg=4096.0000 max=4201 max_rank=1 min=3990 min_rank=4 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16316 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[513, 493, 516, 523, 543, 501, 513, 536, 0, 0, 0, 0, 0, 0, 0, 0, 493, 520, 544, 527, 498, 489, 533, 523, 2, 0, 0, 0, 0, 0, 0, 0, 541, 532, 546, 498, 534, 508, 513, 475, 2, 0, 0, 0, 0, 0, 0, 0, 515, 482, 511, 500, 537, 510, 500, 512, 0, 0, 0, 0, 0, 0, 0, 0, 539, 501, 530, 519, 495, 554, 523, 500, 9, 0, 0, 0, 0, 0, 0, 0, 488, 545, 493, 532, 505, 480, 517, 473, 16, 0, 0, 0, 0, 0, 0, 0, 500, 518, 515, 515, 522, 502, 466, 498, 11, 0, 0, 0, 0, 0, 0, 0, 468, 533, 545, 486, 472, 467, 512, 536, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4138, 4129, 4149, 4067, 4170, 4049, 4047, 4019] avg=4096.0000 max=4170 max_rank=4 min=4019 min_rank=7 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16285 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[480, 518, 551, 500, 518, 520, 536, 496, 0, 0, 0, 0, 0, 0, 0, 0, 518, 499, 492, 506, 484, 505, 500, 521, 2, 0, 0, 0, 0, 0, 0, 0, 499, 507, 513, 544, 528, 469, 491, 453, 2, 0, 0, 0, 0, 0, 0, 0, 524, 505, 487, 565, 528, 529, 495, 535, 0, 0, 0, 0, 0, 0, 0, 0, 492, 487, 573, 497, 506, 480, 517, 504, 9, 0, 0, 0, 0, 0, 0, 0, 498, 502, 562, 538, 488, 515, 511, 488, 16, 0, 0, 0, 0, 0, 0, 0, 521, 532, 509, 516, 512, 505, 520, 536, 10, 0, 0, 0, 0, 0, 0, 0, 528, 502, 495, 531, 544, 523, 475, 506, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4027, 4006, 4168, 4065, 4118, 4161, 4104] avg=4096.0000 max=4168 max_rank=3 min=4006 min_rank=2 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16448 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[562, 478, 505, 512, 470, 497, 499, 479, 0, 0, 0, 0, 0, 0, 0, 0, 475, 546, 492, 533, 503, 509, 486, 483, 2, 0, 0, 0, 0, 0, 0, 0, 537, 517, 546, 541, 514, 521, 477, 510, 2, 0, 0, 0, 0, 0, 0, 0, 549, 531, 524, 519, 487, 514, 509, 537, 0, 0, 0, 0, 0, 0, 0, 0, 489, 554, 539, 534, 490, 508, 520, 523, 9, 0, 0, 0, 0, 0, 0, 0, 519, 530, 522, 493, 516, 553, 519, 532, 15, 0, 0, 0, 0, 0, 0, 0, 489, 504, 515, 459, 531, 470, 508, 513, 10, 0, 0, 0, 0, 0, 0, 0, 469, 505, 532, 489, 505, 502, 512, 524, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4002, 4029, 4165, 4170, 4166, 4199, 3999, 4038] avg=4096.0000 max=4199 max_rank=5 min=3999 min_rank=6 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16402 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[495, 486, 517, 484, 490, 523, 557, 522, 0, 0, 0, 0, 0, 0, 0, 0, 545, 544, 497, 519, 533, 487, 514, 493, 9, 6, 0, 0, 0, 0, 0, 0, 491, 542, 505, 508, 504, 486, 526, 507, 8, 4, 0, 0, 0, 0, 0, 0, 514, 501, 563, 488, 456, 516, 502, 474, 0, 0, 0, 0, 0, 0, 0, 0, 522, 542, 541, 544, 516, 505, 488, 495, 12, 0, 0, 0, 0, 0, 0, 0, 521, 520, 494, 499, 501, 487, 504, 526, 20, 0, 0, 0, 0, 0, 0, 0, 508, 537, 489, 497, 500, 570, 500, 505, 12, 0, 0, 0, 0, 0, 0, 0, 502, 485, 522, 492, 530, 541, 520, 505, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4147, 4081, 4014, 4165, 4072, 4118, 4097] avg=4096.0000 max=4165 max_rank=4 min=4014 min_rank=3 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16452 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[506, 519, 493, 505, 504, 543, 511, 484, 0, 0, 0, 0, 0, 0, 0, 0, 516, 494, 504, 509, 499, 553, 527, 524, 2, 0, 0, 0, 0, 0, 0, 0, 510, 496, 510, 507, 511, 475, 520, 529, 2, 0, 0, 0, 0, 0, 0, 0, 500, 532, 508, 509, 503, 534, 469, 512, 0, 0, 0, 0, 0, 0, 0, 0, 537, 487, 507, 506, 539, 536, 535, 477, 9, 0, 0, 0, 0, 0, 0, 0, 538, 522, 504, 478, 498, 541, 481, 524, 15, 0, 0, 0, 0, 0, 0, 0, 523, 507, 462, 535, 483, 527, 503, 500, 10, 0, 0, 0, 0, 0, 0, 0, 518, 504, 519, 532, 520, 527, 519, 525, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4065, 4128, 4060, 4067, 4133, 4101, 4050, 4164] avg=4096.0000 max=4164 max_rank=7 min=4050 min_rank=6 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16448 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[506, 503, 518, 545, 507, 504, 512, 495, 0, 0, 0, 0, 0, 0, 0, 0, 526, 514, 501, 492, 526, 504, 467, 495, 2, 0, 0, 0, 0, 0, 0, 0, 537, 510, 518, 491, 493, 510, 482, 509, 3, 0, 0, 0, 0, 0, 0, 0, 517, 509, 522, 513, 495, 521, 514, 536, 0, 0, 0, 0, 0, 0, 0, 0, 529, 488, 526, 517, 505, 542, 536, 516, 9, 0, 0, 0, 0, 0, 0, 0, 566, 514, 523, 457, 514, 495, 501, 524, 15, 0, 0, 0, 0, 0, 0, 0, 506, 515, 526, 520, 512, 510, 485, 514, 10, 0, 0, 0, 0, 0, 0, 0, 503, 519, 575, 477, 488, 524, 492, 518, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4027, 4053, 4127, 4168, 4109, 4098, 4096] avg=4096.0000 max=4168 max_rank=4 min=4027 min_rank=1 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16471 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[523, 535, 511, 527, 535, 521, 538, 513, 0, 0, 0, 0, 0, 0, 0, 0, 526, 465, 512, 515, 532, 478, 515, 535, 2, 0, 0, 0, 0, 0, 0, 0, 507, 486, 517, 510, 497, 519, 573, 519, 2, 0, 0, 0, 0, 0, 0, 0, 509, 477, 555, 486, 501, 552, 521, 504, 0, 0, 0, 0, 0, 0, 0, 0, 496, 480, 485, 481, 493, 488, 497, 482, 9, 0, 0, 0, 0, 0, 0, 0, 490, 510, 497, 502, 515, 571, 482, 505, 15, 0, 0, 0, 0, 0, 0, 0, 523, 479, 558, 536, 537, 536, 491, 506, 10, 0, 0, 0, 0, 0, 0, 0, 509, 512, 500, 532, 495, 503, 516, 509, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4203, 4080, 4130, 4105, 3911, 4087, 4176, 4076] avg=4096.0000 max=4203 max_rank=0 min=3911 min_rank=4 imbalance_ratio=1.026123 max_load_reduction=28565 max_load_reduction_pct=0.871735 moved_to_echo=16250 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4352, 4096, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33792 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[535, 528, 525, 483, 529, 521, 509, 517, 519, 484, 490, 492, 511, 475, 516, 489, 517, 448, 515, 523, 503, 531, 538, 497, 499, 499, 487, 530, 536, 507, 525, 506, 546, 508, 527, 509, 508, 512, 507, 513, 484, 513, 518, 508, 488, 499, 510, 515, 522, 471, 507, 522, 506, 544, 511, 510, 512, 532, 543, 542, 517, 527, 521, 532] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[518, 530, 459, 503, 514, 523, 506, 522, 523, 516, 519, 552, 506, 503, 503, 517, 513, 508, 486, 507, 513, 538, 499, 509, 483, 508, 510, 508, 513, 499, 520, 483, 518, 514, 512, 539, 520, 516, 495, 544, 489, 459, 520, 535, 510, 462, 525, 547, 504, 483, 539, 516, 522, 529, 512, 498, 541, 492, 536, 535, 520, 459, 503, 563] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[495, 502, 484, 518, 508, 521, 546, 528, 524, 499, 498, 545, 478, 542, 515, 495, 508, 528, 492, 512, 502, 486, 519, 516, 479, 513, 524, 534, 502, 540, 482, 523, 504, 523, 529, 517, 517, 536, 505, 522, 497, 485, 515, 519, 509, 506, 505, 520, 513, 510, 506, 520, 523, 535, 527, 507, 520, 530, 512, 484, 481, 530, 515, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[541, 535, 517, 519, 472, 503, 506, 486, 534, 498, 496, 528, 520, 482, 474, 527, 493, 493, 498, 509, 535, 531, 514, 535, 488, 546, 504, 496, 539, 499, 513, 524, 520, 534, 533, 523, 536, 553, 496, 506, 517, 503, 521, 524, 479, 501, 511, 532, 556, 525, 504, 503, 482, 527, 508, 508, 479, 487, 513, 541, 513, 515, 488, 475] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[546, 510, 510, 493, 514, 519, 593, 510, 480, 519, 487, 531, 514, 496, 497, 552, 523, 529, 525, 507, 506, 490, 537, 496, 543, 465, 539, 522, 477, 541, 477, 507, 484, 550, 501, 564, 506, 516, 479, 476, 515, 497, 494, 509, 504, 493, 515, 511, 535, 528, 489, 512, 490, 518, 502, 510, 498, 534, 524, 523, 507, 503, 527, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[528, 518, 511, 513, 527, 519, 494, 504, 494, 485, 514, 500, 515, 521, 525, 506, 502, 506, 460, 538, 530, 532, 478, 528, 530, 517, 486, 525, 499, 503, 503, 506, 500, 517, 485, 525, 529, 493, 497, 476, 523, 503, 516, 498, 509, 510, 512, 544, 522, 514, 531, 528, 521, 528, 542, 532, 498, 523, 505, 518, 468, 528, 509, 547] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[520, 500, 518, 488, 508, 537, 527, 483, 508, 486, 522, 502, 508, 522, 494, 524, 512, 502, 499, 494, 477, 481, 546, 513, 546, 509, 493, 490, 498, 536, 520, 499, 502, 517, 496, 506, 512, 514, 511, 517, 477, 544, 486, 495, 466, 512, 519, 508, 535, 536, 500, 533, 510, 518, 539, 520, 506, 551, 517, 535, 529, 543, 514, 538] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[502, 515, 509, 527, 486, 557, 538, 507, 515, 486, 520, 494, 562, 534, 481, 496, 498, 519, 500, 514, 527, 545, 484, 531, 495, 477, 496, 525, 479, 520, 535, 488, 483, 524, 477, 512, 513, 514, 522, 544, 514, 499, 521, 491, 519, 496, 510, 527, 514, 496, 513, 578, 499, 493, 518, 504, 536, 514, 536, 488, 542, 501, 493, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[518, 530, 459, 503, 514, 523, 487, 522, 0, 0, 0, 0, 0, 0, 0, 0, 523, 516, 519, 552, 506, 503, 503, 517, 26, 0, 0, 0, 0, 0, 0, 0, 513, 508, 486, 507, 513, 538, 499, 509, 15, 0, 0, 0, 0, 0, 0, 0, 483, 508, 510, 508, 513, 499, 520, 483, 19, 3, 1, 0, 0, 0, 0, 0, 518, 514, 512, 523, 520, 516, 495, 544, 0, 0, 0, 0, 0, 0, 0, 0, 489, 459, 520, 535, 510, 462, 525, 547, 35, 1, 0, 0, 0, 0, 0, 0, 504, 483, 539, 481, 522, 529, 512, 498, 0, 0, 0, 0, 0, 0, 0, 0, 541, 492, 506, 535, 520, 459, 503, 563, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4056, 4165, 4088, 4047, 4142, 4083, 4068, 4119] avg=4096.0000 max=4165 max_rank=1 min=4047 min_rank=3 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16412 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[541, 535, 517, 519, 472, 503, 487, 486, 0, 0, 0, 0, 0, 0, 0, 0, 534, 498, 496, 528, 520, 482, 474, 527, 25, 0, 0, 0, 0, 0, 0, 0, 493, 493, 498, 509, 535, 531, 514, 535, 15, 0, 0, 0, 0, 0, 0, 0, 488, 546, 504, 496, 539, 499, 513, 524, 19, 2, 1, 0, 0, 0, 0, 0, 520, 534, 533, 507, 536, 553, 496, 506, 0, 0, 0, 0, 0, 0, 0, 0, 517, 503, 521, 524, 479, 501, 511, 532, 34, 1, 0, 0, 0, 0, 0, 0, 556, 525, 504, 469, 482, 527, 508, 508, 0, 0, 0, 0, 0, 0, 0, 0, 479, 487, 485, 541, 513, 515, 488, 475, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4084, 4123, 4131, 4185, 4123, 4079, 3983] avg=4096.0000 max=4185 max_rank=4 min=3983 min_rank=7 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16370 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[495, 502, 484, 518, 508, 521, 525, 528, 0, 0, 0, 0, 0, 0, 0, 0, 524, 499, 498, 545, 478, 542, 515, 495, 25, 0, 0, 0, 0, 0, 0, 0, 508, 528, 492, 512, 502, 486, 519, 516, 15, 0, 0, 0, 0, 0, 0, 0, 479, 513, 524, 534, 502, 540, 482, 523, 21, 2, 1, 0, 0, 0, 0, 0, 504, 523, 529, 501, 517, 536, 505, 522, 0, 0, 0, 0, 0, 0, 0, 0, 497, 485, 515, 519, 509, 506, 505, 520, 35, 1, 0, 0, 0, 0, 0, 0, 513, 510, 506, 485, 523, 535, 527, 507, 0, 0, 0, 0, 0, 0, 0, 0, 520, 530, 484, 484, 481, 530, 515, 488, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4121, 4078, 4121, 4137, 4092, 4106, 4032] avg=4096.0000 max=4137 max_rank=4 min=4032 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[520, 500, 518, 488, 508, 537, 507, 483, 0, 0, 0, 0, 0, 0, 0, 0, 508, 486, 522, 502, 508, 522, 494, 524, 25, 0, 0, 0, 0, 0, 0, 0, 512, 502, 499, 494, 477, 481, 546, 513, 14, 0, 0, 0, 0, 0, 0, 0, 546, 509, 493, 490, 498, 536, 520, 499, 20, 2, 1, 0, 0, 0, 0, 0, 502, 517, 496, 491, 512, 514, 511, 517, 0, 0, 0, 0, 0, 0, 0, 0, 477, 544, 486, 495, 466, 512, 519, 508, 36, 1, 0, 0, 0, 0, 0, 0, 535, 536, 500, 497, 510, 518, 539, 520, 0, 0, 0, 0, 0, 0, 0, 0, 506, 551, 489, 535, 529, 543, 514, 538, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4091, 4038, 4114, 4060, 4044, 4155, 4205] avg=4096.0000 max=4205 max_rank=7 min=4038 min_rank=2 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16464 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[546, 510, 510, 493, 514, 519, 570, 510, 0, 0, 0, 0, 0, 0, 0, 0, 480, 519, 487, 531, 514, 496, 497, 552, 26, 0, 0, 0, 0, 0, 0, 0, 523, 529, 525, 507, 506, 490, 537, 496, 16, 0, 0, 0, 0, 0, 0, 0, 543, 465, 539, 522, 477, 541, 477, 507, 23, 3, 1, 0, 0, 0, 0, 0, 484, 550, 501, 547, 506, 516, 479, 476, 0, 0, 0, 0, 0, 0, 0, 0, 515, 497, 494, 509, 504, 493, 515, 511, 35, 1, 0, 0, 0, 0, 0, 0, 535, 528, 489, 477, 490, 518, 502, 510, 0, 0, 0, 0, 0, 0, 0, 0, 498, 534, 494, 523, 507, 503, 527, 499, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4172, 4102, 4129, 4098, 4059, 4074, 4049, 4085] avg=4096.0000 max=4172 max_rank=0 min=4049 min_rank=6 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16267 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[528, 518, 511, 513, 527, 519, 475, 504, 0, 0, 0, 0, 0, 0, 0, 0, 494, 485, 514, 500, 515, 521, 525, 506, 25, 0, 0, 0, 0, 0, 0, 0, 502, 506, 460, 538, 530, 532, 478, 528, 15, 0, 0, 0, 0, 0, 0, 0, 530, 517, 486, 525, 499, 503, 503, 506, 19, 2, 1, 0, 0, 0, 0, 0, 500, 517, 485, 509, 529, 493, 497, 476, 0, 0, 0, 0, 0, 0, 0, 0, 523, 503, 516, 498, 509, 510, 512, 544, 36, 1, 0, 0, 0, 0, 0, 0, 522, 514, 531, 492, 521, 528, 542, 532, 0, 0, 0, 0, 0, 0, 0, 0, 498, 523, 477, 518, 468, 528, 509, 547, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4085, 4089, 4091, 4006, 4152, 4182, 4068] avg=4096.0000 max=4182 max_rank=6 min=4006 min_rank=4 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16408 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[502, 515, 509, 527, 486, 557, 513, 507, 0, 0, 0, 0, 0, 0, 0, 0, 515, 486, 520, 494, 562, 534, 481, 496, 30, 0, 0, 0, 0, 0, 0, 0, 498, 519, 500, 514, 527, 545, 484, 531, 19, 0, 0, 0, 0, 0, 0, 0, 495, 477, 496, 525, 479, 520, 535, 488, 25, 7, 6, 0, 0, 0, 0, 0, 483, 524, 477, 487, 513, 514, 522, 544, 0, 0, 0, 0, 0, 0, 0, 0, 514, 499, 521, 491, 519, 496, 510, 527, 44, 8, 0, 0, 0, 0, 0, 0, 514, 496, 513, 534, 499, 493, 518, 504, 0, 0, 0, 0, 0, 0, 0, 0, 536, 514, 491, 488, 542, 501, 493, 515, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4118, 4137, 4053, 4064, 4129, 4071, 4080] avg=4096.0000 max=4137 max_rank=2 min=4053 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16344 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[535, 528, 525, 483, 529, 521, 489, 517, 0, 0, 0, 0, 0, 0, 0, 0, 519, 484, 490, 492, 511, 475, 516, 489, 26, 0, 0, 0, 0, 0, 0, 0, 517, 448, 515, 523, 503, 531, 538, 497, 14, 0, 0, 0, 0, 0, 0, 0, 499, 499, 487, 530, 536, 507, 525, 506, 20, 3, 1, 0, 0, 0, 0, 0, 546, 508, 527, 494, 508, 512, 507, 513, 0, 0, 0, 0, 0, 0, 0, 0, 484, 513, 518, 508, 488, 499, 510, 515, 35, 1, 0, 0, 0, 0, 0, 0, 522, 471, 507, 487, 506, 544, 511, 510, 0, 0, 0, 0, 0, 0, 0, 0, 512, 532, 513, 542, 517, 527, 521, 532, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4127, 4002, 4086, 4113, 4115, 4071, 4058, 4196] avg=4096.0000 max=4196 max_rank=7 min=4002 min_rank=1 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16440 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4224, 4096, 4096, 3968, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 3968, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[499, 493, 539, 523, 463, 506, 537, 514, 507, 499, 495, 498, 509, 506, 531, 514, 541, 529, 498, 518, 478, 470, 523, 490, 518, 524, 501, 502, 478, 535, 499, 492, 514, 568, 504, 528, 537, 554, 475, 541, 499, 525, 476, 551, 535, 542, 568, 527, 518, 509, 535, 450, 475, 530, 507, 534, 461, 520, 506, 505, 469, 538, 532, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[481, 517, 504, 488, 534, 543, 490, 523, 531, 505, 511, 533, 503, 496, 498, 552, 529, 519, 498, 534, 521, 514, 512, 494, 486, 568, 525, 520, 501, 537, 522, 482, 535, 516, 520, 502, 474, 495, 489, 515, 487, 543, 507, 486, 515, 527, 501, 488, 511, 517, 496, 508, 523, 479, 491, 546, 498, 514, 508, 514, 534, 517, 527, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[503, 481, 503, 495, 522, 493, 509, 513, 527, 496, 551, 547, 503, 483, 547, 508, 533, 507, 508, 517, 479, 498, 546, 542, 560, 494, 481, 528, 542, 499, 491, 506, 561, 507, 508, 495, 511, 483, 507, 512, 517, 522, 468, 529, 499, 468, 520, 504, 518, 535, 556, 536, 518, 501, 502, 491, 505, 501, 533, 496, 515, 513, 534, 491] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[544, 533, 512, 490, 545, 486, 523, 549, 516, 497, 532, 489, 545, 507, 504, 499, 476, 497, 487, 499, 502, 509, 465, 502, 497, 501, 530, 559, 498, 496, 523, 511, 516, 538, 480, 522, 538, 545, 505, 495, 522, 504, 485, 522, 495, 484, 548, 517, 506, 537, 483, 507, 495, 505, 558, 501, 542, 497, 517, 493, 522, 501, 549, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[495, 565, 476, 534, 524, 478, 549, 507, 467, 497, 521, 511, 536, 515, 478, 523, 508, 521, 509, 529, 513, 506, 536, 515, 513, 507, 515, 473, 498, 499, 545, 521, 523, 491, 531, 490, 481, 516, 500, 542, 503, 532, 493, 510, 496, 516, 516, 495, 538, 533, 496, 518, 507, 504, 510, 503, 516, 485, 547, 531, 524, 549, 486, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[494, 486, 502, 526, 526, 516, 514, 505, 522, 506, 503, 525, 464, 526, 529, 489, 498, 517, 520, 514, 531, 544, 489, 511, 514, 491, 515, 501, 536, 535, 542, 487, 499, 482, 519, 512, 495, 514, 522, 522, 495, 507, 514, 512, 531, 464, 577, 503, 518, 470, 476, 538, 504, 515, 526, 522, 535, 499, 531, 570, 483, 502, 511, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[469, 504, 518, 537, 495, 517, 542, 545, 519, 487, 518, 504, 553, 517, 519, 522, 527, 512, 488, 501, 528, 522, 526, 546, 500, 534, 476, 512, 523, 512, 509, 508, 496, 528, 488, 489, 531, 506, 538, 497, 491, 495, 538, 494, 533, 489, 543, 520, 476, 524, 486, 487, 463, 530, 533, 499, 469, 523, 522, 512, 521, 502, 556, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[555, 545, 472, 510, 565, 521, 495, 502, 529, 507, 509, 479, 508, 545, 494, 519, 484, 531, 527, 479, 517, 537, 501, 487, 553, 503, 526, 514, 524, 497, 507, 484, 510, 549, 532, 539, 514, 484, 524, 511, 518, 512, 570, 514, 494, 472, 487, 522, 495, 554, 484, 499, 519, 459, 483, 525, 503, 528, 469, 530, 491, 500, 550, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[499, 493, 539, 523, 456, 506, 537, 514, 0, 0, 0, 0, 0, 0, 0, 0, 507, 499, 494, 498, 509, 506, 531, 514, 0, 0, 0, 0, 0, 0, 0, 0, 541, 529, 498, 518, 478, 470, 523, 490, 6, 0, 0, 0, 0, 0, 0, 0, 517, 524, 501, 502, 478, 535, 499, 492, 0, 0, 0, 0, 0, 0, 0, 0, 514, 555, 504, 528, 537, 554, 475, 541, 0, 0, 0, 0, 0, 0, 0, 0, 499, 525, 476, 551, 535, 542, 568, 527, 1, 1, 0, 0, 0, 0, 0, 0, 518, 509, 535, 450, 475, 530, 507, 534, 13, 11, 1, 0, 0, 0, 0, 0, 461, 520, 506, 505, 469, 538, 521, 506, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4058, 4053, 4048, 4208, 4225, 4083, 4026] avg=4096.0000 max=4225 max_rank=5 min=4026 min_rank=7 imbalance_ratio=1.031494 max_load_reduction=28543 max_load_reduction_pct=0.871063 moved_to_echo=16542 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[503, 481, 503, 495, 514, 493, 509, 513, 0, 0, 0, 0, 0, 0, 0, 0, 527, 496, 550, 547, 503, 483, 547, 508, 0, 0, 0, 0, 0, 0, 0, 0, 533, 507, 508, 517, 479, 498, 546, 542, 7, 0, 0, 0, 0, 0, 0, 0, 559, 494, 481, 528, 542, 499, 491, 506, 0, 0, 0, 0, 0, 0, 0, 0, 561, 496, 508, 495, 511, 483, 507, 512, 0, 0, 0, 0, 0, 0, 0, 0, 517, 522, 468, 529, 499, 468, 520, 504, 1, 1, 0, 0, 0, 0, 0, 0, 518, 535, 556, 536, 518, 501, 502, 491, 11, 11, 1, 0, 0, 0, 0, 0, 505, 501, 533, 496, 515, 513, 523, 491, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4011, 4161, 4137, 4100, 4073, 4029, 4180, 4077] avg=4096.0000 max=4180 max_rank=6 min=4011 min_rank=0 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16359 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[481, 517, 504, 488, 526, 543, 490, 523, 0, 0, 0, 0, 0, 0, 0, 0, 531, 505, 510, 533, 503, 496, 498, 552, 0, 0, 0, 0, 0, 0, 0, 0, 529, 519, 498, 534, 521, 514, 512, 494, 7, 0, 0, 0, 0, 0, 0, 0, 485, 568, 525, 520, 501, 537, 522, 482, 0, 0, 0, 0, 0, 0, 0, 0, 535, 505, 520, 502, 474, 495, 489, 515, 0, 0, 0, 0, 0, 0, 0, 0, 487, 543, 507, 486, 515, 527, 501, 488, 1, 1, 0, 0, 0, 0, 0, 0, 511, 517, 496, 508, 523, 479, 491, 546, 11, 11, 1, 0, 0, 0, 0, 0, 498, 514, 508, 514, 534, 517, 516, 514, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4128, 4128, 4140, 4035, 4056, 4094, 4115] avg=4096.0000 max=4140 max_rank=3 min=4035 min_rank=4 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16300 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[544, 533, 512, 490, 537, 486, 523, 549, 0, 0, 0, 0, 0, 0, 0, 0, 516, 497, 531, 489, 545, 507, 504, 499, 0, 0, 0, 0, 0, 0, 0, 0, 476, 497, 487, 499, 502, 509, 465, 502, 7, 0, 0, 0, 0, 0, 0, 0, 496, 501, 530, 559, 498, 496, 523, 511, 0, 0, 0, 0, 0, 0, 0, 0, 516, 526, 480, 522, 538, 545, 505, 495, 0, 0, 0, 0, 0, 0, 0, 0, 522, 504, 485, 522, 495, 484, 548, 517, 1, 1, 0, 0, 0, 0, 0, 0, 506, 537, 483, 507, 495, 505, 558, 501, 12, 11, 1, 0, 0, 0, 0, 0, 542, 497, 517, 493, 522, 501, 538, 516, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4174, 4088, 3944, 4114, 4127, 4079, 4116, 4126] avg=4096.0000 max=4174 max_rank=0 min=3944 min_rank=2 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16448 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[495, 565, 476, 534, 502, 478, 549, 507, 0, 0, 0, 0, 0, 0, 0, 0, 467, 497, 516, 511, 536, 515, 478, 523, 0, 0, 0, 0, 0, 0, 0, 0, 508, 521, 509, 529, 513, 506, 536, 515, 12, 0, 0, 0, 0, 0, 0, 0, 508, 507, 515, 473, 498, 499, 545, 521, 0, 0, 0, 0, 0, 0, 0, 0, 523, 476, 531, 490, 481, 516, 500, 542, 0, 0, 0, 0, 0, 0, 0, 0, 503, 532, 493, 510, 496, 516, 516, 495, 5, 5, 7, 0, 0, 0, 0, 0, 538, 533, 496, 518, 507, 504, 510, 503, 15, 13, 3, 0, 0, 0, 0, 0, 516, 485, 547, 531, 524, 549, 473, 502, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4043, 4149, 4066, 4059, 4078, 4140, 4127] avg=4096.0000 max=4149 max_rank=2 min=4043 min_rank=1 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16404 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[494, 486, 502, 526, 518, 516, 514, 505, 0, 0, 0, 0, 0, 0, 0, 0, 522, 506, 502, 525, 464, 526, 529, 489, 0, 0, 0, 0, 0, 0, 0, 0, 498, 517, 520, 514, 531, 544, 489, 511, 7, 0, 0, 0, 0, 0, 0, 0, 513, 491, 515, 501, 536, 535, 542, 487, 0, 0, 0, 0, 0, 0, 0, 0, 499, 471, 519, 512, 495, 514, 522, 522, 0, 0, 0, 0, 0, 0, 0, 0, 495, 507, 514, 512, 531, 464, 577, 503, 1, 1, 0, 0, 0, 0, 0, 0, 518, 470, 476, 538, 504, 515, 526, 522, 11, 10, 1, 0, 0, 0, 0, 0, 535, 499, 531, 570, 483, 502, 501, 522, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4063, 4131, 4120, 4054, 4105, 4091, 4143] avg=4096.0000 max=4143 max_rank=7 min=4054 min_rank=4 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[555, 545, 472, 510, 557, 521, 495, 502, 0, 0, 0, 0, 0, 0, 0, 0, 529, 507, 508, 479, 508, 545, 494, 519, 0, 0, 0, 0, 0, 0, 0, 0, 484, 531, 527, 479, 517, 537, 501, 487, 7, 0, 0, 0, 0, 0, 0, 0, 552, 503, 526, 514, 524, 497, 507, 484, 0, 0, 0, 0, 0, 0, 0, 0, 510, 537, 532, 539, 514, 484, 524, 511, 0, 0, 0, 0, 0, 0, 0, 0, 518, 512, 570, 514, 494, 472, 487, 522, 1, 1, 0, 0, 0, 0, 0, 0, 495, 554, 484, 499, 519, 459, 483, 525, 12, 11, 1, 0, 0, 0, 0, 0, 503, 528, 469, 530, 491, 500, 539, 501, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4157, 4089, 4070, 4107, 4151, 4091, 4042, 4061] avg=4096.0000 max=4157 max_rank=0 min=4042 min_rank=6 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16345 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[469, 504, 518, 537, 488, 517, 542, 545, 0, 0, 0, 0, 0, 0, 0, 0, 519, 487, 517, 504, 553, 517, 519, 522, 0, 0, 0, 0, 0, 0, 0, 0, 527, 512, 488, 501, 528, 522, 526, 546, 6, 0, 0, 0, 0, 0, 0, 0, 499, 534, 476, 512, 523, 512, 509, 508, 0, 0, 0, 0, 0, 0, 0, 0, 496, 516, 488, 489, 531, 506, 538, 497, 0, 0, 0, 0, 0, 0, 0, 0, 491, 495, 538, 494, 533, 489, 543, 520, 1, 1, 0, 0, 0, 0, 0, 0, 476, 524, 486, 487, 463, 530, 533, 499, 12, 11, 1, 0, 0, 0, 0, 0, 469, 523, 522, 512, 521, 502, 545, 499, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4138, 4156, 4073, 4061, 4105, 4022, 4093] avg=4096.0000 max=4156 max_rank=2 min=4022 min_rank=6 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16281 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 3968, 4352, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[497, 488, 532, 478, 488, 500, 545, 492, 506, 477, 493, 524, 551, 514, 541, 510, 480, 514, 501, 547, 505, 485, 496, 487, 539, 528, 517, 525, 498, 501, 515, 489, 519, 488, 526, 505, 524, 515, 517, 499, 528, 490, 553, 514, 483, 530, 515, 489, 502, 527, 513, 509, 509, 533, 531, 522, 543, 520, 535, 541, 499, 529, 521, 476] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[515, 510, 472, 491, 492, 498, 492, 491, 519, 558, 487, 511, 521, 532, 557, 528, 574, 526, 528, 511, 492, 476, 506, 552, 525, 541, 466, 520, 516, 475, 469, 509, 488, 522, 524, 524, 481, 520, 464, 516, 525, 543, 531, 501, 522, 506, 502, 514, 483, 535, 535, 525, 522, 517, 525, 489, 514, 503, 509, 543, 500, 494, 518, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[512, 548, 496, 491, 521, 547, 475, 531, 531, 521, 527, 479, 497, 535, 501, 503, 542, 469, 511, 519, 514, 564, 514, 532, 478, 493, 501, 498, 504, 497, 521, 524, 509, 532, 509, 507, 511, 485, 522, 532, 456, 532, 513, 529, 521, 523, 495, 481, 516, 507, 498, 509, 515, 576, 505, 526, 519, 515, 507, 482, 487, 506, 509, 538] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[507, 523, 489, 487, 552, 515, 526, 483, 496, 525, 506, 507, 513, 503, 526, 529, 510, 550, 499, 592, 525, 525, 496, 529, 531, 480, 496, 492, 533, 461, 507, 526, 555, 487, 525, 474, 510, 518, 487, 505, 493, 496, 494, 477, 525, 542, 494, 535, 518, 538, 507, 479, 546, 513, 487, 508, 515, 510, 538, 513, 513, 521, 514, 492] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[523, 483, 482, 523, 552, 451, 527, 505, 531, 525, 509, 503, 497, 490, 512, 494, 523, 497, 501, 541, 536, 568, 508, 504, 521, 534, 501, 530, 468, 500, 535, 516, 498, 511, 504, 490, 519, 530, 533, 536, 520, 520, 514, 497, 489, 501, 525, 513, 475, 518, 540, 541, 539, 508, 555, 486, 503, 512, 499, 538, 470, 502, 490, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[515, 549, 478, 527, 485, 500, 519, 513, 522, 503, 540, 506, 513, 509, 525, 490, 554, 535, 512, 504, 503, 545, 506, 537, 518, 485, 516, 515, 510, 497, 488, 504, 508, 500, 492, 523, 499, 503, 526, 506, 510, 520, 497, 472, 523, 514, 522, 506, 506, 502, 507, 564, 539, 535, 490, 520, 546, 498, 492, 538, 479, 498, 505, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[551, 477, 551, 523, 490, 503, 503, 475, 523, 489, 514, 478, 522, 503, 506, 525, 512, 500, 506, 527, 523, 511, 514, 545, 498, 519, 508, 503, 517, 520, 527, 481, 517, 517, 498, 483, 539, 548, 517, 478, 528, 512, 488, 489, 508, 543, 518, 515, 563, 555, 551, 533, 525, 497, 510, 491, 506, 486, 509, 511, 460, 526, 507, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[524, 482, 527, 520, 489, 493, 508, 500, 500, 497, 524, 561, 526, 508, 546, 549, 489, 523, 528, 539, 505, 477, 481, 493, 483, 465, 542, 553, 522, 474, 517, 508, 531, 550, 513, 517, 548, 534, 463, 544, 505, 508, 524, 496, 490, 478, 472, 481, 534, 535, 518, 517, 542, 523, 516, 527, 516, 514, 513, 479, 509, 534, 465, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[497, 488, 532, 478, 488, 500, 545, 492, 40, 0, 0, 0, 0, 0, 0, 0, 506, 477, 493, 524, 551, 514, 516, 510, 0, 0, 0, 0, 0, 0, 0, 0, 480, 514, 501, 503, 505, 485, 496, 487, 0, 0, 0, 0, 0, 0, 0, 0, 539, 528, 517, 525, 498, 501, 515, 489, 21, 17, 0, 0, 0, 0, 0, 0, 519, 488, 526, 505, 524, 515, 517, 499, 0, 0, 0, 0, 0, 0, 0, 0, 528, 490, 553, 514, 483, 530, 515, 489, 27, 0, 0, 0, 0, 0, 0, 0, 502, 527, 513, 509, 448, 533, 531, 522, 0, 0, 0, 0, 0, 0, 0, 0, 543, 520, 535, 541, 499, 529, 521, 476, 25, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4091, 3971, 4150, 4093, 4129, 4085, 4189] avg=4096.0000 max=4189 max_rank=7 min=3971 min_rank=2 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16496 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[512, 548, 496, 491, 521, 547, 475, 531, 40, 0, 0, 0, 0, 0, 0, 0, 531, 521, 527, 479, 497, 535, 478, 503, 0, 0, 0, 0, 0, 0, 0, 0, 542, 469, 511, 477, 514, 564, 514, 532, 0, 0, 0, 0, 0, 0, 0, 0, 478, 493, 501, 498, 504, 497, 521, 524, 22, 16, 0, 0, 0, 0, 0, 0, 509, 532, 509, 507, 511, 485, 522, 532, 0, 0, 0, 0, 0, 0, 0, 0, 456, 532, 513, 529, 521, 523, 495, 481, 26, 0, 0, 0, 0, 0, 0, 0, 516, 507, 498, 509, 453, 576, 505, 526, 0, 0, 0, 0, 0, 0, 0, 0, 519, 515, 507, 482, 487, 506, 509, 538, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4161, 4071, 4123, 4054, 4107, 4076, 4090, 4086] avg=4096.0000 max=4161 max_rank=0 min=4054 min_rank=3 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16359 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[507, 523, 489, 487, 552, 515, 526, 483, 43, 0, 0, 0, 0, 0, 0, 0, 496, 525, 506, 507, 513, 503, 502, 529, 0, 0, 0, 0, 0, 0, 0, 0, 510, 550, 499, 545, 525, 525, 496, 529, 0, 0, 0, 0, 0, 0, 0, 0, 531, 480, 496, 492, 533, 461, 507, 526, 23, 18, 0, 0, 0, 0, 0, 0, 555, 487, 525, 474, 510, 518, 487, 505, 0, 0, 0, 0, 0, 0, 0, 0, 493, 496, 494, 477, 525, 542, 494, 535, 29, 0, 0, 0, 0, 0, 0, 0, 518, 538, 507, 479, 480, 513, 487, 508, 0, 0, 0, 0, 0, 0, 0, 0, 515, 510, 538, 513, 513, 521, 514, 492, 24, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4081, 4179, 4067, 4061, 4085, 4030, 4140] avg=4096.0000 max=4179 max_rank=2 min=4030 min_rank=6 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16316 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[515, 510, 472, 491, 492, 498, 492, 491, 46, 0, 0, 0, 0, 0, 0, 0, 519, 558, 487, 511, 521, 532, 524, 528, 0, 0, 0, 0, 0, 0, 0, 0, 574, 526, 528, 464, 492, 476, 506, 552, 0, 0, 0, 0, 0, 0, 0, 0, 525, 541, 466, 520, 516, 475, 469, 509, 26, 19, 0, 0, 0, 0, 0, 0, 488, 522, 524, 524, 481, 513, 464, 516, 0, 0, 0, 0, 0, 0, 0, 0, 525, 543, 531, 501, 522, 506, 502, 514, 28, 3, 0, 0, 0, 0, 0, 0, 483, 535, 535, 525, 450, 517, 525, 489, 0, 0, 0, 0, 0, 0, 0, 0, 514, 503, 509, 543, 500, 494, 518, 513, 30, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4007, 4180, 4118, 4066, 4032, 4175, 4059, 4131] avg=4096.0000 max=4180 max_rank=1 min=4007 min_rank=0 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[515, 549, 478, 527, 485, 500, 519, 513, 42, 0, 0, 0, 0, 0, 0, 0, 522, 503, 540, 506, 513, 509, 501, 490, 0, 0, 0, 0, 0, 0, 0, 0, 554, 535, 512, 464, 503, 545, 506, 537, 0, 0, 0, 0, 0, 0, 0, 0, 518, 485, 516, 515, 510, 497, 488, 504, 23, 15, 0, 0, 0, 0, 0, 0, 508, 500, 492, 523, 499, 503, 526, 506, 0, 0, 0, 0, 0, 0, 0, 0, 510, 520, 497, 472, 523, 514, 522, 506, 25, 0, 0, 0, 0, 0, 0, 0, 506, 502, 507, 564, 474, 535, 490, 520, 0, 0, 0, 0, 0, 0, 0, 0, 546, 498, 492, 538, 479, 498, 505, 505, 24, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4084, 4156, 4071, 4057, 4089, 4098, 4085] avg=4096.0000 max=4156 max_rank=2 min=4057 min_rank=4 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16329 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[551, 477, 551, 523, 490, 503, 503, 475, 41, 0, 0, 0, 0, 0, 0, 0, 523, 489, 514, 478, 522, 503, 483, 525, 0, 0, 0, 0, 0, 0, 0, 0, 512, 500, 506, 485, 523, 511, 514, 545, 0, 0, 0, 0, 0, 0, 0, 0, 498, 519, 508, 503, 517, 520, 527, 481, 22, 16, 0, 0, 0, 0, 0, 0, 517, 517, 498, 483, 539, 548, 517, 478, 0, 0, 0, 0, 0, 0, 0, 0, 528, 512, 488, 489, 508, 543, 518, 515, 26, 0, 0, 0, 0, 0, 0, 0, 563, 555, 551, 533, 462, 497, 510, 491, 0, 0, 0, 0, 0, 0, 0, 0, 506, 486, 509, 511, 460, 526, 507, 496, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4037, 4096, 4111, 4097, 4127, 4162, 4024] avg=4096.0000 max=4162 max_rank=6 min=4024 min_rank=7 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16410 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[523, 483, 482, 523, 552, 451, 527, 505, 42, 0, 0, 0, 0, 0, 0, 0, 531, 525, 509, 503, 497, 490, 489, 494, 0, 0, 0, 0, 0, 0, 0, 0, 523, 497, 501, 497, 536, 568, 508, 504, 0, 0, 0, 0, 0, 0, 0, 0, 521, 534, 501, 530, 468, 500, 535, 516, 23, 17, 0, 0, 0, 0, 0, 0, 498, 511, 504, 490, 519, 530, 533, 536, 0, 0, 0, 0, 0, 0, 0, 0, 520, 520, 514, 497, 489, 501, 525, 513, 27, 0, 0, 0, 0, 0, 0, 0, 475, 518, 540, 541, 474, 508, 555, 486, 0, 0, 0, 0, 0, 0, 0, 0, 503, 512, 499, 538, 470, 502, 490, 502, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4038, 4134, 4145, 4121, 4106, 4097, 4039] avg=4096.0000 max=4145 max_rank=3 min=4038 min_rank=1 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[524, 482, 527, 520, 489, 493, 508, 500, 42, 0, 0, 0, 0, 0, 0, 0, 500, 497, 524, 561, 526, 508, 521, 549, 0, 0, 0, 0, 0, 0, 0, 0, 489, 523, 528, 495, 505, 477, 481, 493, 0, 0, 0, 0, 0, 0, 0, 0, 483, 465, 542, 553, 522, 474, 517, 508, 23, 17, 0, 0, 0, 0, 0, 0, 531, 550, 513, 517, 548, 534, 463, 544, 0, 0, 0, 0, 0, 0, 0, 0, 505, 508, 524, 496, 490, 478, 472, 481, 27, 0, 0, 0, 0, 0, 0, 0, 534, 535, 518, 517, 477, 523, 516, 527, 0, 0, 0, 0, 0, 0, 0, 0, 516, 514, 513, 479, 509, 534, 465, 519, 25, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4186, 3991, 4104, 4200, 3981, 4147, 4074] avg=4096.0000 max=4200 max_rank=4 min=3981 min_rank=5 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16402 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4224, 3968, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 3968, 4096, 4096, 256, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4224, 3840, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 3968, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[519, 490, 526, 504, 503, 519, 513, 490, 513, 500, 522, 539, 488, 518, 503, 502, 515, 536, 517, 527, 513, 527, 529, 522, 468, 491, 549, 495, 520, 497, 536, 494, 553, 503, 518, 518, 505, 533, 491, 515, 510, 497, 493, 535, 549, 538, 475, 507, 530, 512, 546, 524, 545, 504, 519, 487, 478, 523, 499, 498, 524, 474, 499, 481] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[493, 511, 502, 487, 470, 518, 522, 517, 539, 513, 492, 511, 526, 534, 498, 498, 524, 482, 510, 514, 517, 516, 499, 470, 522, 519, 463, 488, 518, 506, 532, 539, 489, 532, 512, 523, 509, 519, 508, 521, 494, 541, 502, 530, 527, 511, 491, 503, 513, 563, 488, 553, 503, 530, 481, 542, 505, 519, 544, 517, 505, 514, 523, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[485, 482, 533, 485, 509, 471, 433, 507, 496, 539, 526, 526, 511, 477, 563, 486, 528, 499, 490, 520, 510, 546, 538, 476, 503, 524, 521, 552, 536, 520, 474, 513, 539, 522, 490, 511, 492, 559, 523, 513, 526, 494, 488, 489, 495, 485, 516, 508, 516, 518, 537, 539, 491, 540, 519, 534, 565, 514, 495, 500, 501, 523, 539, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[569, 535, 471, 514, 495, 538, 487, 519, 506, 512, 523, 514, 520, 477, 512, 459, 522, 527, 518, 504, 523, 507, 484, 491, 511, 529, 535, 518, 514, 498, 531, 505, 517, 515, 503, 495, 537, 469, 496, 450, 524, 478, 488, 499, 532, 520, 534, 515, 524, 518, 554, 511, 527, 519, 569, 505, 513, 516, 513, 520, 547, 487, 498, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[509, 487, 509, 544, 488, 527, 480, 505, 564, 496, 526, 513, 499, 534, 511, 539, 506, 479, 524, 513, 545, 534, 485, 533, 521, 519, 487, 487, 544, 506, 522, 512, 546, 484, 496, 495, 474, 523, 525, 508, 527, 496, 545, 510, 519, 462, 478, 556, 529, 517, 461, 517, 500, 516, 570, 546, 472, 459, 506, 545, 480, 545, 507, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[497, 532, 511, 526, 557, 525, 510, 513, 569, 492, 513, 478, 515, 488, 495, 502, 520, 523, 539, 504, 519, 509, 526, 479, 538, 487, 499, 481, 521, 485, 521, 502, 513, 494, 506, 542, 511, 484, 514, 552, 519, 532, 465, 515, 484, 525, 510, 484, 499, 531, 508, 498, 518, 505, 500, 537, 526, 514, 501, 562, 511, 492, 514, 526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[510, 544, 534, 499, 534, 518, 521, 537, 538, 520, 525, 488, 464, 509, 519, 505, 501, 486, 524, 529, 492, 525, 516, 520, 523, 559, 478, 514, 536, 468, 486, 518, 517, 533, 500, 510, 478, 546, 484, 527, 516, 528, 469, 524, 540, 497, 534, 532, 481, 536, 453, 482, 486, 561, 530, 486, 499, 495, 536, 488, 537, 502, 502, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[508, 485, 523, 519, 536, 516, 505, 492, 475, 516, 505, 512, 517, 463, 504, 518, 493, 534, 533, 508, 489, 504, 517, 540, 488, 512, 511, 532, 558, 537, 543, 513, 476, 557, 525, 489, 525, 489, 506, 521, 493, 486, 502, 513, 521, 484, 505, 515, 516, 532, 516, 492, 501, 508, 527, 531, 522, 511, 491, 494, 540, 542, 557, 475] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[519, 490, 526, 504, 503, 519, 513, 490, 18, 0, 0, 0, 0, 0, 0, 0, 513, 500, 522, 539, 488, 518, 503, 502, 8, 5, 0, 0, 0, 0, 0, 0, 515, 536, 517, 527, 513, 518, 529, 522, 0, 0, 0, 0, 0, 0, 0, 0, 468, 491, 549, 495, 514, 497, 536, 494, 0, 0, 0, 0, 0, 0, 0, 0, 553, 503, 518, 518, 505, 533, 491, 515, 2, 1, 0, 0, 0, 0, 0, 0, 510, 497, 493, 535, 549, 538, 475, 507, 22, 0, 0, 0, 0, 0, 0, 0, 530, 467, 546, 524, 545, 504, 519, 487, 0, 0, 0, 0, 0, 0, 0, 0, 478, 523, 499, 498, 524, 474, 499, 481, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4098, 4177, 4044, 4139, 4126, 4122, 3980] avg=4096.0000 max=4177 max_rank=2 min=3980 min_rank=7 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[493, 511, 502, 487, 470, 518, 522, 517, 19, 0, 0, 0, 0, 0, 0, 0, 539, 513, 492, 511, 526, 534, 498, 498, 8, 5, 0, 0, 0, 0, 0, 0, 524, 482, 510, 514, 517, 507, 499, 470, 0, 0, 0, 0, 0, 0, 0, 0, 522, 519, 463, 488, 512, 506, 532, 539, 0, 0, 0, 0, 0, 0, 0, 0, 489, 532, 512, 523, 509, 519, 508, 521, 2, 1, 0, 0, 0, 0, 0, 0, 494, 541, 502, 530, 527, 511, 491, 503, 25, 0, 0, 0, 0, 0, 0, 0, 513, 514, 488, 553, 503, 530, 481, 542, 0, 0, 0, 0, 0, 0, 0, 0, 505, 519, 544, 517, 505, 514, 523, 506, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4039, 4124, 4023, 4081, 4116, 4124, 4124, 4137] avg=4096.0000 max=4137 max_rank=7 min=4023 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16501 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[485, 482, 533, 485, 509, 471, 433, 507, 18, 0, 0, 0, 0, 0, 0, 0, 496, 539, 526, 526, 511, 477, 563, 486, 8, 5, 0, 0, 0, 0, 0, 0, 528, 499, 490, 520, 510, 537, 538, 476, 0, 0, 0, 0, 0, 0, 0, 0, 503, 524, 521, 552, 529, 520, 474, 513, 0, 0, 0, 0, 0, 0, 0, 0, 539, 522, 490, 511, 492, 559, 523, 513, 3, 1, 0, 0, 0, 0, 0, 0, 526, 494, 488, 489, 495, 485, 516, 508, 23, 0, 0, 0, 0, 0, 0, 0, 516, 472, 537, 539, 491, 540, 519, 534, 0, 0, 0, 0, 0, 0, 0, 0, 565, 514, 495, 500, 501, 523, 539, 508, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3923, 4137, 4098, 4136, 4153, 4024, 4148, 4149] avg=4096.0000 max=4153 max_rank=4 min=3923 min_rank=0 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16474 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[569, 535, 471, 514, 495, 538, 487, 519, 22, 0, 0, 0, 0, 0, 0, 0, 506, 512, 523, 514, 520, 477, 512, 459, 12, 10, 0, 0, 0, 0, 0, 0, 522, 527, 518, 504, 523, 488, 484, 491, 0, 0, 0, 0, 0, 0, 0, 0, 511, 529, 535, 518, 499, 498, 531, 505, 0, 0, 0, 0, 0, 0, 0, 0, 517, 515, 503, 495, 537, 469, 496, 450, 6, 7, 0, 0, 0, 0, 0, 0, 524, 478, 488, 499, 532, 520, 534, 515, 27, 0, 0, 0, 0, 0, 0, 0, 524, 459, 554, 511, 527, 519, 569, 505, 0, 0, 0, 0, 0, 0, 0, 0, 513, 516, 513, 520, 547, 487, 498, 507, 9, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4150, 4045, 4057, 4126, 3995, 4117, 4168, 4110] avg=4096.0000 max=4168 max_rank=6 min=3995 min_rank=4 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[509, 487, 509, 544, 488, 527, 480, 505, 18, 0, 0, 0, 0, 0, 0, 0, 564, 496, 526, 513, 499, 534, 511, 539, 8, 5, 0, 0, 0, 0, 0, 0, 506, 479, 524, 513, 545, 525, 485, 533, 0, 0, 0, 0, 0, 0, 0, 0, 521, 519, 487, 487, 537, 506, 522, 512, 0, 0, 0, 0, 0, 0, 0, 0, 546, 484, 496, 495, 474, 523, 525, 508, 3, 1, 0, 0, 0, 0, 0, 0, 527, 496, 545, 510, 519, 462, 478, 556, 22, 0, 0, 0, 0, 0, 0, 0, 529, 472, 461, 517, 500, 516, 570, 546, 0, 0, 0, 0, 0, 0, 0, 0, 472, 459, 506, 545, 480, 545, 507, 506, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4195, 4110, 4091, 4055, 4115, 4111, 4024] avg=4096.0000 max=4195 max_rank=1 min=4024 min_rank=7 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16305 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[497, 532, 511, 526, 557, 525, 510, 513, 18, 0, 0, 0, 0, 0, 0, 0, 569, 492, 513, 478, 515, 488, 495, 502, 8, 5, 0, 0, 0, 0, 0, 0, 520, 523, 539, 504, 519, 500, 526, 479, 0, 0, 0, 0, 0, 0, 0, 0, 538, 487, 499, 481, 515, 485, 521, 502, 0, 0, 0, 0, 0, 0, 0, 0, 513, 494, 506, 542, 511, 484, 514, 552, 2, 1, 0, 0, 0, 0, 0, 0, 519, 532, 465, 515, 484, 525, 510, 484, 23, 0, 0, 0, 0, 0, 0, 0, 499, 485, 508, 498, 518, 505, 500, 537, 0, 0, 0, 0, 0, 0, 0, 0, 526, 514, 501, 562, 511, 492, 514, 526, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4189, 4065, 4110, 4028, 4119, 4057, 4050, 4150] avg=4096.0000 max=4189 max_rank=0 min=4028 min_rank=3 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16376 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[510, 544, 534, 499, 534, 518, 521, 537, 19, 0, 0, 0, 0, 0, 0, 0, 538, 520, 525, 488, 464, 509, 519, 505, 8, 5, 0, 0, 0, 0, 0, 0, 501, 486, 524, 529, 492, 516, 516, 520, 0, 0, 0, 0, 0, 0, 0, 0, 523, 559, 478, 514, 529, 468, 486, 518, 0, 0, 0, 0, 0, 0, 0, 0, 517, 533, 500, 510, 478, 546, 484, 527, 3, 1, 0, 0, 0, 0, 0, 0, 516, 528, 469, 524, 540, 497, 534, 532, 23, 0, 0, 0, 0, 0, 0, 0, 481, 489, 453, 482, 486, 561, 530, 486, 0, 0, 0, 0, 0, 0, 0, 0, 499, 495, 536, 488, 537, 502, 502, 519, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4216, 4081, 4084, 4075, 4099, 4163, 3968, 4082] avg=4096.0000 max=4216 max_rank=0 min=3968 min_rank=6 imbalance_ratio=1.029297 max_load_reduction=28552 max_load_reduction_pct=0.871338 moved_to_echo=16312 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[508, 485, 523, 519, 536, 516, 505, 492, 18, 0, 0, 0, 0, 0, 0, 0, 475, 516, 505, 512, 517, 463, 504, 518, 8, 5, 0, 0, 0, 0, 0, 0, 493, 534, 533, 508, 489, 495, 517, 540, 0, 0, 0, 0, 0, 0, 0, 0, 488, 512, 511, 532, 551, 537, 543, 513, 0, 0, 0, 0, 0, 0, 0, 0, 476, 557, 525, 489, 525, 489, 506, 521, 3, 1, 0, 0, 0, 0, 0, 0, 493, 486, 502, 513, 521, 484, 505, 515, 23, 0, 0, 0, 0, 0, 0, 0, 516, 486, 516, 492, 501, 508, 527, 531, 0, 0, 0, 0, 0, 0, 0, 0, 522, 511, 491, 494, 540, 542, 557, 475, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4023, 4109, 4187, 4092, 4042, 4077, 4136] avg=4096.0000 max=4187 max_rank=3 min=4023 min_rank=1 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 3968, 4224, 4224, 4096, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 3968, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[537, 473, 536, 505, 513, 512, 483, 500, 526, 522, 535, 545, 505, 513, 495, 538, 518, 505, 509, 504, 521, 523, 481, 553, 494, 508, 518, 482, 462, 487, 500, 506, 496, 531, 518, 523, 516, 530, 528, 487, 513, 506, 513, 494, 527, 491, 533, 536, 508, 523, 521, 491, 507, 539, 520, 541, 472, 542, 480, 551, 490, 524, 501, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[515, 525, 511, 535, 518, 487, 526, 511, 558, 513, 507, 512, 536, 466, 501, 493, 510, 487, 521, 510, 535, 527, 556, 508, 496, 531, 532, 514, 538, 538, 519, 541, 453, 517, 510, 503, 494, 494, 469, 521, 499, 500, 522, 483, 493, 496, 542, 475, 518, 519, 525, 549, 529, 525, 515, 478, 506, 514, 516, 472, 513, 504, 471, 566] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[518, 510, 530, 498, 512, 530, 521, 504, 504, 507, 520, 531, 537, 529, 515, 489, 529, 528, 515, 508, 491, 502, 514, 498, 497, 483, 518, 536, 494, 509, 470, 500, 478, 498, 509, 489, 518, 530, 510, 528, 561, 515, 528, 522, 497, 502, 496, 507, 506, 472, 511, 536, 499, 516, 501, 530, 539, 566, 510, 514, 507, 501, 519, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[520, 507, 518, 512, 520, 552, 503, 509, 509, 517, 512, 533, 498, 506, 534, 502, 533, 493, 526, 515, 500, 509, 525, 514, 496, 499, 514, 498, 506, 520, 519, 526, 500, 521, 515, 492, 518, 504, 512, 527, 482, 493, 520, 511, 545, 472, 547, 532, 475, 480, 480, 540, 509, 544, 526, 530, 527, 509, 502, 480, 526, 509, 518, 477] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[521, 454, 529, 509, 503, 525, 556, 523, 492, 521, 561, 486, 496, 508, 517, 489, 499, 547, 527, 521, 514, 522, 473, 515, 521, 527, 471, 513, 543, 531, 489, 494, 510, 485, 489, 545, 503, 525, 497, 502, 501, 513, 527, 562, 492, 574, 525, 511, 510, 495, 518, 503, 529, 489, 511, 521, 505, 506, 536, 528, 465, 501, 516, 477] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[490, 500, 518, 508, 503, 515, 499, 529, 540, 537, 500, 541, 541, 529, 536, 518, 513, 495, 521, 514, 513, 581, 497, 518, 532, 501, 486, 495, 502, 537, 513, 525, 479, 480, 499, 517, 532, 509, 498, 485, 495, 514, 549, 463, 472, 535, 503, 487, 490, 536, 535, 479, 510, 524, 481, 516, 485, 504, 520, 526, 526, 487, 559, 526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[532, 497, 543, 512, 460, 511, 536, 510, 523, 536, 486, 514, 501, 515, 514, 515, 546, 533, 550, 532, 495, 517, 515, 515, 530, 515, 485, 509, 490, 531, 500, 503, 476, 507, 527, 513, 513, 558, 510, 508, 538, 504, 518, 536, 501, 501, 494, 512, 491, 522, 545, 479, 488, 518, 509, 518, 485, 514, 501, 494, 491, 529, 467, 530] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[479, 524, 541, 536, 489, 525, 541, 521, 538, 489, 507, 556, 518, 523, 500, 510, 478, 497, 513, 535, 538, 540, 527, 499, 491, 468, 518, 537, 493, 496, 529, 490, 478, 508, 493, 510, 490, 523, 512, 521, 508, 518, 534, 466, 468, 526, 501, 507, 483, 519, 545, 477, 479, 516, 507, 538, 484, 537, 528, 561, 490, 536, 550, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[515, 525, 498, 535, 518, 487, 526, 511, 0, 0, 0, 0, 0, 0, 0, 0, 558, 513, 507, 477, 536, 466, 501, 493, 0, 0, 0, 0, 0, 0, 0, 0, 510, 487, 521, 510, 535, 490, 556, 508, 0, 0, 0, 0, 0, 0, 0, 0, 496, 531, 532, 514, 538, 538, 519, 541, 30, 0, 0, 0, 0, 0, 0, 0, 453, 517, 510, 503, 494, 494, 469, 521, 37, 3, 0, 0, 0, 0, 0, 0, 499, 500, 522, 483, 493, 496, 542, 475, 5, 2, 0, 0, 0, 0, 0, 0, 518, 519, 525, 549, 529, 525, 515, 478, 2, 0, 0, 0, 0, 0, 0, 0, 506, 514, 516, 472, 513, 504, 471, 566, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4051, 4117, 4239, 4001, 4017, 4160, 4068] avg=4096.0000 max=4239 max_rank=3 min=4001 min_rank=4 imbalance_ratio=1.034912 max_load_reduction=28529 max_load_reduction_pct=0.870636 moved_to_echo=16246 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[537, 473, 521, 505, 513, 512, 483, 500, 0, 0, 0, 0, 0, 0, 0, 0, 526, 522, 535, 508, 505, 513, 495, 538, 0, 0, 0, 0, 0, 0, 0, 0, 518, 505, 509, 504, 521, 486, 481, 553, 0, 0, 0, 0, 0, 0, 0, 0, 494, 508, 518, 482, 462, 487, 500, 506, 32, 0, 0, 0, 0, 0, 0, 0, 496, 531, 518, 523, 516, 530, 528, 487, 37, 3, 0, 0, 0, 0, 0, 0, 513, 506, 513, 494, 527, 491, 533, 536, 5, 2, 0, 0, 0, 0, 0, 0, 508, 523, 521, 491, 507, 539, 520, 541, 3, 0, 0, 0, 0, 0, 0, 0, 472, 542, 480, 551, 490, 524, 501, 507, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4044, 4142, 4077, 3989, 4169, 4120, 4153, 4074] avg=4096.0000 max=4169 max_rank=4 min=3989 min_rank=3 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16516 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[520, 507, 505, 512, 520, 552, 503, 509, 0, 0, 0, 0, 0, 0, 0, 0, 509, 517, 512, 497, 498, 506, 534, 502, 0, 0, 0, 0, 0, 0, 0, 0, 533, 493, 526, 515, 500, 473, 525, 514, 0, 0, 0, 0, 0, 0, 0, 0, 496, 499, 514, 498, 506, 520, 519, 526, 31, 0, 0, 0, 0, 0, 0, 0, 500, 521, 515, 492, 518, 504, 512, 527, 36, 3, 0, 0, 0, 0, 0, 0, 482, 493, 520, 511, 545, 472, 547, 532, 5, 2, 0, 0, 0, 0, 0, 0, 475, 480, 480, 540, 509, 544, 526, 530, 2, 0, 0, 0, 0, 0, 0, 0, 527, 509, 502, 480, 526, 509, 518, 477, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4075, 4079, 4109, 4128, 4109, 4086, 4054] avg=4096.0000 max=4128 max_rank=0 min=4054 min_rank=7 imbalance_ratio=1.007812 max_load_reduction=28640 max_load_reduction_pct=0.874023 moved_to_echo=16377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[518, 510, 515, 498, 512, 530, 521, 504, 0, 0, 0, 0, 0, 0, 0, 0, 504, 507, 520, 495, 537, 529, 515, 489, 0, 0, 0, 0, 0, 0, 0, 0, 529, 528, 515, 508, 491, 467, 514, 498, 0, 0, 0, 0, 0, 0, 0, 0, 497, 483, 518, 536, 494, 509, 470, 500, 31, 0, 0, 0, 0, 0, 0, 0, 478, 498, 509, 489, 518, 530, 510, 528, 35, 3, 0, 0, 0, 0, 0, 0, 561, 515, 528, 522, 497, 502, 496, 507, 5, 2, 0, 0, 0, 0, 0, 0, 506, 472, 511, 536, 499, 516, 501, 530, 3, 0, 0, 0, 0, 0, 0, 0, 539, 566, 510, 514, 507, 501, 519, 506, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4096, 4050, 4038, 4098, 4135, 4074, 4169] avg=4096.0000 max=4169 max_rank=7 min=4038 min_rank=3 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16476 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[521, 454, 506, 509, 503, 525, 556, 523, 0, 0, 0, 0, 0, 0, 0, 0, 492, 521, 561, 443, 496, 508, 517, 489, 0, 0, 0, 0, 0, 0, 0, 0, 499, 547, 527, 521, 514, 482, 473, 515, 0, 0, 0, 0, 0, 0, 0, 0, 521, 527, 471, 513, 543, 531, 489, 494, 33, 0, 0, 0, 0, 0, 0, 0, 510, 485, 489, 545, 503, 525, 497, 502, 40, 6, 0, 0, 0, 0, 0, 0, 501, 513, 527, 562, 492, 574, 525, 511, 7, 4, 0, 0, 0, 0, 0, 0, 510, 495, 518, 503, 529, 489, 511, 521, 6, 0, 0, 0, 0, 0, 0, 0, 505, 506, 536, 528, 465, 501, 516, 477, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4027, 4078, 4122, 4102, 4216, 4082, 4044] avg=4096.0000 max=4216 max_rank=5 min=4027 min_rank=1 imbalance_ratio=1.029297 max_load_reduction=28552 max_load_reduction_pct=0.871338 moved_to_echo=16444 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[490, 500, 505, 508, 503, 515, 499, 529, 0, 0, 0, 0, 0, 0, 0, 0, 540, 537, 500, 504, 541, 529, 536, 518, 0, 0, 0, 0, 0, 0, 0, 0, 513, 495, 521, 514, 513, 540, 497, 518, 0, 0, 0, 0, 0, 0, 0, 0, 532, 501, 486, 495, 502, 537, 513, 525, 32, 0, 0, 0, 0, 0, 0, 0, 479, 480, 499, 517, 532, 509, 498, 485, 41, 3, 0, 0, 0, 0, 0, 0, 495, 514, 549, 463, 472, 535, 503, 487, 5, 2, 0, 0, 0, 0, 0, 0, 490, 536, 535, 479, 510, 524, 481, 516, 2, 0, 0, 0, 0, 0, 0, 0, 485, 504, 520, 526, 526, 487, 559, 526, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4049, 4205, 4111, 4123, 4043, 4025, 4073, 4139] avg=4096.0000 max=4205 max_rank=1 min=4025 min_rank=5 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16280 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[532, 497, 528, 512, 460, 511, 536, 510, 0, 0, 0, 0, 0, 0, 0, 0, 523, 536, 486, 479, 501, 515, 514, 515, 0, 0, 0, 0, 0, 0, 0, 0, 546, 533, 550, 532, 495, 481, 515, 515, 0, 0, 0, 0, 0, 0, 0, 0, 530, 515, 485, 509, 490, 531, 500, 503, 30, 0, 0, 0, 0, 0, 0, 0, 476, 507, 527, 513, 513, 558, 510, 508, 36, 3, 0, 0, 0, 0, 0, 0, 538, 504, 518, 536, 501, 501, 494, 512, 5, 2, 0, 0, 0, 0, 0, 0, 491, 522, 545, 479, 488, 518, 509, 518, 3, 0, 0, 0, 0, 0, 0, 0, 485, 514, 501, 494, 491, 529, 467, 530, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4069, 4167, 4093, 4151, 4111, 4073, 4018] avg=4096.0000 max=4167 max_rank=2 min=4018 min_rank=7 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[479, 524, 526, 536, 489, 525, 541, 521, 0, 0, 0, 0, 0, 0, 0, 0, 538, 489, 507, 518, 518, 523, 500, 510, 0, 0, 0, 0, 0, 0, 0, 0, 478, 497, 513, 535, 538, 502, 527, 499, 0, 0, 0, 0, 0, 0, 0, 0, 491, 468, 518, 537, 493, 496, 529, 490, 33, 0, 0, 0, 0, 0, 0, 0, 478, 508, 493, 510, 490, 523, 512, 521, 38, 3, 0, 0, 0, 0, 0, 0, 508, 518, 534, 466, 468, 526, 501, 507, 5, 2, 0, 0, 0, 0, 0, 0, 483, 519, 545, 477, 479, 516, 507, 538, 3, 0, 0, 0, 0, 0, 0, 0, 484, 537, 528, 561, 490, 536, 550, 509, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4141, 4103, 4089, 4055, 4076, 4035, 4067, 4202] avg=4096.0000 max=4202 max_rank=7 min=4035 min_rank=5 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3968, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 3968, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 3968, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[485, 507, 506, 524, 507, 515, 506, 503, 498, 577, 495, 491, 502, 527, 551, 532, 498, 520, 525, 505, 519, 486, 537, 511, 525, 529, 502, 506, 493, 525, 491, 505, 519, 509, 524, 461, 564, 530, 492, 474, 506, 524, 518, 530, 525, 526, 525, 535, 494, 492, 494, 523, 513, 512, 473, 506, 540, 505, 504, 504, 514, 502, 503, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[500, 542, 502, 518, 503, 539, 507, 529, 527, 517, 527, 534, 497, 532, 532, 510, 518, 465, 494, 509, 497, 471, 515, 509, 517, 481, 527, 533, 506, 545, 525, 510, 523, 524, 529, 535, 511, 497, 515, 520, 507, 495, 514, 511, 531, 528, 500, 496, 516, 508, 547, 487, 501, 486, 510, 484, 509, 516, 502, 516, 527, 508, 490, 487] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[528, 508, 514, 507, 513, 499, 509, 482, 526, 509, 500, 493, 512, 530, 495, 456, 494, 493, 532, 525, 510, 482, 485, 550, 471, 509, 496, 525, 499, 530, 500, 521, 511, 533, 484, 548, 573, 502, 551, 500, 540, 529, 509, 537, 531, 483, 462, 546, 510, 509, 495, 546, 499, 553, 525, 511, 530, 496, 527, 496, 480, 527, 517, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[515, 493, 510, 498, 507, 540, 477, 519, 514, 497, 509, 508, 490, 526, 524, 502, 505, 503, 525, 499, 513, 525, 515, 523, 513, 495, 523, 494, 541, 536, 495, 526, 509, 476, 521, 538, 529, 503, 482, 547, 565, 490, 519, 554, 514, 506, 514, 511, 480, 512, 507, 526, 486, 516, 514, 532, 556, 483, 516, 469, 520, 475, 518, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[510, 490, 502, 528, 520, 481, 541, 499, 517, 522, 537, 497, 513, 521, 514, 508, 495, 491, 510, 491, 508, 497, 496, 491, 500, 446, 480, 514, 527, 545, 490, 512, 568, 506, 547, 537, 460, 535, 518, 529, 511, 498, 505, 529, 501, 523, 563, 521, 516, 511, 529, 509, 520, 487, 529, 493, 545, 527, 523, 501, 513, 486, 519, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[554, 537, 503, 552, 510, 501, 523, 518, 504, 527, 480, 500, 479, 506, 518, 513, 526, 440, 535, 492, 489, 518, 529, 522, 521, 514, 510, 525, 530, 497, 526, 524, 490, 514, 513, 554, 494, 500, 500, 519, 496, 519, 466, 510, 518, 558, 507, 531, 480, 541, 533, 558, 475, 524, 520, 521, 513, 521, 485, 493, 503, 496, 486, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[517, 522, 545, 535, 518, 487, 522, 524, 507, 538, 477, 498, 463, 508, 539, 484, 514, 561, 545, 537, 499, 523, 506, 480, 511, 477, 481, 494, 514, 466, 555, 501, 533, 499, 517, 508, 489, 500, 484, 485, 563, 513, 508, 493, 498, 488, 558, 550, 494, 490, 523, 531, 530, 525, 566, 508, 492, 502, 479, 501, 513, 519, 509, 552] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[515, 521, 509, 516, 514, 514, 506, 522, 502, 500, 527, 512, 530, 517, 501, 479, 500, 529, 486, 487, 482, 486, 475, 509, 536, 520, 546, 527, 472, 537, 497, 507, 491, 480, 519, 504, 524, 489, 490, 503, 539, 497, 489, 523, 531, 511, 526, 521, 513, 562, 515, 532, 538, 539, 523, 507, 543, 512, 488, 473, 522, 519, 531, 533] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[500, 542, 502, 507, 503, 539, 507, 529, 0, 0, 0, 0, 0, 0, 0, 0, 527, 517, 527, 534, 497, 532, 532, 510, 11, 0, 0, 0, 0, 0, 0, 0, 518, 465, 494, 509, 497, 471, 515, 509, 43, 0, 0, 0, 0, 0, 0, 0, 517, 481, 527, 533, 506, 545, 525, 510, 8, 0, 0, 0, 0, 0, 0, 0, 523, 524, 529, 527, 511, 497, 515, 520, 0, 0, 0, 0, 0, 0, 0, 0, 463, 495, 514, 511, 531, 528, 500, 496, 0, 0, 0, 0, 0, 0, 0, 0, 516, 508, 547, 471, 501, 486, 510, 484, 0, 0, 0, 0, 0, 0, 0, 0, 509, 516, 502, 516, 527, 508, 490, 487, 16, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4129, 4187, 4021, 4152, 4146, 4038, 4023, 4072] avg=4096.0000 max=4187 max_rank=1 min=4021 min_rank=2 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16279 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[515, 493, 510, 488, 507, 540, 477, 519, 0, 0, 0, 0, 0, 0, 0, 0, 514, 497, 509, 508, 490, 526, 524, 502, 10, 0, 0, 0, 0, 0, 0, 0, 505, 503, 525, 499, 513, 525, 515, 523, 48, 0, 0, 0, 0, 0, 0, 0, 513, 495, 523, 494, 541, 536, 495, 526, 8, 0, 0, 0, 0, 0, 0, 0, 509, 476, 521, 530, 529, 503, 482, 547, 0, 0, 0, 0, 0, 0, 0, 0, 515, 490, 519, 554, 514, 506, 514, 511, 0, 0, 0, 0, 0, 0, 0, 0, 480, 512, 507, 509, 486, 516, 514, 532, 0, 0, 0, 0, 0, 0, 0, 0, 556, 483, 516, 469, 520, 475, 518, 520, 17, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4049, 4080, 4156, 4131, 4097, 4123, 4056, 4076] avg=4096.0000 max=4156 max_rank=2 min=4049 min_rank=0 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16352 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[528, 508, 514, 483, 513, 499, 509, 482, 0, 0, 0, 0, 0, 0, 0, 0, 526, 509, 500, 493, 512, 530, 495, 456, 14, 0, 0, 0, 0, 0, 0, 0, 494, 493, 532, 525, 510, 482, 485, 550, 48, 0, 0, 0, 0, 0, 0, 0, 471, 509, 496, 525, 499, 530, 500, 521, 12, 7, 0, 0, 0, 0, 0, 0, 511, 533, 484, 536, 573, 502, 551, 500, 0, 0, 0, 0, 0, 0, 0, 0, 486, 529, 509, 537, 531, 483, 462, 546, 0, 0, 0, 0, 0, 0, 0, 0, 510, 509, 495, 524, 499, 553, 525, 511, 0, 0, 0, 0, 0, 0, 0, 0, 530, 496, 527, 496, 480, 527, 517, 505, 22, 6, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4036, 4035, 4119, 4070, 4190, 4083, 4126, 4109] avg=4096.0000 max=4190 max_rank=4 min=4035 min_rank=1 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16508 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[485, 507, 506, 513, 507, 515, 506, 503, 0, 0, 0, 0, 0, 0, 0, 0, 498, 577, 495, 491, 502, 527, 551, 532, 11, 0, 0, 0, 0, 0, 0, 0, 498, 520, 525, 505, 519, 486, 537, 511, 43, 0, 0, 0, 0, 0, 0, 0, 525, 529, 502, 506, 493, 525, 491, 505, 7, 0, 0, 0, 0, 0, 0, 0, 519, 509, 524, 454, 564, 530, 492, 474, 0, 0, 0, 0, 0, 0, 0, 0, 462, 524, 518, 530, 525, 526, 525, 535, 0, 0, 0, 0, 0, 0, 0, 0, 494, 492, 494, 506, 513, 512, 473, 506, 0, 0, 0, 0, 0, 0, 0, 0, 540, 505, 504, 504, 514, 502, 503, 524, 17, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4042, 4184, 4144, 4083, 4066, 4145, 3990, 4114] avg=4096.0000 max=4184 max_rank=1 min=3990 min_rank=6 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16315 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[510, 490, 502, 517, 520, 481, 541, 499, 0, 0, 0, 0, 0, 0, 0, 0, 517, 522, 537, 497, 513, 521, 514, 508, 11, 0, 0, 0, 0, 0, 0, 0, 495, 491, 510, 491, 508, 497, 496, 491, 43, 0, 0, 0, 0, 0, 0, 0, 500, 446, 480, 514, 527, 545, 490, 512, 8, 0, 0, 0, 0, 0, 0, 0, 568, 506, 547, 529, 460, 535, 518, 529, 0, 0, 0, 0, 0, 0, 0, 0, 467, 498, 505, 529, 501, 523, 563, 521, 0, 0, 0, 0, 0, 0, 0, 0, 516, 511, 529, 492, 520, 487, 529, 493, 0, 0, 0, 0, 0, 0, 0, 0, 545, 527, 523, 501, 513, 486, 519, 516, 17, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4140, 4022, 4022, 4192, 4107, 4077, 4148] avg=4096.0000 max=4192 max_rank=4 min=4022 min_rank=2 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16524 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[554, 537, 503, 541, 510, 501, 523, 518, 0, 0, 0, 0, 0, 0, 0, 0, 504, 527, 480, 500, 479, 506, 518, 513, 11, 0, 0, 0, 0, 0, 0, 0, 526, 440, 535, 492, 489, 518, 529, 522, 42, 0, 0, 0, 0, 0, 0, 0, 521, 514, 510, 525, 530, 497, 526, 524, 8, 0, 0, 0, 0, 0, 0, 0, 490, 514, 513, 546, 494, 500, 500, 519, 0, 0, 0, 0, 0, 0, 0, 0, 453, 519, 466, 510, 518, 558, 507, 531, 0, 0, 0, 0, 0, 0, 0, 0, 480, 541, 533, 540, 475, 524, 520, 521, 0, 0, 0, 0, 0, 0, 0, 0, 513, 521, 485, 493, 503, 496, 486, 507, 18, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4187, 4038, 4093, 4155, 4076, 4062, 4134, 4023] avg=4096.0000 max=4187 max_rank=0 min=4023 min_rank=7 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16295 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[517, 522, 545, 524, 518, 487, 522, 524, 0, 0, 0, 0, 0, 0, 0, 0, 507, 538, 477, 498, 463, 508, 539, 484, 11, 0, 0, 0, 0, 0, 0, 0, 514, 561, 545, 537, 499, 523, 506, 480, 48, 0, 0, 0, 0, 0, 0, 0, 511, 477, 481, 494, 514, 466, 555, 501, 7, 0, 0, 0, 0, 0, 0, 0, 533, 499, 517, 501, 489, 500, 484, 485, 0, 0, 0, 0, 0, 0, 0, 0, 513, 513, 508, 493, 498, 488, 558, 550, 0, 0, 0, 0, 0, 0, 0, 0, 494, 490, 523, 514, 530, 525, 566, 508, 0, 0, 0, 0, 0, 0, 0, 0, 492, 502, 479, 501, 513, 519, 509, 552, 17, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4159, 4025, 4213, 4006, 4008, 4121, 4150, 4086] avg=4096.0000 max=4213 max_rank=2 min=4006 min_rank=3 imbalance_ratio=1.028564 max_load_reduction=28555 max_load_reduction_pct=0.871429 moved_to_echo=16365 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[515, 521, 509, 505, 514, 514, 506, 522, 0, 0, 0, 0, 0, 0, 0, 0, 502, 500, 527, 512, 530, 517, 501, 479, 11, 0, 0, 0, 0, 0, 0, 0, 500, 529, 486, 487, 482, 486, 475, 509, 46, 0, 0, 0, 0, 0, 0, 0, 536, 520, 546, 527, 472, 537, 497, 507, 7, 0, 0, 0, 0, 0, 0, 0, 491, 480, 519, 497, 524, 489, 490, 503, 0, 0, 0, 0, 0, 0, 0, 0, 491, 497, 489, 523, 531, 511, 526, 521, 0, 0, 0, 0, 0, 0, 0, 0, 513, 562, 515, 515, 538, 539, 523, 507, 0, 0, 0, 0, 0, 0, 0, 0, 543, 512, 488, 473, 522, 519, 531, 533, 17, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4079, 4000, 4149, 3993, 4089, 4212, 4140] avg=4096.0000 max=4212 max_rank=6 min=3993 min_rank=4 imbalance_ratio=1.028320 max_load_reduction=28556 max_load_reduction_pct=0.871460 moved_to_echo=16434 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4352, 4096, 4096, 3968, 4096, 4096, 4096, 4224, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3968, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[506, 483, 484, 496, 481, 524, 509, 523, 486, 516, 484, 516, 528, 557, 502, 517, 525, 508, 528, 503, 525, 545, 481, 518, 523, 505, 489, 499, 499, 557, 514, 508, 521, 527, 541, 475, 492, 462, 508, 502, 519, 518, 511, 496, 524, 498, 483, 517, 552, 529, 543, 500, 492, 532, 504, 541, 525, 511, 513, 496, 546, 485, 528, 538] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[543, 538, 529, 532, 508, 522, 520, 524, 501, 535, 496, 482, 513, 492, 489, 567, 524, 514, 519, 520, 498, 487, 522, 477, 557, 509, 511, 497, 501, 526, 532, 501, 492, 510, 510, 501, 501, 534, 518, 501, 455, 530, 527, 523, 502, 492, 493, 496, 532, 490, 540, 534, 551, 520, 506, 490, 449, 487, 535, 524, 504, 535, 520, 480] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[544, 540, 517, 509, 507, 514, 553, 480, 551, 534, 480, 473, 522, 502, 511, 508, 525, 541, 502, 558, 529, 501, 506, 518, 492, 501, 547, 517, 512, 479, 515, 494, 506, 467, 525, 511, 520, 512, 539, 523, 462, 458, 508, 514, 493, 484, 501, 491, 477, 494, 509, 495, 533, 524, 550, 543, 528, 501, 490, 523, 525, 505, 525, 550] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[512, 506, 531, 524, 521, 496, 560, 513, 498, 501, 456, 488, 516, 529, 483, 517, 476, 513, 561, 573, 568, 468, 527, 519, 521, 484, 529, 466, 501, 540, 517, 504, 517, 507, 548, 500, 540, 482, 501, 499, 513, 496, 483, 502, 466, 542, 519, 513, 497, 531, 525, 509, 521, 535, 511, 537, 499, 502, 490, 521, 542, 510, 494, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[508, 509, 528, 534, 502, 525, 522, 530, 476, 516, 515, 508, 520, 495, 527, 524, 498, 515, 507, 490, 507, 491, 538, 515, 495, 499, 508, 509, 513, 492, 516, 504, 534, 498, 518, 545, 559, 535, 480, 518, 534, 493, 499, 529, 512, 482, 505, 499, 538, 487, 489, 507, 535, 487, 526, 509, 505, 497, 551, 517, 504, 554, 490, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[513, 530, 472, 504, 549, 545, 521, 488, 551, 540, 529, 531, 448, 528, 510, 575, 497, 493, 507, 501, 483, 532, 497, 481, 542, 532, 508, 503, 526, 487, 506, 499, 513, 514, 532, 530, 521, 508, 524, 521, 498, 508, 520, 483, 513, 495, 516, 503, 500, 497, 535, 526, 523, 527, 485, 509, 514, 494, 511, 476, 529, 513, 527, 475] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[565, 504, 500, 507, 508, 531, 485, 494, 520, 516, 513, 527, 522, 482, 571, 541, 494, 531, 510, 471, 511, 496, 501, 551, 520, 536, 517, 529, 538, 546, 494, 535, 490, 475, 500, 548, 494, 495, 516, 486, 500, 524, 509, 484, 496, 556, 529, 514, 527, 512, 534, 492, 494, 510, 538, 472, 510, 530, 502, 487, 524, 475, 483, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[518, 533, 476, 505, 478, 553, 550, 533, 548, 542, 517, 528, 526, 511, 510, 503, 496, 531, 480, 533, 504, 490, 518, 524, 513, 476, 497, 479, 498, 485, 528, 498, 526, 521, 515, 523, 563, 494, 518, 486, 513, 467, 512, 507, 479, 515, 496, 480, 524, 514, 541, 544, 525, 498, 550, 506, 531, 495, 489, 508, 534, 487, 505, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[506, 483, 484, 496, 481, 524, 474, 523, 0, 0, 0, 0, 0, 0, 0, 0, 486, 516, 484, 516, 528, 557, 502, 500, 0, 0, 0, 0, 0, 0, 0, 0, 525, 508, 528, 503, 525, 545, 481, 518, 0, 0, 0, 0, 0, 0, 0, 0, 523, 505, 489, 499, 499, 557, 514, 508, 7, 2, 0, 0, 0, 0, 0, 0, 521, 527, 541, 475, 490, 462, 508, 502, 0, 0, 0, 0, 0, 0, 0, 0, 519, 518, 511, 496, 524, 498, 483, 517, 43, 27, 0, 0, 0, 0, 0, 0, 552, 529, 500, 500, 492, 532, 504, 541, 0, 0, 0, 0, 0, 0, 0, 0, 525, 511, 513, 496, 546, 485, 528, 538, 10, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3971, 4089, 4133, 4103, 4026, 4136, 4150, 4160] avg=4096.0000 max=4160 max_rank=7 min=3971 min_rank=0 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16472 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[513, 530, 472, 504, 549, 545, 485, 488, 0, 0, 0, 0, 0, 0, 0, 0, 551, 540, 529, 531, 448, 528, 510, 556, 0, 0, 0, 0, 0, 0, 0, 0, 497, 493, 507, 501, 483, 532, 497, 481, 0, 0, 0, 0, 0, 0, 0, 0, 542, 532, 508, 503, 526, 487, 506, 499, 8, 2, 0, 0, 0, 0, 0, 0, 513, 514, 532, 530, 519, 508, 524, 521, 0, 0, 0, 0, 0, 0, 0, 0, 498, 508, 520, 483, 513, 495, 516, 503, 43, 28, 0, 0, 0, 0, 0, 0, 500, 497, 492, 526, 523, 527, 485, 509, 0, 0, 0, 0, 0, 0, 0, 0, 514, 494, 511, 476, 529, 513, 527, 475, 11, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4193, 3991, 4113, 4161, 4107, 4059, 4058] avg=4096.0000 max=4193 max_rank=1 min=3991 min_rank=2 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[544, 540, 517, 509, 507, 514, 506, 480, 0, 0, 0, 0, 0, 0, 0, 0, 551, 534, 480, 473, 522, 502, 511, 482, 0, 0, 0, 0, 0, 0, 0, 0, 525, 541, 502, 554, 529, 501, 506, 518, 0, 0, 0, 0, 0, 0, 0, 0, 492, 501, 547, 517, 512, 479, 515, 494, 12, 7, 4, 0, 0, 0, 0, 0, 506, 467, 525, 511, 513, 512, 539, 523, 0, 0, 0, 0, 0, 0, 0, 0, 462, 458, 508, 514, 493, 484, 501, 491, 44, 33, 0, 0, 0, 0, 0, 0, 477, 494, 465, 495, 533, 524, 550, 543, 0, 0, 0, 0, 0, 0, 0, 0, 528, 501, 490, 523, 525, 505, 525, 550, 14, 14, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4117, 4055, 4176, 4080, 4096, 3988, 4081, 4175] avg=4096.0000 max=4176 max_rank=2 min=3988 min_rank=5 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16340 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[508, 509, 528, 534, 502, 525, 486, 530, 0, 0, 0, 0, 0, 0, 0, 0, 476, 516, 515, 508, 520, 495, 527, 506, 0, 0, 0, 0, 0, 0, 0, 0, 498, 515, 507, 490, 507, 491, 538, 515, 0, 0, 0, 0, 0, 0, 0, 0, 495, 499, 508, 509, 513, 492, 516, 504, 8, 3, 0, 0, 0, 0, 0, 0, 534, 498, 518, 545, 556, 535, 480, 518, 0, 0, 0, 0, 0, 0, 0, 0, 534, 493, 499, 529, 512, 482, 505, 499, 39, 28, 0, 0, 0, 0, 0, 0, 538, 487, 450, 507, 535, 487, 526, 509, 0, 0, 0, 0, 0, 0, 0, 0, 505, 497, 551, 517, 504, 554, 490, 496, 10, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4122, 4063, 4061, 4047, 4184, 4120, 4039, 4132] avg=4096.0000 max=4184 max_rank=4 min=4039 min_rank=6 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16475 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[512, 506, 531, 524, 521, 496, 521, 513, 0, 0, 0, 0, 0, 0, 0, 0, 498, 501, 456, 488, 516, 529, 483, 500, 0, 0, 0, 0, 0, 0, 0, 0, 476, 513, 561, 573, 568, 468, 527, 519, 0, 0, 0, 0, 0, 0, 0, 0, 521, 484, 529, 466, 501, 540, 517, 504, 7, 3, 0, 0, 0, 0, 0, 0, 517, 507, 548, 500, 537, 482, 501, 499, 0, 0, 0, 0, 0, 0, 0, 0, 513, 496, 483, 502, 466, 542, 519, 513, 42, 30, 0, 0, 0, 0, 0, 0, 497, 531, 483, 509, 521, 535, 511, 537, 0, 0, 0, 0, 0, 0, 0, 0, 499, 502, 490, 521, 542, 510, 494, 498, 10, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4124, 3971, 4205, 4072, 4091, 4106, 4124, 4075] avg=4096.0000 max=4205 max_rank=2 min=3971 min_rank=1 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16396 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[565, 504, 500, 507, 508, 531, 451, 494, 0, 0, 0, 0, 0, 0, 0, 0, 520, 516, 513, 527, 522, 482, 571, 522, 0, 0, 0, 0, 0, 0, 0, 0, 494, 531, 510, 471, 511, 496, 501, 551, 0, 0, 0, 0, 0, 0, 0, 0, 520, 536, 517, 529, 538, 546, 494, 535, 8, 2, 0, 0, 0, 0, 0, 0, 490, 475, 500, 548, 492, 495, 516, 486, 0, 0, 0, 0, 0, 0, 0, 0, 500, 524, 509, 484, 496, 556, 529, 514, 43, 26, 0, 0, 0, 0, 0, 0, 527, 512, 491, 492, 494, 510, 538, 472, 0, 0, 0, 0, 0, 0, 0, 0, 510, 530, 502, 487, 524, 475, 483, 496, 11, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4173, 4065, 4225, 4002, 4181, 4036, 4026] avg=4096.0000 max=4225 max_rank=3 min=4002 min_rank=4 imbalance_ratio=1.031494 max_load_reduction=28543 max_load_reduction_pct=0.871063 moved_to_echo=16245 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[518, 533, 476, 505, 478, 553, 512, 533, 0, 0, 0, 0, 0, 0, 0, 0, 548, 542, 517, 528, 526, 511, 510, 486, 0, 0, 0, 0, 0, 0, 0, 0, 496, 531, 480, 533, 504, 490, 518, 524, 0, 0, 0, 0, 0, 0, 0, 0, 513, 476, 497, 479, 498, 485, 528, 498, 7, 3, 0, 0, 0, 0, 0, 0, 526, 521, 515, 523, 560, 494, 518, 486, 0, 0, 0, 0, 0, 0, 0, 0, 513, 467, 512, 507, 479, 515, 496, 480, 43, 29, 0, 0, 0, 0, 0, 0, 524, 514, 498, 544, 525, 498, 550, 506, 0, 0, 0, 0, 0, 0, 0, 0, 531, 495, 489, 508, 534, 487, 505, 521, 10, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4168, 4076, 3984, 4143, 4041, 4159, 4089] avg=4096.0000 max=4168 max_rank=1 min=3984 min_rank=3 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16432 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[543, 538, 529, 532, 508, 522, 484, 524, 0, 0, 0, 0, 0, 0, 0, 0, 501, 535, 496, 482, 513, 492, 489, 548, 0, 0, 0, 0, 0, 0, 0, 0, 524, 514, 519, 520, 498, 487, 522, 477, 0, 0, 0, 0, 0, 0, 0, 0, 557, 509, 511, 497, 501, 526, 532, 501, 8, 2, 0, 0, 0, 0, 0, 0, 492, 510, 510, 501, 499, 534, 518, 501, 0, 0, 0, 0, 0, 0, 0, 0, 455, 530, 527, 523, 502, 492, 493, 496, 43, 28, 0, 0, 0, 0, 0, 0, 532, 490, 497, 534, 551, 520, 506, 490, 0, 0, 0, 0, 0, 0, 0, 0, 449, 487, 535, 524, 504, 535, 520, 480, 11, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4180, 4056, 4061, 4144, 4065, 4089, 4120, 4053] avg=4096.0000 max=4180 max_rank=0 min=4053 min_rank=7 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16327 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 3968, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 3968, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 384, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[519, 469, 510, 552, 532, 518, 542, 525, 513, 522, 532, 515, 510, 500, 480, 522, 537, 508, 489, 525, 547, 518, 518, 530, 511, 506, 500, 488, 502, 530, 514, 542, 513, 479, 510, 499, 507, 476, 489, 503, 475, 500, 504, 515, 561, 535, 442, 497, 514, 476, 536, 518, 495, 533, 486, 538, 539, 503, 498, 493, 502, 528, 553, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[533, 500, 522, 511, 515, 493, 541, 522, 498, 507, 525, 516, 525, 491, 494, 502, 529, 538, 506, 521, 505, 539, 531, 512, 557, 507, 532, 557, 509, 509, 508, 506, 503, 495, 498, 473, 513, 498, 523, 487, 454, 514, 518, 496, 498, 538, 478, 497, 509, 481, 538, 528, 534, 539, 478, 499, 556, 527, 474, 508, 490, 508, 504, 551] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[515, 513, 539, 523, 530, 539, 509, 506, 495, 494, 518, 490, 499, 490, 533, 516, 518, 510, 520, 523, 472, 501, 538, 511, 536, 546, 515, 504, 520, 532, 496, 527, 482, 493, 518, 512, 500, 523, 527, 506, 543, 523, 501, 515, 496, 542, 509, 524, 530, 488, 504, 544, 480, 475, 510, 530, 538, 505, 483, 497, 498, 484, 491, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[523, 525, 493, 480, 535, 512, 519, 506, 496, 516, 497, 513, 508, 484, 490, 503, 514, 506, 508, 509, 501, 516, 518, 517, 529, 524, 533, 530, 529, 515, 512, 536, 530, 528, 474, 519, 546, 489, 533, 531, 487, 507, 527, 497, 530, 520, 533, 511, 477, 527, 499, 545, 518, 507, 495, 511, 521, 491, 531, 525, 488, 476, 490, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[513, 513, 504, 487, 490, 544, 513, 527, 510, 509, 532, 497, 513, 484, 515, 539, 536, 516, 532, 525, 510, 513, 523, 544, 489, 525, 511, 505, 514, 505, 475, 503, 527, 492, 512, 509, 536, 484, 525, 510, 496, 519, 510, 503, 532, 531, 517, 516, 543, 514, 518, 472, 490, 526, 515, 456, 525, 504, 512, 536, 504, 503, 507, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[546, 470, 494, 492, 524, 481, 476, 538, 506, 490, 498, 499, 512, 532, 537, 513, 522, 514, 502, 519, 521, 519, 522, 501, 504, 505, 512, 516, 526, 513, 565, 496, 495, 528, 501, 504, 517, 531, 506, 512, 557, 536, 488, 524, 510, 524, 501, 514, 554, 502, 493, 484, 474, 523, 487, 502, 495, 532, 540, 520, 502, 513, 489, 545] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[488, 475, 532, 490, 468, 508, 498, 539, 514, 507, 465, 527, 545, 514, 469, 521, 513, 532, 517, 532, 512, 537, 520, 480, 538, 524, 520, 460, 538, 514, 505, 513, 533, 512, 533, 530, 550, 476, 467, 538, 544, 507, 541, 539, 499, 512, 521, 526, 533, 518, 491, 536, 523, 490, 507, 541, 504, 503, 462, 494, 524, 476, 498, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[501, 489, 478, 493, 558, 505, 539, 505, 535, 520, 527, 507, 543, 526, 505, 505, 500, 514, 515, 475, 518, 490, 507, 506, 482, 513, 463, 503, 536, 517, 516, 521, 536, 538, 500, 496, 516, 521, 512, 495, 511, 536, 482, 556, 542, 533, 531, 501, 513, 498, 506, 517, 492, 486, 524, 523, 500, 503, 534, 512, 490, 529, 505, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[519, 469, 510, 552, 532, 518, 542, 525, 2, 0, 0, 0, 0, 0, 0, 0, 513, 522, 532, 515, 510, 500, 480, 522, 9, 9, 0, 0, 0, 0, 0, 0, 537, 508, 489, 525, 547, 518, 488, 530, 0, 0, 0, 0, 0, 0, 0, 0, 511, 506, 500, 488, 476, 530, 514, 542, 0, 0, 0, 0, 0, 0, 0, 0, 513, 479, 510, 499, 507, 476, 489, 503, 18, 0, 0, 0, 0, 0, 0, 0, 475, 500, 504, 515, 561, 514, 442, 497, 0, 0, 0, 0, 0, 0, 0, 0, 514, 476, 536, 518, 495, 533, 486, 538, 21, 0, 0, 0, 0, 0, 0, 0, 539, 503, 498, 493, 502, 528, 553, 525, 17, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4169, 4112, 4142, 4067, 3994, 4008, 4117, 4159] avg=4096.0000 max=4169 max_rank=0 min=3994 min_rank=4 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16278 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[533, 500, 522, 511, 515, 493, 541, 522, 2, 0, 0, 0, 0, 0, 0, 0, 498, 507, 525, 516, 525, 491, 494, 502, 9, 10, 0, 0, 0, 0, 0, 0, 529, 538, 506, 521, 505, 539, 499, 512, 0, 0, 0, 0, 0, 0, 0, 0, 557, 507, 532, 557, 483, 509, 508, 506, 0, 0, 0, 0, 0, 0, 0, 0, 503, 495, 498, 473, 513, 498, 523, 487, 18, 0, 0, 0, 0, 0, 0, 0, 454, 514, 518, 496, 498, 517, 478, 497, 0, 0, 0, 0, 0, 0, 0, 0, 509, 481, 538, 528, 534, 539, 478, 499, 22, 0, 0, 0, 0, 0, 0, 0, 556, 527, 474, 508, 490, 508, 504, 551, 17, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4139, 4077, 4149, 4159, 4008, 3972, 4128, 4136] avg=4096.0000 max=4159 max_rank=3 min=3972 min_rank=5 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16244 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[515, 513, 539, 523, 530, 539, 509, 506, 2, 0, 0, 0, 0, 0, 0, 0, 495, 494, 518, 490, 499, 490, 533, 516, 9, 10, 0, 0, 0, 0, 0, 0, 518, 510, 520, 523, 472, 501, 506, 511, 0, 0, 0, 0, 0, 0, 0, 0, 536, 546, 515, 504, 494, 532, 496, 527, 0, 0, 0, 0, 0, 0, 0, 0, 482, 493, 518, 512, 500, 523, 527, 506, 19, 0, 0, 0, 0, 0, 0, 0, 543, 523, 501, 515, 496, 520, 509, 524, 0, 0, 0, 0, 0, 0, 0, 0, 530, 488, 504, 544, 480, 475, 510, 530, 22, 0, 0, 0, 0, 0, 0, 0, 538, 505, 483, 497, 498, 484, 491, 519, 17, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4176, 4054, 4061, 4150, 4080, 4131, 4083, 4033] avg=4096.0000 max=4176 max_rank=0 min=4033 min_rank=7 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16327 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[523, 525, 493, 480, 535, 512, 519, 506, 2, 0, 0, 0, 0, 0, 0, 0, 496, 516, 497, 513, 508, 484, 490, 503, 10, 9, 0, 0, 0, 0, 0, 0, 514, 506, 508, 509, 501, 516, 488, 517, 0, 0, 0, 0, 0, 0, 0, 0, 529, 524, 533, 530, 502, 515, 512, 536, 0, 0, 0, 0, 0, 0, 0, 0, 530, 528, 474, 519, 546, 489, 533, 531, 18, 0, 0, 0, 0, 0, 0, 0, 487, 507, 527, 497, 530, 499, 533, 511, 0, 0, 0, 0, 0, 0, 0, 0, 477, 527, 499, 545, 518, 507, 495, 511, 21, 0, 0, 0, 0, 0, 0, 0, 521, 491, 531, 525, 488, 476, 490, 508, 17, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4026, 4059, 4181, 4168, 4091, 4100, 4048] avg=4096.0000 max=4181 max_rank=3 min=4026 min_rank=1 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16407 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[513, 513, 504, 487, 490, 544, 513, 527, 5, 0, 0, 0, 0, 0, 0, 0, 510, 509, 532, 497, 513, 484, 515, 539, 13, 14, 0, 0, 0, 0, 0, 0, 536, 516, 532, 525, 510, 513, 483, 544, 0, 0, 0, 0, 0, 0, 0, 0, 489, 525, 511, 505, 480, 505, 475, 503, 0, 0, 0, 0, 0, 0, 0, 0, 527, 492, 512, 509, 536, 484, 525, 510, 22, 0, 0, 0, 0, 0, 0, 0, 496, 519, 510, 503, 532, 501, 517, 516, 0, 0, 0, 0, 0, 0, 0, 0, 543, 514, 518, 472, 490, 526, 515, 456, 26, 0, 0, 0, 0, 0, 0, 0, 525, 504, 512, 536, 504, 503, 507, 508, 21, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4126, 4159, 3993, 4117, 4094, 4060, 4123] avg=4096.0000 max=4159 max_rank=2 min=3993 min_rank=3 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16394 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[488, 475, 532, 490, 468, 508, 498, 539, 2, 0, 0, 0, 0, 0, 0, 0, 514, 507, 465, 527, 545, 514, 469, 521, 10, 9, 0, 0, 0, 0, 0, 0, 513, 532, 517, 532, 512, 537, 490, 480, 0, 0, 0, 0, 0, 0, 0, 0, 538, 524, 520, 460, 510, 514, 505, 513, 0, 0, 0, 0, 0, 0, 0, 0, 533, 512, 533, 530, 550, 476, 467, 538, 18, 0, 0, 0, 0, 0, 0, 0, 544, 507, 541, 539, 499, 491, 521, 526, 0, 0, 0, 0, 0, 0, 0, 0, 533, 518, 491, 536, 523, 490, 507, 541, 21, 0, 0, 0, 0, 0, 0, 0, 504, 503, 462, 494, 524, 476, 498, 525, 18, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4000, 4081, 4113, 4084, 4157, 4168, 4160, 4005] avg=4096.0000 max=4168 max_rank=5 min=4000 min_rank=0 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16490 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[546, 470, 494, 492, 524, 481, 476, 538, 2, 0, 0, 0, 0, 0, 0, 0, 506, 490, 498, 499, 512, 532, 537, 513, 9, 9, 0, 0, 0, 0, 0, 0, 522, 514, 502, 519, 521, 519, 492, 501, 0, 0, 0, 0, 0, 0, 0, 0, 504, 505, 512, 516, 500, 513, 565, 496, 0, 0, 0, 0, 0, 0, 0, 0, 495, 528, 501, 504, 517, 531, 506, 512, 18, 0, 0, 0, 0, 0, 0, 0, 557, 536, 488, 524, 510, 503, 501, 514, 0, 0, 0, 0, 0, 0, 0, 0, 554, 502, 493, 484, 474, 523, 487, 502, 21, 0, 0, 0, 0, 0, 0, 0, 495, 532, 540, 520, 502, 513, 489, 545, 17, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4023, 4105, 4090, 4111, 4112, 4133, 4040, 4154] avg=4096.0000 max=4154 max_rank=7 min=4023 min_rank=0 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16439 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[501, 489, 478, 493, 558, 505, 539, 505, 2, 0, 0, 0, 0, 0, 0, 0, 535, 520, 527, 507, 543, 526, 505, 505, 10, 9, 0, 0, 0, 0, 0, 0, 500, 514, 515, 475, 518, 490, 477, 506, 0, 0, 0, 0, 0, 0, 0, 0, 482, 513, 463, 503, 508, 517, 516, 521, 0, 0, 0, 0, 0, 0, 0, 0, 536, 538, 500, 496, 516, 521, 512, 495, 18, 0, 0, 0, 0, 0, 0, 0, 511, 536, 482, 556, 542, 512, 531, 501, 0, 0, 0, 0, 0, 0, 0, 0, 513, 498, 506, 517, 492, 486, 524, 523, 21, 0, 0, 0, 0, 0, 0, 0, 500, 503, 534, 512, 490, 529, 505, 518, 18, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4070, 4187, 3995, 4023, 4132, 4171, 4080, 4110] avg=4096.0000 max=4187 max_rank=1 min=3995 min_rank=2 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16493 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 3968, 4096, 4096, 4224, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4096, 4096, 3968, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[475, 502, 514, 517, 526, 519, 565, 515, 549, 502, 470, 533, 530, 524, 523, 512, 555, 535, 542, 492, 481, 504, 534, 481, 495, 498, 499, 491, 512, 479, 514, 519, 488, 503, 466, 510, 480, 499, 568, 529, 511, 494, 484, 506, 531, 537, 479, 498, 516, 524, 478, 484, 513, 529, 512, 502, 552, 505, 522, 544, 518, 535, 512, 532] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[487, 508, 504, 490, 490, 522, 521, 518, 539, 482, 513, 484, 521, 540, 496, 520, 531, 483, 481, 520, 481, 526, 501, 518, 499, 544, 516, 509, 494, 492, 538, 517, 494, 495, 516, 487, 537, 523, 509, 538, 549, 501, 473, 514, 501, 532, 549, 521, 493, 496, 525, 533, 531, 514, 485, 509, 556, 514, 497, 512, 520, 535, 508, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[475, 529, 513, 521, 524, 511, 486, 507, 522, 503, 508, 472, 493, 523, 541, 505, 512, 491, 544, 486, 503, 494, 552, 479, 535, 528, 503, 521, 518, 521, 505, 535, 517, 480, 562, 502, 532, 528, 518, 515, 489, 520, 510, 490, 528, 498, 509, 567, 534, 483, 505, 516, 543, 510, 530, 506, 477, 517, 452, 493, 552, 533, 510, 482] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[534, 490, 502, 522, 478, 540, 517, 507, 540, 519, 506, 494, 518, 506, 526, 536, 484, 533, 501, 532, 508, 529, 510, 497, 519, 511, 527, 505, 490, 489, 501, 533, 504, 504, 513, 496, 557, 480, 527, 467, 467, 490, 532, 523, 513, 513, 538, 506, 522, 527, 520, 517, 503, 511, 545, 514, 507, 497, 498, 513, 491, 497, 554, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[521, 514, 509, 492, 513, 532, 509, 507, 520, 482, 535, 537, 535, 535, 491, 509, 548, 522, 559, 470, 495, 515, 517, 527, 526, 516, 517, 517, 510, 528, 499, 489, 539, 509, 544, 534, 518, 503, 481, 493, 537, 481, 527, 468, 493, 517, 520, 527, 515, 506, 498, 505, 519, 518, 510, 504, 491, 492, 491, 487, 487, 539, 512, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[506, 511, 504, 536, 516, 512, 493, 481, 475, 507, 487, 546, 527, 498, 510, 502, 517, 487, 499, 508, 539, 522, 483, 513, 507, 521, 506, 494, 511, 513, 528, 543, 560, 503, 508, 524, 511, 499, 536, 483, 505, 518, 516, 541, 528, 519, 523, 502, 512, 519, 501, 507, 471, 502, 516, 508, 526, 525, 515, 512, 525, 493, 550, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[523, 546, 539, 539, 521, 527, 520, 502, 512, 474, 517, 497, 515, 529, 512, 523, 551, 523, 515, 506, 520, 515, 509, 560, 482, 517, 523, 531, 558, 473, 491, 505, 490, 470, 487, 527, 525, 467, 526, 525, 521, 535, 470, 520, 519, 533, 514, 465, 496, 539, 499, 491, 490, 514, 470, 515, 513, 511, 522, 508, 500, 543, 490, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[505, 511, 512, 503, 444, 539, 516, 495, 509, 500, 535, 500, 486, 516, 493, 504, 503, 532, 444, 546, 530, 549, 513, 487, 528, 543, 513, 515, 503, 520, 484, 487, 522, 510, 489, 569, 516, 488, 515, 514, 494, 505, 533, 532, 500, 505, 528, 533, 492, 506, 530, 561, 524, 498, 521, 513, 520, 495, 547, 508, 497, 537, 508, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[475, 502, 514, 517, 526, 519, 565, 515, 4, 1, 0, 0, 0, 0, 0, 0, 549, 502, 470, 533, 530, 524, 523, 512, 0, 0, 0, 0, 0, 0, 0, 0, 546, 535, 542, 492, 481, 504, 534, 481, 0, 0, 0, 0, 0, 0, 0, 0, 495, 498, 499, 491, 512, 479, 514, 519, 1, 0, 0, 0, 0, 0, 0, 0, 488, 503, 466, 510, 480, 499, 568, 529, 2, 2, 0, 0, 0, 0, 0, 0, 511, 494, 484, 506, 531, 537, 476, 498, 0, 0, 0, 0, 0, 0, 0, 0, 516, 524, 478, 484, 513, 529, 512, 502, 8, 0, 0, 0, 0, 0, 0, 0, 552, 505, 522, 544, 518, 529, 512, 532, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4138, 4143, 4115, 4008, 4047, 4037, 4066, 4214] avg=4096.0000 max=4214 max_rank=7 min=4008 min_rank=3 imbalance_ratio=1.028809 max_load_reduction=28554 max_load_reduction_pct=0.871399 moved_to_echo=16364 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[487, 508, 504, 490, 490, 522, 521, 518, 4, 1, 0, 0, 0, 0, 0, 0, 539, 482, 513, 484, 521, 540, 496, 520, 0, 0, 0, 0, 0, 0, 0, 0, 522, 483, 481, 520, 481, 526, 501, 518, 0, 0, 0, 0, 0, 0, 0, 0, 499, 544, 516, 509, 494, 492, 538, 517, 1, 0, 0, 0, 0, 0, 0, 0, 494, 495, 516, 487, 537, 523, 509, 538, 2, 2, 0, 0, 0, 0, 0, 0, 549, 501, 473, 514, 501, 532, 546, 521, 0, 0, 0, 0, 0, 0, 0, 0, 493, 496, 525, 533, 531, 514, 485, 509, 8, 0, 0, 0, 0, 0, 0, 0, 556, 514, 497, 512, 520, 529, 508, 516, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4045, 4095, 4032, 4110, 4103, 4137, 4094, 4152] avg=4096.0000 max=4152 max_rank=7 min=4032 min_rank=2 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16486 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[475, 529, 513, 521, 524, 511, 486, 507, 4, 0, 0, 0, 0, 0, 0, 0, 522, 503, 508, 472, 493, 523, 541, 505, 0, 0, 0, 0, 0, 0, 0, 0, 504, 491, 544, 486, 503, 494, 552, 479, 0, 0, 0, 0, 0, 0, 0, 0, 535, 528, 503, 521, 518, 521, 505, 535, 1, 0, 0, 0, 0, 0, 0, 0, 517, 480, 562, 502, 532, 528, 518, 515, 2, 2, 0, 0, 0, 0, 0, 0, 489, 520, 510, 490, 528, 498, 506, 567, 0, 0, 0, 0, 0, 0, 0, 0, 534, 483, 505, 516, 543, 510, 530, 506, 8, 0, 0, 0, 0, 0, 0, 0, 477, 517, 452, 493, 552, 527, 510, 482, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4070, 4067, 4053, 4167, 4158, 4108, 4135, 4010] avg=4096.0000 max=4167 max_rank=3 min=4010 min_rank=7 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16411 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[534, 490, 502, 522, 478, 540, 517, 507, 3, 0, 0, 0, 0, 0, 0, 0, 540, 519, 506, 494, 518, 506, 526, 536, 0, 0, 0, 0, 0, 0, 0, 0, 477, 533, 501, 532, 508, 529, 510, 497, 0, 0, 0, 0, 0, 0, 0, 0, 519, 511, 527, 505, 490, 489, 501, 533, 1, 0, 0, 0, 0, 0, 0, 0, 504, 504, 513, 496, 557, 480, 527, 467, 2, 2, 0, 0, 0, 0, 0, 0, 467, 490, 532, 523, 513, 513, 535, 506, 0, 0, 0, 0, 0, 0, 0, 0, 522, 527, 520, 517, 503, 511, 545, 514, 7, 0, 0, 0, 0, 0, 0, 0, 507, 497, 498, 513, 491, 492, 554, 518, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4145, 4087, 4076, 4052, 4079, 4166, 4070] avg=4096.0000 max=4166 max_rank=6 min=4052 min_rank=4 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16367 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[521, 514, 509, 492, 513, 532, 509, 507, 7, 5, 0, 0, 0, 0, 0, 0, 520, 482, 535, 537, 535, 535, 491, 509, 0, 0, 0, 0, 0, 0, 0, 0, 530, 522, 559, 470, 495, 515, 517, 527, 0, 0, 0, 0, 0, 0, 0, 0, 526, 516, 517, 517, 510, 528, 499, 489, 6, 0, 0, 0, 0, 0, 0, 0, 539, 509, 544, 534, 518, 503, 481, 493, 7, 4, 0, 0, 0, 0, 0, 0, 537, 481, 527, 468, 493, 517, 507, 527, 0, 0, 0, 0, 0, 0, 0, 0, 515, 506, 498, 505, 519, 518, 510, 504, 13, 0, 0, 0, 0, 0, 0, 0, 491, 492, 491, 487, 487, 528, 512, 507, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4144, 4135, 4108, 4132, 4057, 4088, 3995] avg=4096.0000 max=4144 max_rank=1 min=3995 min_rank=7 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16272 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[506, 511, 504, 536, 516, 512, 493, 481, 3, 0, 0, 0, 0, 0, 0, 0, 475, 507, 487, 546, 527, 498, 510, 502, 0, 0, 0, 0, 0, 0, 0, 0, 509, 487, 499, 508, 539, 522, 483, 513, 0, 0, 0, 0, 0, 0, 0, 0, 507, 521, 506, 494, 511, 513, 528, 543, 1, 0, 0, 0, 0, 0, 0, 0, 560, 503, 508, 524, 511, 499, 536, 483, 2, 2, 0, 0, 0, 0, 0, 0, 505, 518, 516, 541, 528, 519, 520, 502, 0, 0, 0, 0, 0, 0, 0, 0, 512, 519, 501, 507, 471, 502, 516, 508, 8, 0, 0, 0, 0, 0, 0, 0, 526, 525, 515, 512, 525, 488, 550, 508, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4062, 4052, 4060, 4124, 4128, 4149, 4044, 4149] avg=4096.0000 max=4149 max_rank=5 min=4044 min_rank=6 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16470 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[505, 511, 512, 503, 444, 539, 516, 495, 4, 0, 0, 0, 0, 0, 0, 0, 509, 500, 535, 500, 486, 516, 493, 504, 0, 0, 0, 0, 0, 0, 0, 0, 495, 532, 444, 546, 530, 549, 513, 487, 0, 0, 0, 0, 0, 0, 0, 0, 528, 543, 513, 515, 503, 520, 484, 487, 1, 0, 0, 0, 0, 0, 0, 0, 522, 510, 489, 569, 516, 488, 515, 514, 2, 2, 0, 0, 0, 0, 0, 0, 494, 505, 533, 532, 500, 505, 525, 533, 0, 0, 0, 0, 0, 0, 0, 0, 492, 506, 530, 561, 524, 498, 521, 513, 8, 0, 0, 0, 0, 0, 0, 0, 520, 495, 547, 508, 497, 531, 508, 493, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4029, 4043, 4096, 4094, 4127, 4127, 4153, 4099] avg=4096.0000 max=4153 max_rank=6 min=4029 min_rank=0 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16506 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[523, 546, 539, 539, 521, 527, 520, 502, 4, 1, 0, 0, 0, 0, 0, 0, 512, 474, 517, 497, 515, 529, 512, 523, 0, 0, 0, 0, 0, 0, 0, 0, 542, 523, 515, 506, 520, 515, 509, 560, 0, 0, 0, 0, 0, 0, 0, 0, 482, 517, 523, 531, 558, 473, 491, 505, 1, 0, 0, 0, 0, 0, 0, 0, 490, 470, 487, 527, 525, 467, 526, 525, 2, 2, 0, 0, 0, 0, 0, 0, 521, 535, 470, 520, 519, 533, 511, 465, 0, 0, 0, 0, 0, 0, 0, 0, 496, 539, 499, 491, 490, 514, 470, 515, 8, 0, 0, 0, 0, 0, 0, 0, 513, 511, 522, 508, 500, 537, 490, 498, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4222, 4079, 4190, 4081, 4021, 4074, 4022, 4079] avg=4096.0000 max=4222 max_rank=0 min=4021 min_rank=4 imbalance_ratio=1.030762 max_load_reduction=28546 max_load_reduction_pct=0.871155 moved_to_echo=16196 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[495, 498, 531, 512, 513, 530, 517, 535, 506, 505, 529, 455, 520, 532, 507, 481, 482, 529, 524, 503, 484, 539, 477, 479, 528, 491, 540, 545, 509, 532, 513, 516, 503, 534, 505, 508, 458, 466, 522, 524, 535, 498, 529, 514, 473, 543, 511, 488, 527, 518, 497, 527, 532, 525, 524, 477, 528, 532, 546, 513, 522, 498, 532, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[509, 520, 496, 501, 505, 522, 479, 509, 511, 493, 519, 495, 490, 523, 537, 519, 525, 529, 517, 500, 546, 526, 507, 558, 511, 472, 486, 497, 510, 497, 524, 499, 514, 526, 460, 496, 532, 504, 501, 506, 515, 529, 536, 489, 514, 549, 535, 488, 527, 537, 535, 520, 520, 504, 525, 543, 518, 534, 487, 529, 512, 462, 486, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[540, 508, 484, 507, 526, 516, 510, 549, 523, 514, 517, 482, 501, 480, 525, 520, 513, 527, 494, 500, 559, 513, 504, 549, 509, 522, 529, 508, 525, 491, 535, 491, 475, 472, 519, 485, 522, 552, 505, 482, 484, 535, 501, 543, 493, 518, 538, 543, 463, 518, 515, 497, 495, 503, 501, 514, 477, 533, 523, 517, 549, 498, 506, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[513, 523, 515, 531, 494, 496, 513, 545, 512, 517, 498, 527, 510, 585, 535, 486, 508, 500, 502, 519, 509, 519, 523, 505, 513, 511, 511, 545, 515, 522, 507, 485, 540, 535, 508, 517, 516, 518, 486, 520, 512, 499, 467, 513, 517, 502, 502, 536, 511, 512, 520, 481, 509, 479, 502, 539, 515, 485, 514, 497, 514, 490, 516, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[524, 539, 509, 507, 515, 537, 529, 514, 549, 529, 552, 531, 446, 523, 486, 499, 551, 502, 470, 478, 537, 493, 526, 509, 531, 556, 538, 516, 496, 482, 528, 525, 493, 460, 523, 508, 503, 480, 546, 512, 486, 490, 532, 481, 497, 518, 450, 514, 532, 497, 500, 540, 511, 522, 524, 506, 513, 513, 509, 523, 528, 492, 522, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[523, 529, 495, 534, 519, 529, 552, 522, 482, 504, 510, 523, 525, 549, 522, 512, 537, 541, 496, 534, 505, 513, 512, 533, 541, 492, 465, 492, 521, 507, 491, 497, 485, 503, 521, 505, 501, 512, 510, 484, 526, 514, 472, 506, 507, 503, 529, 521, 492, 475, 527, 515, 511, 538, 527, 471, 542, 514, 520, 554, 485, 496, 501, 494] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[512, 462, 567, 526, 493, 526, 534, 500, 510, 548, 540, 501, 518, 506, 520, 515, 520, 504, 525, 518, 531, 478, 518, 533, 527, 497, 514, 478, 515, 473, 484, 545, 522, 490, 518, 486, 495, 525, 511, 528, 524, 526, 506, 486, 503, 499, 530, 495, 503, 500, 540, 523, 531, 489, 517, 525, 507, 480, 502, 499, 529, 518, 521, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[537, 485, 508, 501, 519, 523, 519, 491, 476, 555, 512, 492, 535, 522, 541, 527, 519, 503, 491, 523, 500, 503, 544, 552, 521, 467, 495, 494, 491, 501, 541, 519, 526, 540, 529, 514, 506, 522, 481, 502, 513, 490, 473, 525, 486, 522, 504, 497, 525, 532, 496, 524, 485, 518, 507, 532, 520, 501, 534, 519, 484, 520, 497, 537] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[495, 498, 531, 512, 513, 498, 517, 535, 0, 0, 0, 0, 0, 0, 0, 0, 506, 505, 529, 455, 520, 514, 507, 481, 0, 0, 0, 0, 0, 0, 0, 0, 482, 529, 524, 503, 484, 539, 477, 457, 0, 0, 0, 0, 0, 0, 0, 0, 528, 491, 540, 545, 509, 532, 513, 516, 11, 5, 0, 0, 0, 0, 0, 0, 503, 534, 505, 508, 458, 466, 522, 524, 32, 7, 0, 0, 0, 0, 0, 0, 535, 498, 529, 514, 473, 543, 511, 488, 15, 7, 0, 0, 0, 0, 0, 0, 527, 518, 491, 527, 532, 525, 524, 477, 0, 0, 0, 0, 0, 0, 0, 0, 528, 532, 546, 513, 522, 498, 532, 502, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4017, 3995, 4190, 4059, 4113, 4121, 4174] avg=4096.0000 max=4190 max_rank=3 min=3995 min_rank=2 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16467 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[513, 523, 515, 531, 494, 466, 513, 545, 0, 0, 0, 0, 0, 0, 0, 0, 512, 517, 498, 527, 510, 566, 535, 486, 0, 0, 0, 0, 0, 0, 0, 0, 508, 500, 502, 519, 509, 519, 523, 482, 0, 0, 0, 0, 0, 0, 0, 0, 513, 511, 511, 545, 515, 522, 507, 485, 12, 6, 0, 0, 0, 0, 0, 0, 540, 535, 508, 517, 516, 518, 486, 520, 30, 7, 0, 0, 0, 0, 0, 0, 512, 499, 467, 513, 517, 502, 502, 536, 16, 7, 0, 0, 0, 0, 0, 0, 511, 512, 513, 481, 509, 479, 502, 539, 0, 0, 0, 0, 0, 0, 0, 0, 515, 485, 514, 497, 514, 490, 516, 502, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4151, 4062, 4127, 4177, 4071, 4046, 4034] avg=4096.0000 max=4177 max_rank=4 min=4034 min_rank=7 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16328 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[523, 529, 495, 534, 519, 497, 552, 522, 0, 0, 0, 0, 0, 0, 0, 0, 482, 504, 510, 523, 525, 531, 522, 512, 0, 0, 0, 0, 0, 0, 0, 0, 537, 541, 496, 534, 505, 513, 512, 509, 0, 0, 0, 0, 0, 0, 0, 0, 541, 492, 465, 492, 521, 507, 491, 497, 11, 6, 0, 0, 0, 0, 0, 0, 485, 503, 521, 505, 501, 512, 510, 484, 32, 7, 0, 0, 0, 0, 0, 0, 526, 514, 472, 506, 507, 503, 529, 521, 17, 7, 0, 0, 0, 0, 0, 0, 492, 475, 520, 515, 511, 538, 527, 471, 0, 0, 0, 0, 0, 0, 0, 0, 542, 514, 520, 554, 485, 496, 501, 494, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4171, 4109, 4147, 4023, 4060, 4102, 4049, 4107] avg=4096.0000 max=4171 max_rank=0 min=4023 min_rank=3 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16318 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[509, 520, 496, 501, 505, 487, 479, 509, 0, 0, 0, 0, 0, 0, 0, 0, 511, 493, 519, 495, 490, 498, 537, 519, 0, 0, 0, 0, 0, 0, 0, 0, 525, 529, 517, 500, 546, 526, 507, 525, 0, 0, 0, 0, 0, 0, 0, 0, 511, 472, 486, 497, 510, 497, 524, 499, 14, 10, 0, 0, 0, 0, 0, 0, 514, 526, 460, 496, 532, 504, 501, 506, 35, 11, 0, 0, 0, 0, 0, 0, 515, 529, 536, 489, 514, 549, 535, 488, 22, 11, 0, 0, 0, 0, 0, 0, 527, 537, 517, 520, 520, 504, 525, 543, 0, 0, 0, 0, 0, 0, 0, 0, 518, 534, 487, 529, 512, 462, 486, 503, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4006, 4062, 4175, 4020, 4085, 4188, 4193, 4039] avg=4096.0000 max=4193 max_rank=6 min=4006 min_rank=0 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16505 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[524, 539, 509, 507, 515, 505, 529, 514, 0, 0, 0, 0, 0, 0, 0, 0, 549, 529, 552, 531, 446, 506, 486, 499, 0, 0, 0, 0, 0, 0, 0, 0, 551, 502, 470, 478, 537, 493, 526, 486, 0, 0, 0, 0, 0, 0, 0, 0, 531, 556, 538, 516, 496, 482, 528, 525, 11, 5, 0, 0, 0, 0, 0, 0, 493, 460, 523, 508, 503, 480, 546, 512, 32, 7, 0, 0, 0, 0, 0, 0, 486, 490, 532, 481, 497, 518, 450, 514, 16, 6, 0, 0, 0, 0, 0, 0, 532, 497, 494, 540, 511, 522, 524, 506, 0, 0, 0, 0, 0, 0, 0, 0, 513, 513, 509, 523, 528, 492, 522, 516, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4142, 4098, 4043, 4188, 4064, 3990, 4126, 4117] avg=4096.0000 max=4188 max_rank=3 min=3990 min_rank=5 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16297 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[540, 508, 484, 507, 526, 485, 510, 549, 0, 0, 0, 0, 0, 0, 0, 0, 523, 514, 517, 482, 501, 464, 525, 520, 0, 0, 0, 0, 0, 0, 0, 0, 513, 527, 494, 500, 559, 513, 504, 524, 0, 0, 0, 0, 0, 0, 0, 0, 509, 522, 529, 508, 525, 491, 535, 491, 10, 6, 0, 0, 0, 0, 0, 0, 475, 472, 519, 485, 522, 552, 505, 482, 31, 8, 0, 0, 0, 0, 0, 0, 484, 535, 501, 543, 493, 518, 538, 543, 17, 6, 0, 0, 0, 0, 0, 0, 463, 518, 508, 497, 495, 503, 501, 514, 0, 0, 0, 0, 0, 0, 0, 0, 477, 533, 523, 517, 549, 498, 506, 521, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4046, 4134, 4126, 4051, 4178, 3999, 4125] avg=4096.0000 max=4178 max_rank=5 min=3999 min_rank=6 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[512, 462, 567, 526, 493, 495, 534, 500, 0, 0, 0, 0, 0, 0, 0, 0, 510, 548, 540, 501, 518, 490, 520, 515, 0, 0, 0, 0, 0, 0, 0, 0, 520, 504, 525, 518, 531, 478, 518, 509, 0, 0, 0, 0, 0, 0, 0, 0, 527, 497, 514, 478, 515, 473, 484, 545, 10, 6, 0, 0, 0, 0, 0, 0, 522, 490, 518, 486, 495, 525, 511, 528, 31, 7, 0, 0, 0, 0, 0, 0, 524, 526, 506, 486, 503, 499, 530, 495, 17, 6, 0, 0, 0, 0, 0, 0, 503, 500, 533, 523, 531, 489, 517, 525, 0, 0, 0, 0, 0, 0, 0, 0, 507, 480, 502, 499, 529, 518, 521, 502, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4142, 4103, 4049, 4113, 4092, 4121, 4059] avg=4096.0000 max=4142 max_rank=1 min=4049 min_rank=3 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[537, 485, 508, 501, 519, 492, 519, 491, 0, 0, 0, 0, 0, 0, 0, 0, 476, 555, 512, 492, 535, 505, 541, 527, 0, 0, 0, 0, 0, 0, 0, 0, 519, 503, 491, 523, 500, 503, 544, 526, 0, 0, 0, 0, 0, 0, 0, 0, 521, 467, 495, 494, 491, 501, 541, 519, 11, 5, 0, 0, 0, 0, 0, 0, 526, 540, 529, 514, 506, 522, 481, 502, 31, 8, 0, 0, 0, 0, 0, 0, 513, 490, 473, 525, 486, 522, 504, 497, 18, 6, 0, 0, 0, 0, 0, 0, 525, 532, 490, 524, 485, 518, 507, 532, 0, 0, 0, 0, 0, 0, 0, 0, 520, 501, 534, 519, 484, 520, 497, 537, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4052, 4143, 4109, 4045, 4159, 4034, 4113, 4113] avg=4096.0000 max=4159 max_rank=4 min=4034 min_rank=5 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16419 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 3968, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[544, 521, 518, 500, 511, 521, 539, 484, 514, 532, 474, 502, 486, 497, 517, 513, 500, 507, 504, 473, 502, 507, 592, 524, 531, 527, 556, 493, 514, 465, 497, 547, 467, 502, 498, 506, 492, 512, 488, 504, 563, 484, 496, 533, 496, 505, 524, 517, 526, 512, 525, 539, 508, 522, 494, 512, 546, 542, 491, 513, 516, 477, 511, 535] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[522, 491, 536, 506, 491, 493, 503, 534, 503, 546, 504, 520, 514, 498, 519, 484, 494, 529, 499, 463, 506, 512, 483, 561, 490, 495, 502, 565, 510, 544, 491, 517, 496, 486, 525, 483, 503, 525, 519, 526, 508, 521, 482, 515, 468, 537, 544, 502, 540, 521, 545, 520, 488, 511, 492, 527, 490, 552, 520, 515, 515, 542, 497, 528] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[521, 509, 515, 505, 499, 494, 467, 536, 492, 533, 501, 502, 546, 506, 502, 507, 491, 498, 499, 512, 507, 500, 553, 512, 522, 503, 524, 505, 521, 511, 524, 509, 520, 522, 479, 500, 541, 494, 535, 501, 529, 529, 500, 518, 539, 506, 503, 549, 513, 494, 534, 522, 498, 545, 490, 499, 531, 512, 504, 493, 485, 518, 490, 549] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[497, 555, 485, 551, 503, 524, 489, 509, 500, 490, 486, 533, 491, 511, 498, 498, 524, 518, 513, 465, 508, 471, 530, 531, 531, 537, 558, 511, 506, 494, 496, 501, 511, 522, 497, 518, 489, 498, 497, 502, 513, 557, 511, 496, 503, 567, 522, 505, 498, 502, 528, 522, 537, 494, 546, 518, 480, 504, 520, 504, 514, 509, 541, 529] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[482, 499, 485, 537, 515, 513, 505, 529, 534, 524, 531, 461, 562, 512, 523, 509, 497, 517, 498, 479, 489, 550, 519, 507, 479, 502, 511, 505, 512, 494, 537, 518, 538, 515, 501, 491, 533, 527, 494, 540, 509, 536, 529, 539, 492, 538, 532, 527, 508, 494, 512, 454, 525, 488, 496, 521, 533, 493, 483, 519, 493, 540, 546, 487] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[506, 564, 497, 528, 535, 554, 548, 555, 506, 515, 502, 506, 527, 482, 516, 497, 502, 513, 505, 493, 496, 506, 508, 505, 481, 482, 519, 549, 497, 527, 477, 519, 530, 477, 531, 525, 530, 483, 531, 557, 507, 506, 516, 500, 474, 543, 523, 510, 541, 505, 500, 557, 546, 522, 445, 514, 476, 486, 484, 467, 531, 521, 511, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[509, 480, 543, 517, 511, 497, 490, 496, 539, 512, 517, 527, 509, 493, 541, 521, 513, 540, 503, 533, 501, 476, 507, 511, 531, 491, 537, 453, 489, 527, 484, 505, 516, 510, 528, 522, 549, 518, 512, 483, 498, 525, 484, 513, 540, 541, 495, 526, 503, 515, 531, 520, 511, 536, 519, 485, 508, 507, 487, 552, 500, 507, 496, 528] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[530, 478, 544, 498, 538, 506, 516, 520, 542, 535, 555, 505, 509, 495, 496, 520, 495, 528, 533, 488, 500, 484, 536, 531, 515, 519, 516, 521, 512, 512, 514, 517, 537, 472, 476, 512, 514, 482, 500, 482, 510, 477, 525, 504, 505, 502, 504, 515, 527, 511, 509, 530, 525, 505, 515, 552, 468, 521, 502, 529, 493, 530, 530, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[521, 509, 515, 505, 499, 494, 467, 521, 0, 0, 0, 0, 0, 0, 0, 0, 492, 533, 501, 502, 546, 506, 502, 507, 3, 0, 0, 0, 0, 0, 0, 0, 491, 498, 499, 512, 507, 500, 553, 512, 25, 0, 0, 0, 0, 0, 0, 0, 522, 503, 524, 505, 521, 511, 524, 509, 2, 0, 0, 0, 0, 0, 0, 0, 520, 522, 479, 500, 541, 494, 535, 501, 22, 2, 0, 0, 0, 0, 0, 0, 529, 529, 500, 518, 539, 481, 503, 549, 0, 0, 0, 0, 0, 0, 0, 0, 513, 494, 512, 522, 498, 545, 490, 499, 0, 0, 0, 0, 0, 0, 0, 0, 531, 512, 504, 493, 485, 518, 490, 549, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4031, 4092, 4097, 4121, 4116, 4148, 4073, 4090] avg=4096.0000 max=4148 max_rank=5 min=4031 min_rank=0 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16427 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[544, 521, 518, 500, 511, 521, 539, 470, 0, 0, 0, 0, 0, 0, 0, 0, 514, 532, 474, 502, 486, 497, 517, 513, 3, 0, 0, 0, 0, 0, 0, 0, 500, 507, 504, 473, 502, 507, 592, 524, 25, 0, 0, 0, 0, 0, 0, 0, 531, 527, 556, 493, 514, 465, 497, 547, 2, 0, 0, 0, 0, 0, 0, 0, 467, 502, 498, 506, 492, 512, 488, 504, 22, 1, 0, 0, 0, 0, 0, 0, 563, 484, 496, 533, 496, 480, 524, 517, 0, 0, 0, 0, 0, 0, 0, 0, 526, 512, 503, 539, 508, 522, 494, 512, 0, 0, 0, 0, 0, 0, 0, 0, 546, 542, 491, 513, 516, 477, 511, 535, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4124, 4038, 4134, 4132, 3992, 4093, 4116, 4139] avg=4096.0000 max=4139 max_rank=7 min=3992 min_rank=4 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16340 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[497, 555, 485, 551, 503, 524, 489, 495, 0, 0, 0, 0, 0, 0, 0, 0, 500, 490, 486, 533, 491, 511, 498, 498, 3, 0, 0, 0, 0, 0, 0, 0, 524, 518, 513, 465, 508, 471, 530, 531, 29, 0, 0, 0, 0, 0, 0, 0, 531, 537, 558, 511, 506, 494, 496, 501, 2, 0, 0, 0, 0, 0, 0, 0, 511, 522, 497, 518, 489, 498, 497, 502, 22, 1, 0, 0, 0, 0, 0, 0, 513, 557, 511, 496, 503, 538, 522, 505, 0, 0, 0, 0, 0, 0, 0, 0, 498, 502, 506, 522, 537, 494, 546, 518, 0, 0, 0, 0, 0, 0, 0, 0, 480, 504, 520, 504, 514, 509, 541, 529, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4010, 4089, 4136, 4057, 4145, 4123, 4109] avg=4096.0000 max=4145 max_rank=5 min=4010 min_rank=1 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16434 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[522, 491, 536, 506, 491, 493, 503, 519, 0, 0, 0, 0, 0, 0, 0, 0, 503, 546, 504, 520, 514, 498, 519, 484, 3, 0, 0, 0, 0, 0, 0, 0, 494, 529, 499, 463, 506, 512, 483, 561, 27, 0, 0, 0, 0, 0, 0, 0, 490, 495, 502, 565, 510, 544, 491, 517, 2, 0, 0, 0, 0, 0, 0, 0, 496, 486, 525, 483, 503, 525, 519, 526, 22, 2, 0, 0, 0, 0, 0, 0, 508, 521, 482, 515, 468, 510, 544, 502, 0, 0, 0, 0, 0, 0, 0, 0, 540, 521, 523, 520, 488, 511, 492, 527, 0, 0, 0, 0, 0, 0, 0, 0, 490, 552, 520, 515, 515, 542, 497, 528, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4091, 4074, 4116, 4087, 4050, 4122, 4167] avg=4096.0000 max=4167 max_rank=7 min=4050 min_rank=5 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16426 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[482, 499, 485, 537, 515, 513, 505, 514, 0, 0, 0, 0, 0, 0, 0, 0, 534, 524, 531, 461, 562, 512, 523, 509, 3, 0, 0, 0, 0, 0, 0, 0, 497, 517, 498, 479, 489, 550, 519, 507, 27, 0, 0, 0, 0, 0, 0, 0, 479, 502, 511, 505, 512, 494, 537, 518, 2, 0, 0, 0, 0, 0, 0, 0, 538, 515, 501, 491, 533, 527, 494, 540, 21, 2, 0, 0, 0, 0, 0, 0, 509, 536, 529, 539, 492, 511, 532, 527, 0, 0, 0, 0, 0, 0, 0, 0, 508, 494, 491, 454, 525, 488, 496, 521, 0, 0, 0, 0, 0, 0, 0, 0, 533, 493, 483, 519, 493, 540, 546, 487, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4050, 4159, 4083, 4060, 4162, 4175, 3977, 4102] avg=4096.0000 max=4175 max_rank=5 min=3977 min_rank=6 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16416 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[506, 564, 497, 528, 535, 554, 548, 526, 0, 0, 0, 0, 0, 0, 0, 0, 506, 515, 502, 506, 527, 482, 516, 497, 5, 0, 0, 0, 0, 0, 0, 0, 502, 513, 505, 493, 496, 506, 508, 505, 32, 0, 0, 0, 0, 0, 0, 0, 481, 482, 519, 549, 497, 527, 477, 519, 5, 0, 0, 0, 0, 0, 0, 0, 530, 477, 531, 525, 530, 483, 531, 557, 24, 6, 2, 0, 0, 0, 0, 0, 507, 506, 516, 500, 474, 509, 523, 510, 0, 0, 0, 0, 0, 0, 0, 0, 541, 505, 476, 557, 546, 522, 445, 514, 0, 0, 0, 0, 0, 0, 0, 0, 476, 486, 484, 467, 531, 521, 511, 502, 13, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4258, 4056, 4060, 4056, 4196, 4045, 4106, 3991] avg=4096.0000 max=4258 max_rank=0 min=3991 min_rank=7 imbalance_ratio=1.039551 max_load_reduction=28510 max_load_reduction_pct=0.870056 moved_to_echo=16338 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[509, 480, 543, 517, 511, 497, 490, 482, 0, 0, 0, 0, 0, 0, 0, 0, 539, 512, 517, 527, 509, 493, 541, 521, 3, 0, 0, 0, 0, 0, 0, 0, 513, 540, 503, 533, 501, 476, 507, 511, 27, 0, 0, 0, 0, 0, 0, 0, 531, 491, 537, 453, 489, 527, 484, 505, 2, 0, 0, 0, 0, 0, 0, 0, 516, 510, 528, 522, 549, 518, 512, 483, 22, 1, 0, 0, 0, 0, 0, 0, 498, 525, 484, 513, 540, 514, 495, 526, 0, 0, 0, 0, 0, 0, 0, 0, 503, 515, 509, 520, 511, 536, 519, 485, 0, 0, 0, 0, 0, 0, 0, 0, 508, 507, 487, 552, 500, 507, 496, 528, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4029, 4162, 4111, 4019, 4161, 4095, 4098, 4093] avg=4096.0000 max=4162 max_rank=1 min=4019 min_rank=3 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16447 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[530, 478, 544, 498, 538, 506, 516, 506, 0, 0, 0, 0, 0, 0, 0, 0, 542, 535, 555, 505, 509, 495, 496, 520, 3, 0, 0, 0, 0, 0, 0, 0, 495, 528, 533, 488, 500, 484, 536, 531, 25, 0, 0, 0, 0, 0, 0, 0, 515, 519, 516, 521, 512, 512, 514, 517, 2, 0, 0, 0, 0, 0, 0, 0, 537, 472, 476, 512, 514, 482, 500, 482, 21, 1, 0, 0, 0, 0, 0, 0, 510, 477, 525, 504, 505, 477, 504, 515, 0, 0, 0, 0, 0, 0, 0, 0, 527, 511, 488, 530, 525, 505, 515, 552, 0, 0, 0, 0, 0, 0, 0, 0, 468, 521, 502, 529, 493, 530, 530, 496, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4160, 4120, 4128, 3997, 4017, 4153, 4077] avg=4096.0000 max=4160 max_rank=1 min=3997 min_rank=4 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16244 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 3968, 4096, 4096, 4352, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[505, 477, 501, 530, 492, 518, 514, 514, 501, 510, 528, 506, 490, 509, 516, 523, 517, 483, 537, 482, 504, 528, 517, 513, 531, 511, 519, 491, 520, 497, 460, 509, 490, 533, 538, 504, 507, 539, 492, 545, 530, 484, 489, 519, 529, 522, 509, 541, 531, 492, 534, 475, 528, 528, 522, 510, 514, 528, 515, 521, 502, 505, 511, 528] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[506, 528, 523, 498, 467, 506, 521, 519, 477, 512, 501, 489, 547, 511, 512, 513, 480, 467, 515, 515, 542, 567, 514, 502, 513, 493, 500, 481, 507, 509, 514, 518, 519, 544, 512, 494, 515, 503, 513, 500, 545, 497, 523, 500, 544, 504, 501, 550, 497, 488, 513, 542, 551, 532, 506, 537, 495, 532, 459, 543, 514, 509, 506, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[476, 540, 478, 521, 456, 490, 495, 512, 541, 542, 525, 516, 533, 521, 483, 506, 536, 510, 516, 509, 529, 521, 524, 485, 519, 537, 497, 551, 469, 508, 539, 509, 548, 540, 492, 527, 541, 528, 522, 511, 524, 517, 540, 480, 508, 473, 487, 508, 514, 508, 519, 510, 489, 519, 496, 495, 511, 482, 524, 514, 489, 514, 491, 553] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[515, 519, 504, 504, 500, 525, 523, 493, 492, 476, 502, 493, 528, 507, 520, 545, 505, 527, 527, 553, 539, 488, 493, 517, 527, 510, 501, 497, 495, 503, 538, 532, 492, 513, 533, 519, 562, 520, 534, 496, 537, 499, 476, 533, 496, 520, 489, 537, 512, 469, 491, 474, 486, 501, 518, 520, 549, 511, 521, 516, 532, 503, 504, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[510, 525, 497, 511, 516, 530, 510, 511, 494, 528, 525, 519, 482, 548, 515, 504, 500, 510, 524, 484, 503, 530, 540, 472, 510, 503, 504, 561, 490, 513, 540, 534, 536, 501, 538, 513, 473, 524, 553, 494, 478, 518, 518, 515, 527, 493, 511, 525, 498, 474, 498, 533, 522, 504, 502, 511, 526, 486, 510, 499, 502, 532, 513, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[516, 508, 509, 528, 514, 495, 478, 484, 524, 522, 501, 493, 516, 524, 518, 500, 493, 540, 529, 501, 489, 521, 554, 539, 505, 535, 526, 494, 485, 501, 565, 507, 501, 492, 531, 542, 484, 511, 518, 504, 487, 521, 502, 508, 494, 515, 511, 510, 516, 506, 524, 501, 531, 549, 518, 488, 544, 511, 497, 475, 516, 507, 532, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[529, 515, 480, 557, 543, 522, 489, 535, 517, 519, 486, 501, 502, 552, 488, 507, 518, 513, 544, 503, 491, 467, 489, 516, 478, 519, 526, 547, 548, 477, 517, 476, 526, 488, 507, 499, 512, 538, 498, 523, 537, 555, 487, 562, 503, 488, 500, 524, 482, 534, 516, 489, 499, 510, 498, 531, 512, 533, 492, 511, 506, 498, 514, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[498, 528, 506, 524, 523, 506, 491, 500, 512, 549, 488, 530, 512, 524, 468, 513, 514, 520, 514, 484, 506, 489, 502, 538, 531, 547, 521, 490, 497, 512, 500, 527, 491, 533, 494, 502, 557, 513, 503, 507, 513, 537, 555, 512, 518, 508, 517, 505, 449, 492, 512, 507, 518, 512, 508, 469, 527, 528, 487, 539, 497, 525, 516, 553] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[505, 477, 501, 530, 492, 518, 514, 514, 12, 8, 3, 1, 0, 0, 0, 0, 501, 510, 528, 506, 490, 509, 516, 523, 1, 0, 0, 0, 0, 0, 0, 0, 517, 483, 534, 482, 504, 528, 517, 513, 0, 0, 0, 0, 0, 0, 0, 0, 531, 511, 519, 491, 520, 497, 458, 509, 0, 0, 0, 0, 0, 0, 0, 0, 490, 533, 538, 504, 507, 506, 492, 545, 0, 0, 0, 0, 0, 0, 0, 0, 530, 484, 489, 519, 529, 522, 509, 529, 0, 0, 0, 0, 0, 0, 0, 0, 531, 492, 534, 475, 528, 528, 522, 510, 33, 0, 0, 0, 0, 0, 0, 0, 514, 528, 515, 521, 502, 505, 511, 520, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4084, 4078, 4036, 4115, 4111, 4153, 4116] avg=4096.0000 max=4153 max_rank=6 min=4036 min_rank=3 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16495 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[476, 540, 478, 521, 456, 490, 495, 512, 11, 8, 3, 1, 0, 0, 0, 0, 541, 542, 525, 516, 533, 521, 483, 506, 1, 0, 0, 0, 0, 0, 0, 0, 536, 510, 513, 509, 529, 521, 524, 485, 0, 0, 0, 0, 0, 0, 0, 0, 519, 537, 497, 551, 469, 508, 537, 509, 0, 0, 0, 0, 0, 0, 0, 0, 548, 540, 492, 527, 541, 496, 522, 511, 0, 0, 0, 0, 0, 0, 0, 0, 524, 517, 540, 480, 508, 473, 487, 497, 0, 0, 0, 0, 0, 0, 0, 0, 514, 508, 519, 510, 489, 519, 496, 495, 32, 0, 0, 0, 0, 0, 0, 0, 511, 482, 524, 514, 489, 514, 491, 545, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3991, 4168, 4127, 4127, 4177, 4026, 4082, 4070] avg=4096.0000 max=4177 max_rank=4 min=3991 min_rank=0 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16355 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[515, 519, 504, 504, 500, 525, 523, 493, 12, 8, 3, 1, 0, 0, 0, 0, 492, 476, 502, 493, 528, 507, 520, 545, 1, 0, 0, 0, 0, 0, 0, 0, 505, 527, 524, 553, 539, 488, 493, 517, 0, 0, 0, 0, 0, 0, 0, 0, 527, 510, 501, 497, 495, 503, 536, 532, 0, 0, 0, 0, 0, 0, 0, 0, 492, 513, 533, 519, 562, 488, 534, 496, 0, 0, 0, 0, 0, 0, 0, 0, 537, 499, 476, 533, 496, 520, 489, 525, 0, 0, 0, 0, 0, 0, 0, 0, 512, 469, 491, 474, 486, 501, 518, 520, 32, 0, 0, 0, 0, 0, 0, 0, 549, 511, 521, 516, 532, 503, 504, 499, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4064, 4146, 4101, 4137, 4075, 4003, 4135] avg=4096.0000 max=4146 max_rank=2 min=4003 min_rank=6 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16350 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[506, 528, 523, 498, 467, 506, 521, 519, 12, 8, 3, 1, 0, 0, 0, 0, 477, 512, 501, 489, 547, 511, 512, 513, 1, 0, 0, 0, 0, 0, 0, 0, 480, 467, 512, 515, 542, 567, 514, 502, 0, 0, 0, 0, 0, 0, 0, 0, 513, 493, 500, 481, 507, 509, 512, 518, 0, 0, 0, 0, 0, 0, 0, 0, 519, 544, 512, 494, 515, 472, 513, 500, 0, 0, 0, 0, 0, 0, 0, 0, 545, 497, 523, 500, 544, 504, 501, 538, 0, 0, 0, 0, 0, 0, 0, 0, 497, 488, 513, 542, 551, 532, 506, 537, 31, 0, 0, 0, 0, 0, 0, 0, 495, 532, 459, 543, 514, 509, 506, 505, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4063, 4099, 4033, 4069, 4152, 4197, 4063] avg=4096.0000 max=4197 max_rank=6 min=4033 min_rank=3 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16481 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[510, 525, 497, 511, 516, 530, 510, 511, 12, 7, 3, 1, 0, 0, 0, 0, 494, 528, 525, 519, 482, 548, 515, 504, 1, 0, 0, 0, 0, 0, 0, 0, 500, 510, 521, 484, 503, 530, 540, 472, 0, 0, 0, 0, 0, 0, 0, 0, 510, 503, 504, 561, 490, 513, 538, 534, 0, 0, 0, 0, 0, 0, 0, 0, 536, 501, 538, 513, 473, 492, 553, 494, 0, 0, 0, 0, 0, 0, 0, 0, 478, 518, 518, 515, 527, 493, 511, 513, 0, 0, 0, 0, 0, 0, 0, 0, 498, 474, 498, 533, 522, 504, 502, 511, 32, 0, 0, 0, 0, 0, 0, 0, 526, 486, 510, 499, 502, 532, 513, 491, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4133, 4116, 4060, 4153, 4100, 4073, 4074, 4059] avg=4096.0000 max=4153 max_rank=3 min=4059 min_rank=7 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16306 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[529, 515, 480, 557, 543, 522, 489, 535, 16, 12, 10, 4, 4, 0, 0, 0, 517, 519, 486, 501, 502, 552, 488, 507, 5, 0, 0, 0, 0, 0, 0, 0, 518, 513, 534, 503, 491, 467, 489, 516, 0, 0, 0, 0, 0, 0, 0, 0, 478, 519, 526, 547, 548, 477, 508, 476, 0, 0, 0, 0, 0, 0, 0, 0, 526, 488, 507, 499, 512, 496, 498, 523, 0, 0, 0, 0, 0, 0, 0, 0, 537, 555, 487, 562, 503, 488, 500, 508, 0, 0, 0, 0, 0, 0, 0, 0, 482, 534, 516, 489, 499, 510, 498, 531, 38, 0, 0, 0, 0, 0, 0, 0, 512, 533, 492, 511, 506, 498, 514, 513, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4216, 4077, 4031, 4079, 4049, 4140, 4097, 4079] avg=4096.0000 max=4216 max_rank=0 min=4031 min_rank=2 imbalance_ratio=1.029297 max_load_reduction=28552 max_load_reduction_pct=0.871338 moved_to_echo=16365 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[516, 508, 509, 528, 514, 495, 478, 484, 11, 8, 3, 1, 0, 0, 0, 0, 524, 522, 501, 493, 516, 524, 518, 500, 1, 0, 0, 0, 0, 0, 0, 0, 493, 540, 526, 501, 489, 521, 554, 539, 0, 0, 0, 0, 0, 0, 0, 0, 505, 535, 526, 494, 485, 501, 563, 507, 0, 0, 0, 0, 0, 0, 0, 0, 501, 492, 531, 542, 484, 480, 518, 504, 0, 0, 0, 0, 0, 0, 0, 0, 487, 521, 502, 508, 494, 515, 511, 499, 0, 0, 0, 0, 0, 0, 0, 0, 516, 506, 524, 501, 531, 549, 518, 488, 31, 0, 0, 0, 0, 0, 0, 0, 544, 511, 497, 475, 516, 507, 532, 500, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4055, 4099, 4163, 4116, 4052, 4037, 4164, 4082] avg=4096.0000 max=4164 max_rank=6 min=4037 min_rank=5 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16335 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[498, 528, 506, 524, 523, 506, 491, 500, 11, 8, 3, 1, 0, 0, 0, 0, 512, 549, 488, 530, 512, 524, 468, 513, 1, 0, 0, 0, 0, 0, 0, 0, 514, 520, 511, 484, 506, 489, 502, 538, 0, 0, 0, 0, 0, 0, 0, 0, 531, 547, 521, 490, 497, 512, 498, 527, 0, 0, 0, 0, 0, 0, 0, 0, 491, 533, 494, 502, 557, 482, 503, 507, 0, 0, 0, 0, 0, 0, 0, 0, 513, 537, 555, 512, 518, 508, 517, 494, 0, 0, 0, 0, 0, 0, 0, 0, 449, 492, 512, 507, 518, 512, 508, 469, 31, 0, 0, 0, 0, 0, 0, 0, 527, 528, 487, 539, 497, 525, 516, 545, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4097, 4064, 4123, 4069, 4154, 3998, 4164] avg=4096.0000 max=4164 max_rank=7 min=3998 min_rank=6 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 128, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 3968, 4224, 4096, 4224, 4224, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 3968, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[487, 488, 511, 515, 551, 528, 516, 486, 493, 464, 509, 469, 525, 499, 487, 527, 510, 521, 522, 508, 489, 534, 536, 529, 513, 479, 531, 519, 524, 500, 511, 526, 544, 514, 553, 522, 521, 509, 551, 481, 525, 477, 483, 551, 527, 508, 504, 497, 479, 486, 500, 552, 496, 516, 527, 566, 537, 538, 442, 492, 523, 482, 528, 530] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[509, 567, 479, 506, 519, 554, 532, 516, 499, 500, 519, 541, 539, 472, 497, 503, 512, 526, 530, 515, 474, 522, 539, 494, 516, 545, 523, 492, 492, 500, 483, 517, 482, 501, 524, 524, 524, 518, 508, 488, 517, 486, 507, 558, 497, 499, 530, 524, 511, 522, 555, 500, 524, 488, 497, 521, 508, 531, 508, 483, 506, 502, 491, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[509, 567, 490, 506, 491, 525, 514, 526, 531, 490, 509, 519, 533, 519, 550, 511, 527, 483, 484, 502, 495, 525, 517, 519, 522, 542, 473, 518, 495, 541, 504, 486, 519, 493, 502, 510, 480, 517, 523, 531, 508, 529, 515, 517, 515, 517, 509, 525, 468, 517, 497, 488, 524, 484, 534, 506, 541, 521, 523, 493, 548, 519, 491, 481] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[512, 529, 562, 483, 498, 507, 551, 529, 501, 510, 503, 511, 541, 511, 469, 532, 497, 525, 532, 503, 538, 489, 466, 533, 532, 524, 516, 489, 526, 537, 529, 538, 519, 519, 497, 497, 511, 492, 517, 475, 503, 496, 503, 536, 494, 541, 538, 517, 468, 497, 544, 489, 505, 479, 506, 536, 513, 530, 500, 514, 516, 501, 516, 476] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[517, 507, 528, 537, 542, 525, 517, 505, 481, 512, 528, 506, 523, 546, 537, 472, 520, 539, 514, 499, 490, 505, 499, 529, 504, 507, 503, 524, 530, 543, 546, 512, 516, 489, 491, 508, 498, 504, 527, 490, 515, 465, 507, 526, 532, 511, 512, 475, 522, 501, 484, 486, 505, 521, 526, 541, 527, 485, 529, 545, 502, 471, 497, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[505, 513, 480, 532, 531, 508, 493, 522, 520, 516, 539, 521, 513, 509, 536, 526, 483, 545, 506, 516, 523, 546, 524, 506, 490, 518, 526, 500, 515, 512, 499, 524, 517, 501, 506, 509, 520, 498, 505, 505, 497, 527, 521, 517, 506, 497, 482, 537, 497, 492, 518, 551, 522, 476, 538, 502, 483, 471, 503, 498, 486, 536, 522, 531] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[472, 492, 493, 506, 516, 500, 479, 485, 534, 484, 515, 511, 511, 527, 483, 512, 512, 513, 478, 512, 525, 512, 541, 490, 495, 533, 527, 515, 544, 521, 510, 511, 506, 498, 533, 514, 510, 558, 487, 507, 506, 523, 524, 524, 513, 515, 510, 509, 531, 564, 514, 510, 506, 510, 510, 486, 486, 513, 546, 532, 500, 547, 507, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[513, 496, 531, 500, 496, 530, 527, 514, 555, 498, 497, 511, 512, 507, 538, 492, 539, 491, 535, 490, 499, 532, 521, 531, 501, 486, 500, 539, 529, 528, 509, 513, 520, 514, 506, 489, 531, 482, 520, 554, 511, 532, 509, 537, 516, 487, 518, 516, 492, 522, 515, 511, 480, 532, 543, 504, 543, 487, 486, 478, 505, 497, 509, 462] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[509, 567, 479, 506, 519, 541, 532, 516, 0, 0, 0, 0, 0, 0, 0, 0, 499, 500, 519, 541, 539, 472, 497, 503, 1, 0, 0, 0, 0, 0, 0, 0, 512, 526, 530, 515, 474, 511, 539, 494, 0, 0, 0, 0, 0, 0, 0, 0, 516, 545, 523, 492, 492, 478, 483, 517, 0, 0, 0, 0, 0, 0, 0, 0, 482, 501, 524, 524, 524, 518, 508, 488, 8, 2, 0, 0, 0, 0, 0, 0, 517, 486, 507, 551, 497, 499, 530, 524, 0, 0, 0, 0, 0, 0, 0, 0, 511, 522, 555, 500, 524, 488, 497, 521, 6, 3, 0, 0, 0, 0, 0, 0, 508, 531, 508, 483, 506, 502, 491, 502, 22, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4169, 4071, 4101, 4046, 4079, 4111, 4127, 4064] avg=4096.0000 max=4169 max_rank=0 min=4046 min_rank=3 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[487, 488, 511, 515, 551, 516, 516, 486, 0, 0, 0, 0, 0, 0, 0, 0, 493, 464, 509, 469, 525, 499, 487, 527, 1, 0, 0, 0, 0, 0, 0, 0, 510, 521, 522, 508, 489, 523, 536, 529, 0, 0, 0, 0, 0, 0, 0, 0, 513, 479, 531, 519, 524, 478, 511, 526, 0, 0, 0, 0, 0, 0, 0, 0, 544, 514, 553, 522, 521, 509, 551, 481, 8, 2, 0, 0, 0, 0, 0, 0, 525, 477, 483, 544, 527, 508, 504, 497, 0, 0, 0, 0, 0, 0, 0, 0, 479, 486, 500, 552, 496, 516, 527, 566, 6, 3, 0, 0, 0, 0, 0, 0, 537, 538, 442, 492, 523, 482, 528, 530, 22, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4070, 3974, 4138, 4081, 4205, 4065, 4131, 4104] avg=4096.0000 max=4205 max_rank=4 min=3974 min_rank=1 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16505 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[517, 507, 528, 537, 542, 513, 517, 505, 0, 0, 0, 0, 0, 0, 0, 0, 481, 512, 528, 506, 523, 546, 537, 472, 1, 0, 0, 0, 0, 0, 0, 0, 520, 539, 514, 499, 490, 494, 499, 529, 0, 0, 0, 0, 0, 0, 0, 0, 504, 507, 503, 524, 530, 519, 546, 512, 0, 0, 0, 0, 0, 0, 0, 0, 516, 489, 491, 508, 498, 504, 527, 490, 8, 2, 0, 0, 0, 0, 0, 0, 515, 465, 507, 519, 532, 511, 512, 475, 0, 0, 0, 0, 0, 0, 0, 0, 522, 501, 484, 486, 505, 521, 526, 541, 6, 3, 0, 0, 0, 0, 0, 0, 527, 485, 529, 545, 502, 471, 497, 513, 24, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4166, 4106, 4084, 4145, 4033, 4036, 4095, 4103] avg=4096.0000 max=4166 max_rank=0 min=4033 min_rank=4 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16267 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[512, 529, 562, 483, 498, 485, 551, 529, 0, 0, 0, 0, 0, 0, 0, 0, 501, 510, 503, 511, 541, 511, 469, 532, 6, 0, 0, 0, 0, 0, 0, 0, 497, 525, 532, 503, 538, 473, 466, 533, 0, 0, 0, 0, 0, 0, 0, 0, 532, 524, 516, 489, 526, 509, 529, 538, 0, 0, 0, 0, 0, 0, 0, 0, 519, 519, 497, 497, 511, 492, 517, 475, 12, 7, 0, 0, 0, 0, 0, 0, 503, 496, 503, 521, 494, 541, 538, 517, 0, 0, 0, 0, 0, 0, 0, 0, 468, 497, 544, 489, 505, 479, 506, 536, 9, 4, 0, 0, 0, 0, 0, 0, 513, 530, 500, 514, 516, 501, 516, 476, 28, 15, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4149, 4084, 4067, 4163, 4046, 4113, 4037, 4109] avg=4096.0000 max=4163 max_rank=3 min=4037 min_rank=6 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16305 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[509, 567, 490, 506, 491, 513, 514, 526, 0, 0, 0, 0, 0, 0, 0, 0, 531, 490, 509, 519, 533, 519, 550, 511, 1, 0, 0, 0, 0, 0, 0, 0, 527, 483, 484, 502, 495, 514, 517, 519, 0, 0, 0, 0, 0, 0, 0, 0, 522, 542, 473, 518, 495, 517, 504, 486, 0, 0, 0, 0, 0, 0, 0, 0, 519, 493, 502, 510, 480, 517, 523, 531, 8, 2, 0, 0, 0, 0, 0, 0, 508, 529, 515, 511, 515, 517, 509, 525, 0, 0, 0, 0, 0, 0, 0, 0, 468, 517, 497, 488, 524, 484, 534, 506, 5, 3, 0, 0, 0, 0, 0, 0, 541, 521, 523, 493, 548, 519, 491, 481, 24, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4163, 4041, 4057, 4085, 4129, 4026, 4151] avg=4096.0000 max=4163 max_rank=1 min=4026 min_rank=6 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[472, 492, 493, 506, 516, 488, 479, 485, 0, 0, 0, 0, 0, 0, 0, 0, 534, 484, 515, 511, 511, 527, 483, 512, 1, 0, 0, 0, 0, 0, 0, 0, 512, 513, 478, 512, 525, 501, 541, 490, 0, 0, 0, 0, 0, 0, 0, 0, 495, 533, 527, 515, 544, 498, 510, 511, 0, 0, 0, 0, 0, 0, 0, 0, 506, 498, 533, 514, 510, 558, 487, 507, 8, 2, 0, 0, 0, 0, 0, 0, 506, 523, 524, 517, 513, 515, 510, 509, 0, 0, 0, 0, 0, 0, 0, 0, 531, 564, 514, 510, 506, 510, 510, 486, 6, 3, 0, 0, 0, 0, 0, 0, 486, 513, 546, 532, 500, 547, 507, 510, 23, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3931, 4078, 4072, 4133, 4123, 4117, 4140, 4174] avg=4096.0000 max=4174 max_rank=7 min=3931 min_rank=0 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16554 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[505, 513, 480, 532, 531, 496, 493, 522, 0, 0, 0, 0, 0, 0, 0, 0, 520, 516, 539, 521, 513, 509, 536, 526, 1, 0, 0, 0, 0, 0, 0, 0, 483, 545, 506, 516, 523, 535, 524, 506, 0, 0, 0, 0, 0, 0, 0, 0, 490, 518, 526, 500, 515, 489, 499, 524, 0, 0, 0, 0, 0, 0, 0, 0, 517, 501, 506, 509, 520, 498, 505, 505, 8, 2, 0, 0, 0, 0, 0, 0, 497, 527, 521, 511, 506, 497, 482, 537, 0, 0, 0, 0, 0, 0, 0, 0, 497, 492, 518, 551, 522, 476, 538, 502, 5, 3, 0, 0, 0, 0, 0, 0, 483, 471, 503, 498, 486, 536, 522, 531, 23, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4181, 4138, 4061, 4071, 4078, 4104, 4063] avg=4096.0000 max=4181 max_rank=1 min=4061 min_rank=3 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16316 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[513, 496, 531, 500, 496, 518, 527, 514, 0, 0, 0, 0, 0, 0, 0, 0, 555, 498, 497, 511, 512, 507, 538, 492, 1, 0, 0, 0, 0, 0, 0, 0, 539, 491, 535, 490, 499, 521, 521, 531, 0, 0, 0, 0, 0, 0, 0, 0, 501, 486, 500, 539, 529, 505, 509, 513, 0, 0, 0, 0, 0, 0, 0, 0, 520, 514, 506, 489, 531, 482, 520, 554, 8, 2, 0, 0, 0, 0, 0, 0, 511, 532, 509, 530, 516, 487, 518, 516, 0, 0, 0, 0, 0, 0, 0, 0, 492, 522, 515, 511, 480, 532, 543, 504, 6, 3, 0, 0, 0, 0, 0, 0, 543, 487, 486, 478, 505, 497, 509, 462, 23, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4111, 4127, 4082, 4126, 4119, 4108, 4000] avg=4096.0000 max=4127 max_rank=2 min=4000 min_rank=7 imbalance_ratio=1.007568 max_load_reduction=28641 max_load_reduction_pct=0.874054 moved_to_echo=16353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3968, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[495, 509, 520, 515, 491, 489, 526, 543, 524, 474, 534, 518, 487, 504, 525, 503, 514, 535, 509, 519, 494, 554, 507, 525, 516, 483, 480, 465, 522, 536, 527, 547, 526, 522, 470, 534, 465, 491, 511, 525, 546, 505, 515, 508, 538, 503, 499, 499, 523, 532, 533, 516, 495, 503, 505, 558, 506, 516, 505, 474, 516, 501, 488, 550] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[508, 483, 520, 516, 521, 470, 503, 533, 521, 485, 514, 548, 496, 511, 522, 554, 505, 532, 544, 529, 510, 528, 568, 490, 500, 499, 492, 490, 531, 487, 543, 529, 496, 525, 534, 497, 505, 521, 539, 492, 532, 507, 501, 543, 522, 535, 481, 506, 500, 503, 498, 502, 525, 493, 480, 498, 508, 508, 524, 499, 515, 512, 492, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[519, 519, 533, 517, 537, 498, 478, 532, 516, 521, 500, 461, 524, 502, 527, 468, 532, 512, 465, 561, 488, 503, 520, 520, 538, 553, 522, 537, 506, 546, 491, 568, 559, 484, 509, 526, 491, 547, 478, 510, 467, 496, 517, 513, 487, 533, 512, 523, 485, 501, 531, 487, 523, 525, 517, 528, 506, 509, 502, 489, 542, 494, 492, 471] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[507, 488, 494, 523, 480, 520, 504, 507, 502, 539, 518, 525, 517, 491, 499, 539, 501, 509, 489, 508, 533, 552, 508, 498, 524, 555, 548, 495, 535, 509, 473, 510, 479, 506, 524, 534, 522, 539, 503, 527, 506, 537, 491, 517, 530, 515, 498, 519, 521, 495, 555, 497, 490, 493, 476, 540, 506, 513, 499, 463, 483, 517, 543, 530] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[521, 535, 471, 526, 528, 514, 518, 505, 525, 522, 519, 517, 495, 515, 497, 500, 516, 480, 491, 508, 468, 509, 517, 477, 529, 508, 518, 501, 511, 509, 476, 531, 491, 522, 567, 509, 532, 543, 503, 504, 509, 499, 499, 503, 500, 553, 533, 505, 521, 509, 495, 501, 536, 518, 485, 515, 512, 519, 522, 507, 502, 565, 508, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[551, 506, 501, 524, 494, 481, 521, 552, 479, 467, 521, 522, 518, 507, 511, 502, 499, 523, 514, 489, 541, 502, 553, 498, 524, 546, 520, 511, 520, 506, 527, 493, 554, 551, 529, 491, 524, 513, 485, 508, 569, 505, 501, 478, 490, 514, 554, 515, 485, 502, 524, 536, 528, 494, 515, 485, 481, 499, 484, 528, 496, 528, 496, 483] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[535, 477, 532, 515, 503, 517, 501, 541, 507, 503, 506, 511, 525, 486, 471, 504, 526, 486, 551, 506, 510, 503, 536, 514, 544, 519, 534, 514, 489, 537, 536, 496, 504, 474, 515, 537, 500, 558, 506, 533, 499, 514, 503, 497, 525, 509, 497, 518, 533, 505, 520, 520, 537, 502, 496, 486, 503, 494, 519, 519, 542, 466, 498, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[525, 478, 540, 527, 532, 493, 521, 488, 514, 489, 500, 537, 491, 504, 527, 548, 525, 527, 518, 525, 507, 526, 498, 476, 513, 510, 499, 541, 530, 519, 508, 497, 482, 549, 538, 507, 533, 489, 489, 510, 484, 534, 495, 512, 509, 507, 477, 514, 467, 527, 501, 503, 531, 518, 548, 546, 489, 551, 485, 505, 510, 523, 498, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[495, 509, 520, 515, 491, 489, 526, 543, 0, 0, 0, 0, 0, 0, 0, 0, 524, 474, 534, 518, 487, 504, 525, 503, 16, 2, 0, 0, 0, 0, 0, 0, 514, 535, 509, 519, 494, 554, 494, 525, 0, 0, 0, 0, 0, 0, 0, 0, 479, 483, 480, 465, 522, 536, 527, 547, 0, 0, 0, 0, 0, 0, 0, 0, 526, 522, 470, 534, 465, 472, 511, 525, 0, 0, 0, 0, 0, 0, 0, 0, 546, 505, 515, 508, 538, 503, 499, 499, 5, 0, 0, 0, 0, 0, 0, 0, 523, 532, 533, 516, 495, 503, 505, 558, 6, 0, 0, 0, 0, 0, 0, 0, 506, 516, 505, 474, 516, 501, 488, 550, 37, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4087, 4144, 4039, 4025, 4118, 4171, 4096] avg=4096.0000 max=4171 max_rank=6 min=4025 min_rank=4 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16410 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[519, 519, 533, 517, 537, 498, 478, 532, 0, 0, 0, 0, 0, 0, 0, 0, 516, 521, 500, 461, 524, 502, 527, 468, 18, 2, 0, 0, 0, 0, 0, 0, 532, 512, 465, 561, 488, 503, 507, 520, 0, 0, 0, 0, 0, 0, 0, 0, 499, 553, 522, 537, 506, 546, 491, 568, 0, 0, 0, 0, 0, 0, 0, 0, 559, 484, 509, 526, 491, 525, 478, 510, 0, 0, 0, 0, 0, 0, 0, 0, 467, 496, 517, 513, 487, 533, 512, 523, 5, 0, 0, 0, 0, 0, 0, 0, 485, 501, 531, 487, 523, 525, 517, 528, 6, 0, 0, 0, 0, 0, 0, 0, 506, 509, 502, 489, 542, 494, 492, 471, 39, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4133, 4039, 4088, 4222, 4082, 4053, 4103, 4048] avg=4096.0000 max=4222 max_rank=3 min=4039 min_rank=1 imbalance_ratio=1.030762 max_load_reduction=28546 max_load_reduction_pct=0.871155 moved_to_echo=16286 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[508, 483, 520, 516, 521, 470, 503, 533, 0, 0, 0, 0, 0, 0, 0, 0, 521, 485, 514, 548, 496, 511, 522, 554, 17, 2, 0, 0, 0, 0, 0, 0, 505, 532, 544, 529, 510, 528, 555, 490, 0, 0, 0, 0, 0, 0, 0, 0, 464, 499, 492, 490, 531, 487, 543, 529, 0, 0, 0, 0, 0, 0, 0, 0, 496, 525, 534, 497, 505, 500, 539, 492, 0, 0, 0, 0, 0, 0, 0, 0, 532, 507, 501, 543, 522, 535, 481, 506, 5, 0, 0, 0, 0, 0, 0, 0, 500, 503, 498, 502, 525, 493, 480, 498, 6, 0, 0, 0, 0, 0, 0, 0, 508, 508, 524, 499, 515, 512, 492, 493, 36, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4054, 4170, 4193, 4035, 4088, 4132, 4005, 4091] avg=4096.0000 max=4193 max_rank=2 min=4005 min_rank=6 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16316 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[507, 488, 494, 523, 480, 520, 504, 507, 0, 0, 0, 0, 0, 0, 0, 0, 502, 539, 518, 525, 517, 491, 499, 539, 17, 2, 0, 0, 0, 0, 0, 0, 501, 509, 489, 508, 533, 552, 495, 498, 0, 0, 0, 0, 0, 0, 0, 0, 486, 555, 548, 495, 535, 509, 473, 510, 0, 0, 0, 0, 0, 0, 0, 0, 479, 506, 524, 534, 522, 518, 503, 527, 0, 0, 0, 0, 0, 0, 0, 0, 506, 537, 491, 517, 530, 515, 498, 519, 5, 0, 0, 0, 0, 0, 0, 0, 521, 495, 555, 497, 490, 493, 476, 540, 6, 0, 0, 0, 0, 0, 0, 0, 506, 513, 499, 463, 483, 517, 543, 530, 38, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4023, 4149, 4085, 4111, 4113, 4118, 4073, 4096] avg=4096.0000 max=4149 max_rank=1 min=4023 min_rank=0 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16400 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[535, 477, 532, 515, 503, 517, 501, 541, 0, 0, 0, 0, 0, 0, 0, 0, 507, 503, 506, 511, 525, 486, 471, 504, 18, 2, 0, 0, 0, 0, 0, 0, 526, 486, 551, 506, 510, 503, 523, 514, 0, 0, 0, 0, 0, 0, 0, 0, 505, 519, 534, 514, 489, 537, 536, 496, 0, 0, 0, 0, 0, 0, 0, 0, 504, 474, 515, 537, 500, 536, 506, 533, 0, 0, 0, 0, 0, 0, 0, 0, 499, 514, 503, 497, 525, 509, 497, 518, 5, 0, 0, 0, 0, 0, 0, 0, 533, 505, 520, 520, 537, 502, 496, 486, 6, 0, 0, 0, 0, 0, 0, 0, 503, 494, 519, 519, 542, 466, 498, 504, 39, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4033, 4119, 4130, 4105, 4067, 4105, 4088] avg=4096.0000 max=4130 max_rank=3 min=4033 min_rank=1 imbalance_ratio=1.008301 max_load_reduction=28638 max_load_reduction_pct=0.873962 moved_to_echo=16365 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[521, 535, 471, 526, 528, 514, 518, 502, 0, 0, 0, 0, 0, 0, 0, 0, 525, 522, 519, 517, 495, 515, 497, 500, 21, 5, 0, 0, 0, 0, 0, 0, 516, 480, 491, 508, 468, 509, 495, 477, 0, 0, 0, 0, 0, 0, 0, 0, 488, 508, 518, 501, 511, 509, 476, 531, 0, 0, 0, 0, 0, 0, 0, 0, 491, 522, 567, 509, 532, 515, 503, 504, 0, 0, 0, 0, 0, 0, 0, 0, 509, 499, 499, 503, 500, 553, 533, 505, 8, 3, 0, 0, 0, 0, 0, 0, 521, 509, 495, 501, 536, 518, 485, 515, 9, 0, 0, 0, 0, 0, 0, 0, 512, 519, 522, 507, 502, 565, 508, 524, 41, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4116, 3944, 4042, 4143, 4112, 4089, 4207] avg=4096.0000 max=4207 max_rank=7 min=3944 min_rank=2 imbalance_ratio=1.027100 max_load_reduction=28561 max_load_reduction_pct=0.871613 moved_to_echo=16551 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[551, 506, 501, 524, 494, 481, 521, 552, 0, 0, 0, 0, 0, 0, 0, 0, 479, 467, 521, 522, 518, 507, 511, 502, 17, 2, 0, 0, 0, 0, 0, 0, 499, 523, 514, 489, 541, 502, 540, 498, 0, 0, 0, 0, 0, 0, 0, 0, 486, 546, 520, 511, 520, 506, 527, 493, 0, 0, 0, 0, 0, 0, 0, 0, 554, 551, 529, 491, 524, 492, 485, 508, 0, 0, 0, 0, 0, 0, 0, 0, 569, 505, 501, 478, 490, 514, 554, 515, 5, 0, 0, 0, 0, 0, 0, 0, 485, 502, 524, 536, 528, 494, 515, 485, 6, 0, 0, 0, 0, 0, 0, 0, 481, 499, 484, 528, 496, 528, 496, 483, 38, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4130, 4046, 4106, 4109, 4134, 4131, 4075, 4037] avg=4096.0000 max=4134 max_rank=4 min=4037 min_rank=7 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[525, 478, 540, 527, 532, 493, 521, 488, 0, 0, 0, 0, 0, 0, 0, 0, 514, 489, 500, 537, 491, 504, 527, 548, 16, 2, 0, 0, 0, 0, 0, 0, 525, 527, 518, 525, 507, 526, 485, 476, 0, 0, 0, 0, 0, 0, 0, 0, 476, 510, 499, 541, 530, 519, 508, 497, 0, 0, 0, 0, 0, 0, 0, 0, 482, 549, 538, 507, 533, 470, 489, 510, 0, 0, 0, 0, 0, 0, 0, 0, 484, 534, 495, 512, 509, 507, 477, 514, 5, 0, 0, 0, 0, 0, 0, 0, 467, 527, 501, 503, 531, 518, 548, 546, 6, 0, 0, 0, 0, 0, 0, 0, 489, 551, 485, 505, 510, 523, 498, 504, 37, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4128, 4089, 4080, 4078, 4037, 4147, 4105] avg=4096.0000 max=4147 max_rank=6 min=4037 min_rank=5 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3968, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[546, 537, 477, 516, 540, 550, 517, 517, 498, 507, 479, 494, 524, 529, 494, 506, 481, 500, 527, 514, 482, 495, 505, 509, 484, 522, 529, 523, 521, 523, 528, 536, 544, 536, 501, 519, 515, 491, 533, 520, 511, 515, 490, 495, 465, 498, 507, 482, 519, 539, 510, 491, 532, 503, 498, 515, 506, 532, 508, 533, 510, 507, 505, 528] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[528, 540, 516, 494, 509, 546, 523, 506, 521, 501, 492, 497, 512, 540, 487, 495, 518, 525, 533, 495, 492, 532, 514, 515, 511, 521, 519, 496, 487, 489, 555, 532, 479, 504, 503, 527, 497, 505, 513, 523, 531, 548, 498, 506, 542, 506, 515, 519, 515, 512, 489, 508, 517, 492, 489, 496, 497, 510, 488, 545, 497, 523, 491, 542] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[500, 506, 517, 478, 511, 518, 536, 473, 477, 503, 524, 540, 575, 519, 504, 505, 561, 526, 523, 494, 474, 543, 508, 511, 504, 535, 503, 512, 471, 526, 504, 491, 515, 531, 509, 503, 505, 517, 503, 513, 485, 519, 474, 528, 497, 544, 496, 520, 527, 507, 495, 537, 505, 504, 524, 511, 477, 560, 502, 541, 476, 482, 529, 560] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[525, 527, 514, 528, 513, 491, 517, 514, 517, 522, 542, 514, 543, 541, 536, 496, 512, 499, 492, 523, 511, 502, 523, 489, 507, 500, 538, 513, 532, 513, 499, 507, 516, 523, 511, 522, 524, 516, 492, 506, 478, 516, 520, 492, 538, 478, 500, 509, 504, 484, 508, 446, 527, 557, 508, 496, 506, 536, 515, 501, 524, 520, 500, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[498, 513, 518, 561, 533, 579, 534, 515, 520, 506, 498, 487, 562, 509, 521, 488, 520, 512, 466, 530, 489, 496, 481, 515, 510, 540, 518, 498, 494, 478, 518, 477, 496, 490, 493, 517, 504, 491, 522, 506, 498, 514, 539, 559, 509, 514, 501, 540, 472, 557, 485, 555, 500, 498, 533, 527, 492, 482, 489, 484, 537, 534, 487, 559] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[498, 520, 505, 499, 480, 482, 522, 508, 497, 505, 515, 505, 482, 495, 505, 519, 502, 508, 513, 539, 526, 518, 529, 537, 501, 512, 510, 535, 498, 520, 505, 544, 519, 479, 529, 487, 498, 530, 524, 511, 513, 513, 529, 544, 521, 523, 536, 573, 506, 515, 516, 532, 504, 511, 513, 507, 497, 488, 481, 477, 507, 493, 509, 549] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[532, 506, 510, 486, 486, 478, 514, 547, 484, 539, 500, 500, 494, 501, 498, 513, 513, 510, 542, 483, 511, 489, 534, 540, 492, 530, 514, 517, 521, 515, 534, 515, 531, 495, 469, 540, 511, 511, 505, 532, 479, 480, 546, 506, 529, 489, 542, 526, 518, 526, 525, 500, 511, 524, 495, 534, 531, 523, 481, 519, 494, 500, 513, 535] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[477, 552, 487, 488, 481, 546, 505, 503, 486, 511, 503, 474, 535, 525, 521, 490, 495, 490, 507, 534, 512, 493, 532, 526, 510, 506, 513, 482, 503, 507, 509, 531, 500, 475, 539, 523, 498, 520, 504, 548, 499, 517, 539, 488, 523, 497, 527, 529, 577, 529, 516, 501, 517, 506, 525, 544, 498, 516, 528, 494, 522, 499, 522, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[498, 498, 518, 561, 533, 579, 534, 515, 0, 0, 0, 0, 0, 0, 0, 0, 520, 506, 498, 487, 562, 509, 521, 488, 15, 2, 0, 0, 0, 0, 0, 0, 520, 512, 466, 530, 489, 496, 481, 515, 3, 2, 0, 0, 0, 0, 0, 0, 510, 538, 518, 498, 494, 478, 518, 477, 0, 0, 0, 0, 0, 0, 0, 0, 496, 490, 493, 517, 504, 491, 522, 506, 6, 0, 0, 0, 0, 0, 0, 0, 498, 514, 539, 559, 509, 514, 501, 530, 0, 0, 0, 0, 0, 0, 0, 0, 472, 548, 485, 555, 500, 498, 533, 527, 0, 0, 0, 0, 0, 0, 0, 0, 492, 482, 489, 484, 537, 534, 487, 559, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4236, 4108, 4014, 4031, 4025, 4164, 4118, 4072] avg=4096.0000 max=4236 max_rank=0 min=4014 min_rank=2 imbalance_ratio=1.034180 max_load_reduction=28532 max_load_reduction_pct=0.870728 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[500, 491, 517, 478, 511, 518, 536, 473, 0, 0, 0, 0, 0, 0, 0, 0, 477, 503, 524, 540, 575, 519, 504, 505, 15, 2, 0, 0, 0, 0, 0, 0, 561, 526, 523, 494, 474, 543, 508, 511, 3, 2, 0, 0, 0, 0, 0, 0, 504, 533, 503, 512, 471, 526, 504, 491, 0, 0, 0, 0, 0, 0, 0, 0, 515, 531, 509, 503, 505, 517, 503, 513, 5, 0, 0, 0, 0, 0, 0, 0, 485, 519, 474, 528, 497, 544, 496, 510, 0, 0, 0, 0, 0, 0, 0, 0, 527, 499, 495, 537, 505, 504, 524, 511, 0, 0, 0, 0, 0, 0, 0, 0, 477, 560, 502, 541, 476, 482, 529, 560, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4024, 4164, 4145, 4044, 4101, 4053, 4102, 4135] avg=4096.0000 max=4164 max_rank=1 min=4024 min_rank=0 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[525, 512, 514, 528, 513, 491, 517, 514, 0, 0, 0, 0, 0, 0, 0, 0, 517, 522, 542, 514, 543, 541, 536, 496, 15, 2, 0, 0, 0, 0, 0, 0, 512, 499, 492, 523, 511, 502, 523, 489, 3, 2, 0, 0, 0, 0, 0, 0, 507, 498, 538, 513, 532, 513, 499, 507, 0, 0, 0, 0, 0, 0, 0, 0, 516, 523, 511, 522, 524, 516, 492, 506, 5, 0, 0, 0, 0, 0, 0, 0, 478, 516, 520, 492, 538, 478, 500, 499, 0, 0, 0, 0, 0, 0, 0, 0, 504, 476, 508, 446, 527, 557, 508, 496, 0, 0, 0, 0, 0, 0, 0, 0, 506, 536, 515, 501, 524, 520, 500, 495, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4228, 4056, 4107, 4115, 4021, 4022, 4105] avg=4096.0000 max=4228 max_rank=1 min=4021 min_rank=5 imbalance_ratio=1.032227 max_load_reduction=28540 max_load_reduction_pct=0.870972 moved_to_echo=16263 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[546, 522, 477, 516, 540, 550, 517, 517, 0, 0, 0, 0, 0, 0, 0, 0, 498, 507, 479, 494, 524, 529, 494, 506, 15, 2, 0, 0, 0, 0, 0, 0, 481, 500, 527, 514, 482, 495, 505, 509, 3, 2, 0, 0, 0, 0, 0, 0, 484, 520, 529, 523, 521, 523, 528, 536, 0, 0, 0, 0, 0, 0, 0, 0, 544, 536, 501, 519, 515, 491, 533, 520, 6, 0, 0, 0, 0, 0, 0, 0, 511, 515, 490, 495, 465, 498, 507, 473, 0, 0, 0, 0, 0, 0, 0, 0, 519, 530, 510, 491, 532, 503, 498, 515, 0, 0, 0, 0, 0, 0, 0, 0, 506, 532, 508, 533, 510, 507, 505, 528, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4185, 4048, 4018, 4164, 4165, 3954, 4098, 4136] avg=4096.0000 max=4185 max_rank=0 min=3954 min_rank=5 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[528, 524, 516, 494, 509, 546, 523, 506, 0, 0, 0, 0, 0, 0, 0, 0, 521, 501, 492, 497, 512, 540, 487, 495, 16, 2, 0, 0, 0, 0, 0, 0, 518, 525, 533, 495, 492, 532, 514, 515, 3, 2, 0, 0, 0, 0, 0, 0, 511, 519, 519, 496, 487, 489, 555, 532, 0, 0, 0, 0, 0, 0, 0, 0, 479, 504, 503, 527, 497, 505, 513, 523, 5, 0, 0, 0, 0, 0, 0, 0, 531, 548, 498, 506, 542, 506, 515, 509, 0, 0, 0, 0, 0, 0, 0, 0, 515, 504, 489, 508, 517, 492, 489, 496, 0, 0, 0, 0, 0, 0, 0, 0, 497, 510, 488, 545, 497, 523, 491, 542, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4146, 4063, 4129, 4108, 4056, 4155, 4010, 4101] avg=4096.0000 max=4155 max_rank=5 min=4010 min_rank=6 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16322 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[498, 502, 505, 499, 480, 482, 522, 508, 0, 0, 0, 0, 0, 0, 0, 0, 497, 505, 515, 505, 482, 495, 505, 519, 18, 7, 0, 0, 0, 0, 0, 0, 502, 508, 513, 539, 526, 518, 529, 537, 7, 6, 0, 0, 0, 0, 0, 0, 501, 506, 510, 535, 498, 520, 505, 544, 0, 0, 0, 0, 0, 0, 0, 0, 519, 479, 529, 487, 498, 530, 524, 511, 9, 7, 0, 0, 0, 0, 0, 0, 513, 513, 529, 544, 521, 523, 536, 546, 0, 0, 0, 0, 0, 0, 0, 0, 506, 499, 516, 532, 504, 511, 513, 507, 0, 0, 0, 0, 0, 0, 0, 0, 497, 488, 481, 477, 507, 493, 509, 549, 13, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3996, 4048, 4185, 4119, 4093, 4225, 4088, 4014] avg=4096.0000 max=4225 max_rank=5 min=3996 min_rank=0 imbalance_ratio=1.031494 max_load_reduction=28543 max_load_reduction_pct=0.871063 moved_to_echo=16420 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[532, 491, 510, 486, 486, 478, 514, 547, 0, 0, 0, 0, 0, 0, 0, 0, 484, 539, 500, 500, 494, 501, 498, 513, 15, 2, 0, 0, 0, 0, 0, 0, 513, 510, 542, 483, 511, 489, 534, 540, 3, 2, 0, 0, 0, 0, 0, 0, 492, 528, 514, 517, 521, 515, 534, 515, 0, 0, 0, 0, 0, 0, 0, 0, 531, 495, 469, 540, 511, 511, 505, 532, 6, 0, 0, 0, 0, 0, 0, 0, 479, 480, 546, 506, 529, 489, 542, 516, 0, 0, 0, 0, 0, 0, 0, 0, 518, 517, 525, 500, 511, 524, 495, 534, 0, 0, 0, 0, 0, 0, 0, 0, 531, 523, 481, 519, 494, 500, 513, 535, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4044, 4046, 4127, 4136, 4100, 4087, 4124, 4104] avg=4096.0000 max=4136 max_rank=3 min=4044 min_rank=0 imbalance_ratio=1.009766 max_load_reduction=28632 max_load_reduction_pct=0.873779 moved_to_echo=16415 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[477, 536, 487, 488, 481, 546, 505, 503, 0, 0, 0, 0, 0, 0, 0, 0, 486, 511, 503, 474, 535, 525, 521, 490, 16, 2, 0, 0, 0, 0, 0, 0, 495, 490, 507, 534, 512, 493, 532, 526, 3, 2, 0, 0, 0, 0, 0, 0, 510, 504, 513, 482, 503, 507, 509, 531, 0, 0, 0, 0, 0, 0, 0, 0, 500, 475, 539, 523, 498, 520, 504, 548, 6, 0, 0, 0, 0, 0, 0, 0, 499, 517, 539, 488, 523, 497, 527, 519, 0, 0, 0, 0, 0, 0, 0, 0, 577, 520, 516, 501, 517, 506, 525, 544, 0, 0, 0, 0, 0, 0, 0, 0, 498, 516, 528, 494, 522, 499, 522, 514, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4023, 4063, 4094, 4059, 4113, 4109, 4206, 4101] avg=4096.0000 max=4206 max_rank=6 min=4023 min_rank=0 imbalance_ratio=1.026855 max_load_reduction=28562 max_load_reduction_pct=0.871643 moved_to_echo=16529 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4352, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4096, 4352, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True + [2026-09-21 08:26:49] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 261.0 | throughput per GPU (TFLOP/s/GPU): 123.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.897497E+00 | loss scale: 1.0 | grad norm: 7.460 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[506, 476, 514, 516, 516, 524, 520, 519, 502, 501, 558, 513, 540, 486, 511, 518, 459, 485, 539, 522, 512, 511, 522, 529, 520, 535, 501, 503, 485, 548, 531, 538, 517, 490, 481, 470, 482, 536, 527, 555, 481, 517, 568, 514, 507, 516, 527, 452, 486, 509, 539, 513, 476, 495, 533, 535, 515, 486, 506, 466, 526, 493, 564, 526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[530, 525, 472, 546, 498, 492, 527, 463, 498, 524, 516, 514, 582, 481, 511, 541, 545, 522, 514, 529, 502, 448, 510, 487, 481, 484, 470, 512, 491, 495, 496, 547, 473, 524, 512, 528, 495, 493, 566, 501, 517, 492, 511, 543, 528, 519, 479, 477, 504, 500, 581, 507, 534, 529, 487, 530, 507, 539, 537, 522, 507, 529, 503, 541] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[495, 480, 527, 515, 527, 541, 468, 512, 509, 486, 522, 496, 532, 533, 511, 522, 525, 488, 557, 541, 499, 487, 539, 473, 498, 528, 514, 513, 479, 521, 514, 496, 505, 477, 516, 500, 503, 538, 503, 523, 507, 497, 499, 526, 526, 523, 542, 550, 546, 519, 496, 499, 529, 525, 531, 520, 505, 465, 503, 504, 510, 493, 518, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[507, 498, 500, 484, 515, 515, 497, 501, 514, 511, 537, 541, 521, 536, 530, 510, 489, 526, 484, 508, 468, 531, 523, 524, 557, 548, 521, 485, 506, 540, 491, 528, 492, 491, 503, 499, 500, 534, 490, 506, 513, 560, 500, 524, 507, 508, 545, 520, 509, 520, 530, 517, 486, 478, 471, 497, 562, 528, 463, 518, 526, 499, 510, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[527, 533, 525, 519, 537, 534, 479, 491, 492, 540, 512, 519, 538, 495, 539, 491, 504, 517, 501, 494, 499, 515, 478, 529, 514, 511, 537, 573, 488, 472, 538, 446, 505, 516, 504, 535, 528, 512, 507, 527, 512, 524, 499, 519, 508, 537, 503, 513, 490, 522, 532, 486, 470, 560, 472, 500, 488, 528, 522, 517, 514, 504, 495, 532] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[532, 503, 502, 531, 498, 466, 484, 509, 531, 498, 515, 521, 507, 516, 518, 530, 493, 506, 482, 477, 500, 516, 490, 507, 523, 510, 535, 478, 517, 544, 492, 504, 496, 521, 540, 555, 517, 530, 522, 537, 541, 460, 474, 444, 525, 543, 536, 544, 509, 521, 504, 521, 571, 478, 496, 510, 517, 558, 486, 533, 493, 483, 546, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[499, 523, 485, 513, 545, 537, 510, 526, 515, 548, 519, 548, 505, 507, 527, 519, 513, 436, 519, 532, 498, 526, 536, 517, 530, 513, 507, 477, 484, 550, 516, 504, 514, 492, 513, 495, 519, 528, 515, 535, 513, 515, 504, 516, 485, 528, 516, 509, 492, 511, 479, 521, 513, 487, 490, 498, 497, 505, 520, 515, 518, 532, 510, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[532, 493, 475, 498, 524, 498, 526, 543, 488, 519, 525, 532, 486, 482, 520, 519, 540, 494, 519, 483, 523, 522, 548, 531, 482, 530, 519, 474, 494, 581, 494, 506, 524, 457, 524, 532, 535, 522, 535, 523, 506, 533, 537, 489, 477, 525, 520, 514, 521, 540, 485, 505, 482, 495, 501, 544, 503, 466, 511, 486, 538, 547, 516, 475] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[530, 525, 472, 546, 498, 492, 527, 463, 18, 1, 0, 0, 0, 0, 0, 0, 498, 524, 516, 514, 533, 481, 511, 541, 0, 0, 0, 0, 0, 0, 0, 0, 545, 522, 514, 529, 502, 448, 510, 487, 31, 0, 0, 0, 0, 0, 0, 0, 481, 484, 470, 512, 491, 495, 496, 547, 3, 2, 0, 0, 0, 0, 0, 0, 473, 524, 512, 528, 495, 493, 566, 493, 0, 0, 0, 0, 0, 0, 0, 0, 517, 492, 511, 543, 528, 508, 479, 477, 0, 0, 0, 0, 0, 0, 0, 0, 504, 500, 581, 507, 534, 529, 487, 530, 10, 5, 0, 0, 0, 0, 0, 0, 507, 539, 537, 522, 507, 529, 501, 541, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4118, 4088, 3981, 4084, 4055, 4187, 4183] avg=4096.0000 max=4187 max_rank=6 min=3981 min_rank=3 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16509 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[495, 480, 527, 515, 527, 541, 468, 512, 17, 1, 0, 0, 0, 0, 0, 0, 509, 486, 522, 496, 487, 533, 511, 522, 0, 0, 0, 0, 0, 0, 0, 0, 525, 488, 557, 541, 499, 487, 539, 473, 28, 0, 0, 0, 0, 0, 0, 0, 498, 528, 514, 513, 479, 521, 514, 496, 3, 2, 0, 0, 0, 0, 0, 0, 505, 477, 516, 500, 503, 538, 503, 515, 0, 0, 0, 0, 0, 0, 0, 0, 507, 497, 499, 526, 526, 512, 542, 550, 0, 0, 0, 0, 0, 0, 0, 0, 546, 519, 496, 499, 529, 525, 531, 520, 10, 5, 0, 0, 0, 0, 0, 0, 505, 465, 503, 504, 510, 493, 516, 522, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4066, 4137, 4068, 4057, 4159, 4180, 4018] avg=4096.0000 max=4180 max_rank=6 min=4018 min_rank=7 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16414 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[506, 476, 514, 516, 516, 524, 520, 519, 17, 1, 0, 0, 0, 0, 0, 0, 502, 501, 558, 513, 495, 486, 511, 518, 0, 0, 0, 0, 0, 0, 0, 0, 459, 485, 539, 522, 512, 511, 522, 529, 28, 0, 0, 0, 0, 0, 0, 0, 520, 535, 501, 503, 485, 548, 531, 538, 4, 2, 0, 0, 0, 0, 0, 0, 517, 490, 481, 470, 482, 536, 527, 546, 0, 0, 0, 0, 0, 0, 0, 0, 481, 517, 568, 514, 507, 505, 527, 452, 0, 0, 0, 0, 0, 0, 0, 0, 486, 509, 539, 513, 476, 495, 533, 535, 10, 5, 0, 0, 0, 0, 0, 0, 515, 486, 506, 466, 526, 493, 562, 526, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4084, 4107, 4167, 4049, 4071, 4101, 4080] avg=4096.0000 max=4167 max_rank=3 min=4049 min_rank=4 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16301 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[507, 498, 500, 484, 515, 515, 497, 501, 16, 1, 0, 0, 0, 0, 0, 0, 514, 511, 537, 541, 478, 536, 530, 510, 0, 0, 0, 0, 0, 0, 0, 0, 489, 526, 484, 508, 468, 531, 523, 524, 27, 0, 0, 0, 0, 0, 0, 0, 557, 548, 521, 485, 506, 540, 491, 528, 3, 2, 0, 0, 0, 0, 0, 0, 492, 491, 503, 499, 500, 534, 490, 498, 0, 0, 0, 0, 0, 0, 0, 0, 513, 560, 500, 524, 507, 497, 545, 520, 0, 0, 0, 0, 0, 0, 0, 0, 509, 520, 530, 517, 486, 478, 471, 497, 10, 5, 0, 0, 0, 0, 0, 0, 562, 528, 463, 518, 526, 499, 508, 516, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4034, 4157, 4080, 4181, 4007, 4166, 4023, 4120] avg=4096.0000 max=4181 max_rank=3 min=4007 min_rank=4 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16316 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[527, 533, 525, 519, 537, 534, 479, 491, 17, 1, 0, 0, 0, 0, 0, 0, 492, 540, 512, 519, 493, 495, 539, 491, 0, 0, 0, 0, 0, 0, 0, 0, 504, 517, 501, 494, 499, 515, 478, 529, 28, 0, 0, 0, 0, 0, 0, 0, 514, 511, 537, 573, 488, 472, 538, 446, 4, 2, 0, 0, 0, 0, 0, 0, 505, 516, 504, 535, 528, 512, 507, 518, 0, 0, 0, 0, 0, 0, 0, 0, 512, 524, 499, 519, 508, 526, 503, 513, 0, 0, 0, 0, 0, 0, 0, 0, 490, 522, 532, 486, 470, 560, 472, 500, 10, 5, 0, 0, 0, 0, 0, 0, 488, 528, 522, 517, 514, 504, 493, 532, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4163, 4081, 4065, 4085, 4125, 4104, 4047, 4098] avg=4096.0000 max=4163 max_rank=0 min=4047 min_rank=6 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16374 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[532, 503, 502, 531, 498, 466, 484, 509, 19, 3, 0, 0, 0, 0, 0, 0, 531, 498, 515, 521, 456, 516, 518, 530, 0, 0, 0, 0, 0, 0, 0, 0, 493, 506, 482, 477, 500, 516, 490, 507, 32, 0, 0, 0, 0, 0, 0, 0, 523, 510, 535, 478, 517, 544, 492, 504, 8, 3, 0, 0, 0, 0, 0, 0, 496, 521, 540, 555, 517, 530, 522, 519, 0, 0, 0, 0, 0, 0, 0, 0, 541, 460, 474, 444, 525, 524, 536, 544, 0, 0, 0, 0, 0, 0, 0, 0, 509, 521, 504, 521, 571, 478, 496, 510, 16, 10, 0, 0, 0, 0, 0, 0, 517, 558, 486, 533, 493, 483, 543, 522, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4047, 4085, 4003, 4114, 4200, 4048, 4136, 4135] avg=4096.0000 max=4200 max_rank=4 min=4003 min_rank=2 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16519 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[499, 523, 485, 513, 545, 537, 510, 526, 16, 1, 0, 0, 0, 0, 0, 0, 515, 548, 519, 548, 463, 507, 527, 519, 0, 0, 0, 0, 0, 0, 0, 0, 513, 436, 519, 532, 498, 526, 536, 517, 26, 0, 0, 0, 0, 0, 0, 0, 530, 513, 507, 477, 484, 550, 516, 504, 4, 2, 0, 0, 0, 0, 0, 0, 514, 492, 513, 495, 519, 528, 515, 526, 0, 0, 0, 0, 0, 0, 0, 0, 513, 515, 504, 516, 485, 517, 516, 509, 0, 0, 0, 0, 0, 0, 0, 0, 492, 511, 479, 521, 513, 487, 490, 498, 10, 5, 0, 0, 0, 0, 0, 0, 497, 505, 520, 515, 518, 532, 508, 499, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4155, 4146, 4103, 4087, 4102, 4075, 4006, 4094] avg=4096.0000 max=4155 max_rank=0 min=4006 min_rank=6 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16277 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[532, 493, 475, 498, 524, 498, 526, 543, 15, 1, 0, 0, 0, 0, 0, 0, 488, 519, 525, 532, 446, 482, 520, 519, 0, 0, 0, 0, 0, 0, 0, 0, 540, 494, 519, 483, 523, 522, 548, 531, 25, 0, 0, 0, 0, 0, 0, 0, 482, 530, 519, 474, 494, 581, 494, 506, 3, 2, 0, 0, 0, 0, 0, 0, 524, 457, 524, 532, 535, 522, 535, 515, 0, 0, 0, 0, 0, 0, 0, 0, 506, 533, 537, 489, 477, 514, 520, 514, 0, 0, 0, 0, 0, 0, 0, 0, 521, 540, 485, 505, 482, 495, 501, 544, 10, 5, 0, 0, 0, 0, 0, 0, 503, 466, 511, 486, 538, 547, 514, 475, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4031, 4185, 4085, 4144, 4090, 4088, 4040] avg=4096.0000 max=4185 max_rank=2 min=4031 min_rank=1 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16362 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4224, 3968, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 3968, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4224, 4096, 3968, 4352, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[531, 503, 513, 511, 479, 494, 483, 523, 530, 488, 502, 463, 468, 534, 495, 515, 546, 525, 571, 490, 530, 495, 531, 552, 487, 545, 522, 522, 516, 505, 541, 499, 518, 481, 524, 468, 519, 491, 503, 527, 487, 474, 519, 508, 501, 497, 525, 517, 541, 510, 492, 539, 529, 525, 502, 549, 509, 522, 525, 516, 517, 498, 494, 532] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[525, 521, 513, 527, 529, 493, 498, 491, 495, 486, 521, 516, 520, 466, 515, 523, 505, 502, 521, 504, 487, 529, 527, 495, 483, 516, 493, 527, 498, 507, 527, 520, 487, 529, 528, 530, 495, 515, 500, 497, 552, 514, 505, 513, 511, 519, 534, 551, 515, 549, 492, 522, 537, 499, 486, 518, 549, 499, 513, 505, 514, 509, 500, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[509, 551, 470, 518, 476, 509, 464, 542, 511, 519, 525, 504, 516, 520, 466, 478, 502, 500, 532, 529, 524, 494, 505, 521, 521, 519, 481, 499, 536, 524, 492, 533, 500, 545, 499, 514, 500, 484, 535, 539, 504, 490, 484, 526, 517, 535, 569, 503, 492, 522, 486, 547, 514, 499, 543, 489, 533, 497, 526, 477, 529, 520, 523, 537] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[534, 559, 504, 542, 484, 521, 486, 529, 497, 503, 509, 543, 527, 508, 532, 507, 533, 521, 544, 513, 521, 520, 510, 559, 507, 490, 495, 505, 493, 521, 479, 530, 532, 507, 498, 492, 506, 524, 512, 500, 521, 496, 499, 516, 495, 498, 500, 509, 497, 526, 484, 506, 542, 515, 533, 525, 482, 505, 490, 535, 491, 500, 529, 477] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[508, 498, 523, 506, 529, 533, 543, 537, 468, 500, 548, 523, 506, 493, 496, 495, 520, 550, 516, 510, 513, 527, 486, 552, 524, 480, 467, 508, 517, 541, 534, 527, 497, 499, 508, 494, 504, 518, 491, 500, 491, 507, 550, 525, 478, 524, 507, 488, 476, 491, 532, 499, 519, 531, 527, 514, 533, 521, 525, 484, 516, 513, 506, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[532, 515, 502, 530, 465, 527, 543, 518, 534, 470, 539, 490, 481, 517, 540, 486, 529, 496, 560, 507, 492, 494, 499, 517, 525, 506, 520, 538, 513, 508, 543, 500, 535, 489, 481, 511, 521, 512, 494, 449, 536, 506, 525, 492, 532, 522, 511, 533, 497, 529, 516, 516, 496, 509, 539, 505, 499, 515, 529, 550, 506, 500, 482, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[489, 501, 513, 496, 493, 491, 526, 504, 515, 493, 504, 498, 486, 492, 502, 496, 511, 501, 503, 519, 483, 471, 519, 501, 533, 531, 522, 529, 507, 544, 506, 513, 520, 518, 536, 535, 512, 533, 506, 491, 537, 532, 547, 509, 491, 506, 519, 480, 470, 550, 551, 515, 543, 509, 536, 496, 493, 518, 533, 532, 537, 512, 510, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[513, 517, 471, 525, 528, 536, 515, 493, 501, 508, 521, 512, 506, 463, 527, 502, 511, 531, 504, 514, 492, 568, 509, 520, 522, 492, 524, 521, 500, 515, 501, 510, 507, 529, 493, 513, 540, 523, 522, 538, 496, 479, 473, 476, 520, 525, 504, 549, 505, 497, 475, 524, 526, 516, 480, 522, 512, 544, 499, 572, 527, 511, 521, 478] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[531, 503, 513, 511, 479, 494, 483, 523, 1, 0, 0, 0, 0, 0, 0, 0, 530, 488, 502, 463, 468, 534, 495, 515, 39, 19, 0, 0, 0, 0, 0, 0, 546, 525, 532, 490, 530, 495, 531, 552, 0, 0, 0, 0, 0, 0, 0, 0, 487, 545, 522, 522, 516, 495, 541, 499, 0, 0, 0, 0, 0, 0, 0, 0, 518, 481, 524, 468, 519, 491, 503, 527, 10, 6, 1, 0, 0, 0, 0, 0, 487, 474, 519, 508, 501, 497, 525, 517, 1, 0, 0, 0, 0, 0, 0, 0, 541, 510, 492, 539, 509, 525, 502, 549, 0, 0, 0, 0, 0, 0, 0, 0, 509, 522, 525, 508, 517, 498, 494, 532, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4038, 4053, 4201, 4127, 4048, 4029, 4167, 4105] avg=4096.0000 max=4201 max_rank=2 min=4029 min_rank=5 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16349 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[525, 521, 513, 527, 529, 493, 498, 491, 1, 0, 0, 0, 0, 0, 0, 0, 495, 486, 521, 516, 520, 466, 515, 523, 36, 20, 0, 0, 0, 0, 0, 0, 505, 502, 485, 504, 487, 529, 527, 495, 0, 0, 0, 0, 0, 0, 0, 0, 483, 516, 493, 527, 498, 497, 527, 520, 0, 0, 0, 0, 0, 0, 0, 0, 487, 529, 528, 530, 495, 515, 500, 497, 10, 6, 1, 0, 0, 0, 0, 0, 552, 514, 505, 513, 511, 519, 534, 551, 1, 0, 0, 0, 0, 0, 0, 0, 515, 549, 492, 522, 516, 499, 486, 518, 0, 0, 0, 0, 0, 0, 0, 0, 549, 499, 513, 497, 514, 509, 500, 501, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4098, 4034, 4061, 4098, 4200, 4097, 4082] avg=4096.0000 max=4200 max_rank=5 min=4034 min_rank=2 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16477 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[509, 551, 470, 518, 476, 509, 464, 542, 1, 0, 0, 0, 0, 0, 0, 0, 511, 519, 525, 504, 516, 520, 466, 478, 36, 19, 0, 0, 0, 0, 0, 0, 502, 500, 496, 529, 524, 494, 505, 521, 0, 0, 0, 0, 0, 0, 0, 0, 521, 519, 481, 499, 536, 514, 492, 533, 0, 0, 0, 0, 0, 0, 0, 0, 500, 545, 499, 514, 500, 484, 535, 539, 10, 6, 1, 0, 0, 0, 0, 0, 504, 490, 484, 526, 517, 535, 569, 503, 1, 0, 0, 0, 0, 0, 0, 0, 492, 522, 486, 547, 494, 499, 543, 489, 0, 0, 0, 0, 0, 0, 0, 0, 533, 497, 526, 469, 529, 520, 523, 537, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4040, 4094, 4071, 4095, 4133, 4129, 4072, 4134] avg=4096.0000 max=4134 max_rank=7 min=4040 min_rank=0 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16468 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[534, 559, 504, 542, 484, 521, 486, 529, 2, 0, 0, 0, 0, 0, 0, 0, 497, 503, 509, 543, 527, 508, 532, 507, 37, 20, 0, 0, 0, 0, 0, 0, 533, 521, 507, 513, 521, 520, 510, 559, 0, 0, 0, 0, 0, 0, 0, 0, 507, 490, 495, 505, 493, 511, 479, 530, 0, 0, 0, 0, 0, 0, 0, 0, 532, 507, 498, 492, 506, 524, 512, 500, 10, 7, 1, 0, 0, 0, 0, 0, 521, 496, 499, 516, 495, 498, 500, 509, 1, 0, 0, 0, 0, 0, 0, 0, 497, 526, 484, 506, 521, 515, 533, 525, 0, 0, 0, 0, 0, 0, 0, 0, 482, 505, 490, 525, 491, 500, 529, 477, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4161, 4183, 4184, 4010, 4089, 4035, 4107, 3999] avg=4096.0000 max=4184 max_rank=2 min=3999 min_rank=7 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16230 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[508, 498, 523, 506, 529, 533, 543, 537, 1, 0, 0, 0, 0, 0, 0, 0, 468, 500, 548, 523, 506, 493, 496, 495, 35, 19, 0, 0, 0, 0, 0, 0, 520, 550, 481, 510, 513, 527, 486, 552, 0, 0, 0, 0, 0, 0, 0, 0, 524, 480, 467, 508, 517, 530, 534, 527, 0, 0, 0, 0, 0, 0, 0, 0, 497, 499, 508, 494, 504, 518, 491, 500, 11, 6, 1, 0, 0, 0, 0, 0, 491, 507, 550, 525, 478, 524, 507, 488, 1, 0, 0, 0, 0, 0, 0, 0, 476, 491, 532, 499, 499, 531, 527, 514, 0, 0, 0, 0, 0, 0, 0, 0, 533, 521, 525, 476, 516, 513, 506, 522, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4178, 4083, 4139, 4087, 4029, 4071, 4069, 4112] avg=4096.0000 max=4178 max_rank=0 min=4029 min_rank=4 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16281 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[532, 515, 502, 530, 465, 527, 543, 518, 2, 0, 0, 0, 0, 0, 0, 0, 534, 470, 539, 490, 481, 517, 540, 486, 38, 18, 0, 0, 0, 0, 0, 0, 529, 496, 522, 507, 492, 494, 499, 517, 0, 0, 0, 0, 0, 0, 0, 0, 525, 506, 520, 538, 513, 498, 543, 500, 0, 0, 0, 0, 0, 0, 0, 0, 535, 489, 481, 511, 521, 512, 494, 449, 10, 7, 1, 0, 0, 0, 0, 0, 536, 506, 525, 492, 532, 522, 511, 533, 1, 0, 0, 0, 0, 0, 0, 0, 497, 529, 516, 516, 477, 509, 539, 505, 0, 0, 0, 0, 0, 0, 0, 0, 499, 515, 529, 540, 506, 500, 482, 495, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4134, 4113, 4056, 4143, 4010, 4158, 4088, 4066] avg=4096.0000 max=4158 max_rank=5 min=4010 min_rank=4 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16322 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[513, 517, 471, 525, 528, 536, 515, 493, 6, 0, 0, 0, 0, 0, 0, 0, 501, 508, 521, 512, 506, 463, 527, 502, 40, 24, 0, 0, 0, 0, 0, 0, 511, 531, 464, 514, 492, 568, 509, 520, 0, 0, 0, 0, 0, 0, 0, 0, 522, 492, 524, 521, 500, 501, 501, 510, 0, 0, 0, 0, 0, 0, 0, 0, 507, 529, 493, 513, 540, 523, 522, 538, 14, 10, 2, 0, 0, 0, 0, 0, 496, 479, 473, 476, 520, 525, 504, 549, 2, 0, 0, 0, 0, 0, 0, 0, 505, 497, 475, 524, 500, 516, 480, 522, 0, 0, 0, 0, 0, 0, 0, 0, 512, 544, 499, 554, 527, 511, 521, 478, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4104, 4109, 4071, 4191, 4024, 4019, 4146] avg=4096.0000 max=4191 max_rank=4 min=4019 min_rank=6 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[489, 501, 513, 496, 493, 491, 526, 504, 2, 0, 0, 0, 0, 0, 0, 0, 515, 493, 504, 498, 486, 492, 502, 496, 34, 20, 0, 0, 0, 0, 0, 0, 511, 501, 469, 519, 483, 471, 519, 501, 0, 0, 0, 0, 0, 0, 0, 0, 533, 531, 522, 529, 507, 533, 506, 513, 0, 0, 0, 0, 0, 0, 0, 0, 520, 518, 536, 535, 512, 533, 506, 491, 11, 7, 1, 0, 0, 0, 0, 0, 537, 532, 547, 509, 491, 506, 519, 480, 1, 0, 0, 0, 0, 0, 0, 0, 470, 550, 551, 515, 522, 509, 536, 496, 0, 0, 0, 0, 0, 0, 0, 0, 493, 518, 533, 522, 537, 512, 510, 499, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4015, 4040, 3974, 4174, 4170, 4122, 4149, 4124] avg=4096.0000 max=4174 max_rank=3 min=3974 min_rank=2 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16565 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 3968, 4224, 4096, 4096, 4096, 4096, 4096, 384, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 3968, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[524, 486, 481, 523, 464, 506, 535, 523, 520, 537, 510, 519, 542, 476, 535, 541, 472, 471, 560, 506, 496, 519, 514, 510, 491, 526, 501, 526, 515, 516, 501, 539, 545, 491, 533, 519, 532, 515, 533, 531, 523, 494, 488, 501, 510, 530, 522, 476, 513, 511, 500, 470, 531, 511, 558, 533, 519, 503, 477, 516, 519, 484, 506, 489] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[541, 526, 545, 542, 516, 459, 491, 483, 546, 514, 518, 548, 505, 497, 527, 483, 513, 501, 498, 529, 474, 514, 561, 564, 468, 512, 518, 529, 480, 525, 523, 490, 566, 511, 475, 546, 508, 499, 526, 492, 503, 514, 468, 540, 508, 501, 502, 495, 499, 524, 510, 513, 558, 480, 495, 514, 521, 503, 502, 509, 512, 501, 548, 485] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[521, 523, 493, 527, 509, 508, 515, 506, 496, 487, 490, 518, 491, 491, 534, 495, 499, 532, 517, 557, 509, 497, 509, 491, 484, 530, 548, 491, 486, 513, 500, 509, 519, 538, 504, 517, 540, 565, 515, 512, 530, 519, 500, 534, 541, 517, 520, 488, 507, 505, 508, 502, 517, 581, 506, 505, 525, 509, 505, 494, 521, 480, 464, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[508, 511, 514, 521, 494, 525, 507, 505, 536, 526, 529, 505, 477, 506, 506, 545, 528, 517, 475, 509, 541, 506, 512, 468, 478, 534, 466, 528, 513, 508, 484, 535, 496, 510, 516, 466, 540, 539, 517, 506, 491, 508, 514, 556, 497, 481, 521, 521, 546, 514, 536, 489, 508, 536, 518, 503, 514, 533, 497, 538, 504, 535, 518, 483] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[512, 550, 550, 509, 480, 547, 500, 543, 496, 494, 521, 539, 497, 505, 522, 516, 547, 478, 513, 510, 485, 506, 515, 488, 515, 532, 510, 493, 519, 546, 499, 531, 485, 511, 513, 522, 485, 531, 514, 503, 499, 499, 521, 539, 521, 498, 522, 525, 492, 495, 537, 515, 488, 507, 497, 544, 491, 508, 498, 513, 483, 517, 525, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[504, 516, 480, 549, 503, 465, 550, 498, 548, 500, 523, 525, 515, 531, 512, 489, 520, 503, 511, 533, 518, 528, 516, 544, 476, 499, 489, 520, 533, 532, 452, 504, 516, 523, 493, 520, 525, 524, 524, 535, 529, 508, 516, 523, 542, 548, 509, 494, 537, 539, 501, 489, 497, 499, 507, 523, 482, 464, 482, 493, 469, 511, 535, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[513, 487, 556, 509, 522, 496, 448, 497, 489, 509, 533, 471, 513, 562, 507, 526, 527, 513, 528, 495, 553, 517, 496, 497, 496, 537, 524, 522, 504, 523, 520, 528, 524, 514, 555, 521, 518, 538, 469, 505, 525, 550, 542, 507, 515, 535, 525, 518, 511, 511, 500, 524, 465, 504, 509, 509, 503, 474, 521, 477, 499, 500, 486, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[505, 500, 506, 543, 512, 534, 539, 481, 531, 506, 512, 483, 515, 510, 526, 475, 474, 534, 501, 537, 507, 537, 545, 527, 472, 505, 543, 498, 486, 517, 507, 515, 497, 536, 506, 510, 509, 493, 528, 503, 529, 504, 526, 514, 546, 506, 512, 532, 555, 514, 490, 519, 484, 504, 518, 503, 518, 518, 486, 487, 502, 499, 518, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[541, 526, 545, 542, 516, 459, 491, 483, 2, 1, 0, 0, 0, 0, 0, 0, 546, 514, 518, 548, 505, 497, 518, 483, 0, 0, 0, 0, 0, 0, 0, 0, 513, 501, 498, 517, 474, 514, 561, 564, 0, 0, 0, 0, 0, 0, 0, 0, 468, 512, 518, 529, 480, 525, 523, 490, 10, 8, 0, 0, 0, 0, 0, 0, 566, 511, 475, 546, 508, 463, 526, 492, 0, 0, 0, 0, 0, 0, 0, 0, 503, 514, 468, 512, 508, 501, 502, 495, 0, 0, 0, 0, 0, 0, 0, 0, 497, 524, 510, 513, 558, 480, 495, 514, 0, 0, 0, 0, 0, 0, 0, 0, 521, 503, 502, 509, 512, 501, 548, 485, 36, 28, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4129, 4142, 4063, 4087, 4003, 4091, 4147] avg=4096.0000 max=4147 max_rank=7 min=4003 min_rank=5 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16328 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[524, 486, 481, 523, 464, 506, 535, 523, 2, 1, 0, 0, 0, 0, 0, 0, 520, 537, 510, 519, 542, 476, 525, 541, 0, 0, 0, 0, 0, 0, 0, 0, 472, 471, 560, 494, 496, 519, 514, 510, 0, 0, 0, 0, 0, 0, 0, 0, 491, 526, 501, 526, 515, 516, 501, 539, 10, 9, 0, 0, 0, 0, 0, 0, 545, 491, 533, 519, 532, 478, 533, 531, 0, 0, 0, 0, 0, 0, 0, 0, 523, 494, 488, 475, 510, 530, 522, 476, 0, 0, 0, 0, 0, 0, 0, 0, 511, 511, 500, 470, 531, 511, 558, 533, 0, 0, 0, 0, 0, 0, 0, 0, 519, 503, 477, 516, 519, 484, 506, 489, 37, 26, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4045, 4170, 4036, 4134, 4162, 4018, 4125, 4078] avg=4096.0000 max=4170 max_rank=1 min=4018 min_rank=5 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[508, 511, 514, 521, 494, 525, 507, 505, 2, 1, 0, 0, 0, 0, 0, 0, 536, 526, 529, 505, 477, 506, 497, 545, 0, 0, 0, 0, 0, 0, 0, 0, 528, 517, 475, 497, 541, 506, 512, 468, 0, 0, 0, 0, 0, 0, 0, 0, 478, 534, 466, 528, 513, 508, 484, 535, 10, 8, 0, 0, 0, 0, 0, 0, 496, 510, 516, 466, 540, 501, 517, 506, 0, 0, 0, 0, 0, 0, 0, 0, 491, 508, 514, 527, 497, 481, 521, 521, 0, 0, 0, 0, 0, 0, 0, 0, 544, 514, 536, 489, 508, 536, 518, 503, 0, 0, 0, 0, 0, 0, 0, 0, 514, 533, 497, 538, 504, 535, 518, 483, 38, 29, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4121, 4044, 4064, 4052, 4060, 4148, 4191] avg=4096.0000 max=4191 max_rank=7 min=4044 min_rank=2 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16451 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[504, 516, 480, 549, 503, 465, 550, 498, 2, 1, 0, 0, 0, 0, 0, 0, 548, 500, 523, 525, 515, 531, 503, 489, 0, 0, 0, 0, 0, 0, 0, 0, 520, 503, 511, 521, 518, 528, 516, 544, 0, 0, 0, 0, 0, 0, 0, 0, 476, 499, 489, 520, 533, 532, 452, 504, 10, 8, 0, 0, 0, 0, 0, 0, 516, 523, 493, 520, 525, 487, 524, 535, 0, 0, 0, 0, 0, 0, 0, 0, 529, 508, 516, 496, 542, 548, 509, 494, 0, 0, 0, 0, 0, 0, 0, 0, 535, 539, 501, 489, 497, 499, 507, 523, 0, 0, 0, 0, 0, 0, 0, 0, 482, 464, 482, 493, 469, 511, 535, 525, 37, 27, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4134, 4161, 4023, 4123, 4142, 4090, 4027] avg=4096.0000 max=4161 max_rank=2 min=4023 min_rank=3 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[512, 550, 550, 509, 480, 547, 500, 543, 2, 1, 0, 0, 0, 0, 0, 0, 496, 494, 521, 539, 497, 505, 513, 516, 0, 0, 0, 0, 0, 0, 0, 0, 547, 478, 513, 498, 485, 506, 515, 488, 0, 0, 0, 0, 0, 0, 0, 0, 515, 532, 510, 493, 519, 546, 499, 531, 10, 8, 0, 0, 0, 0, 0, 0, 485, 511, 513, 522, 485, 493, 514, 503, 0, 0, 0, 0, 0, 0, 0, 0, 499, 499, 521, 511, 521, 498, 522, 525, 0, 0, 0, 0, 0, 0, 0, 0, 490, 495, 537, 515, 488, 507, 497, 544, 0, 0, 0, 0, 0, 0, 0, 0, 491, 508, 498, 513, 483, 517, 525, 502, 38, 28, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4194, 4081, 4030, 4163, 4026, 4096, 4073, 4105] avg=4096.0000 max=4194 max_rank=0 min=4026 min_rank=4 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16300 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[513, 487, 556, 509, 522, 496, 448, 497, 6, 5, 0, 0, 0, 0, 0, 0, 489, 509, 533, 471, 513, 562, 489, 526, 0, 0, 0, 0, 0, 0, 0, 0, 527, 513, 528, 476, 553, 517, 496, 497, 0, 0, 0, 0, 0, 0, 0, 0, 496, 537, 524, 522, 504, 523, 520, 528, 12, 13, 0, 0, 0, 0, 0, 0, 524, 514, 555, 521, 518, 495, 469, 505, 0, 0, 0, 0, 0, 0, 0, 0, 525, 550, 542, 476, 515, 535, 525, 518, 0, 0, 0, 0, 0, 0, 0, 0, 505, 511, 500, 524, 465, 504, 509, 509, 0, 0, 0, 0, 0, 0, 0, 0, 503, 474, 521, 477, 499, 500, 486, 496, 43, 31, 7, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4039, 4092, 4107, 4179, 4101, 4186, 4027, 4037] avg=4096.0000 max=4186 max_rank=5 min=4027 min_rank=6 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16351 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[505, 500, 506, 543, 512, 534, 539, 481, 2, 1, 0, 0, 0, 0, 0, 0, 531, 506, 512, 483, 515, 510, 517, 475, 0, 0, 0, 0, 0, 0, 0, 0, 474, 534, 501, 525, 507, 537, 545, 527, 0, 0, 0, 0, 0, 0, 0, 0, 472, 505, 543, 498, 486, 517, 507, 515, 10, 8, 0, 0, 0, 0, 0, 0, 497, 536, 506, 510, 509, 458, 528, 503, 0, 0, 0, 0, 0, 0, 0, 0, 529, 504, 526, 487, 546, 506, 512, 532, 0, 0, 0, 0, 0, 0, 0, 0, 553, 514, 490, 519, 484, 504, 518, 503, 0, 0, 0, 0, 0, 0, 0, 0, 518, 518, 486, 487, 502, 499, 518, 519, 35, 27, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4123, 4049, 4150, 4061, 4047, 4142, 4085, 4111] avg=4096.0000 max=4150 max_rank=2 min=4047 min_rank=4 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[521, 523, 493, 527, 509, 508, 515, 506, 2, 1, 0, 0, 0, 0, 0, 0, 496, 487, 490, 518, 491, 491, 524, 495, 0, 0, 0, 0, 0, 0, 0, 0, 499, 532, 517, 544, 509, 497, 509, 491, 0, 0, 0, 0, 0, 0, 0, 0, 484, 530, 548, 491, 486, 513, 500, 509, 11, 9, 0, 0, 0, 0, 0, 0, 519, 538, 504, 517, 540, 525, 515, 512, 0, 0, 0, 0, 0, 0, 0, 0, 530, 519, 500, 506, 541, 517, 520, 488, 0, 0, 0, 0, 0, 0, 0, 0, 505, 505, 508, 502, 517, 581, 506, 505, 0, 0, 0, 0, 0, 0, 0, 0, 525, 509, 505, 494, 521, 480, 464, 504, 40, 28, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 3992, 4098, 4081, 4170, 4121, 4129, 4072] avg=4096.0000 max=4170 max_rank=4 min=3992 min_rank=1 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16492 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 3968, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 3968, 4096, 4096, 4096, 4224, 4096, 384, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[3968, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[536, 514, 515, 510, 525, 519, 519, 534, 528, 514, 542, 515, 541, 516, 527, 492, 504, 491, 559, 485, 471, 527, 486, 485, 523, 518, 500, 525, 493, 507, 520, 546, 493, 543, 513, 467, 508, 535, 507, 502, 515, 470, 486, 557, 516, 532, 513, 534, 491, 518, 527, 485, 457, 514, 478, 506, 524, 520, 510, 532, 519, 494, 509, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[516, 499, 536, 514, 498, 535, 550, 522, 494, 474, 520, 496, 532, 490, 521, 507, 497, 528, 491, 498, 526, 502, 513, 499, 490, 537, 498, 493, 528, 490, 486, 518, 514, 536, 495, 520, 518, 517, 501, 508, 532, 516, 523, 538, 502, 484, 536, 504, 535, 480, 525, 479, 553, 537, 544, 480, 518, 515, 497, 515, 510, 543, 525, 470] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[531, 523, 561, 509, 504, 465, 503, 521, 530, 523, 481, 510, 524, 526, 506, 530, 501, 549, 498, 543, 506, 493, 494, 508, 520, 481, 497, 517, 471, 491, 505, 504, 534, 498, 492, 522, 498, 533, 510, 500, 517, 535, 500, 506, 479, 478, 568, 543, 481, 539, 482, 527, 501, 486, 525, 535, 518, 542, 506, 501, 529, 535, 494, 529] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[470, 519, 486, 516, 549, 493, 478, 461, 524, 511, 508, 552, 530, 489, 535, 538, 523, 529, 540, 544, 523, 507, 528, 518, 506, 509, 504, 491, 515, 509, 471, 506, 514, 520, 521, 508, 459, 519, 504, 550, 504, 505, 487, 538, 496, 526, 530, 506, 545, 527, 516, 531, 483, 491, 494, 507, 497, 517, 507, 518, 531, 519, 527, 489] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[510, 561, 481, 517, 525, 552, 507, 536, 492, 472, 559, 513, 499, 487, 492, 503, 542, 507, 511, 502, 501, 457, 489, 490, 528, 530, 480, 562, 501, 512, 531, 480, 490, 523, 528, 509, 516, 549, 507, 501, 529, 509, 521, 508, 510, 531, 482, 534, 506, 517, 548, 504, 500, 513, 514, 526, 502, 522, 485, 529, 515, 490, 481, 540] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[497, 477, 524, 500, 506, 500, 525, 498, 521, 525, 516, 493, 516, 473, 530, 496, 522, 482, 544, 539, 494, 480, 523, 519, 504, 526, 526, 489, 488, 533, 533, 517, 493, 519, 543, 537, 499, 499, 523, 515, 504, 471, 515, 536, 567, 509, 482, 520, 523, 488, 521, 456, 527, 517, 498, 502, 543, 536, 529, 498, 490, 537, 536, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[525, 505, 492, 475, 512, 534, 496, 540, 474, 504, 551, 520, 488, 506, 506, 521, 484, 473, 507, 538, 497, 459, 475, 492, 528, 526, 501, 497, 532, 545, 486, 518, 513, 518, 523, 544, 531, 545, 517, 481, 508, 500, 527, 524, 507, 482, 515, 542, 557, 498, 497, 550, 549, 513, 514, 499, 526, 531, 514, 507, 505, 487, 532, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[515, 488, 514, 545, 513, 521, 514, 520, 502, 483, 510, 501, 498, 516, 492, 496, 476, 523, 498, 501, 516, 490, 522, 503, 499, 531, 526, 520, 547, 471, 498, 554, 509, 513, 528, 497, 548, 507, 517, 482, 521, 508, 484, 527, 513, 504, 554, 530, 536, 511, 496, 534, 512, 495, 527, 490, 521, 486, 510, 481, 529, 510, 542, 543] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[536, 514, 515, 510, 525, 519, 519, 524, 0, 0, 0, 0, 0, 0, 0, 0, 528, 514, 542, 515, 541, 516, 527, 492, 9, 1, 0, 0, 0, 0, 0, 0, 504, 491, 559, 485, 471, 527, 486, 485, 23, 20, 0, 0, 0, 0, 0, 0, 523, 518, 500, 525, 493, 507, 520, 546, 5, 4, 0, 0, 0, 0, 0, 0, 493, 543, 513, 467, 508, 521, 507, 502, 0, 0, 0, 0, 0, 0, 0, 0, 515, 470, 486, 534, 516, 532, 513, 534, 0, 0, 0, 0, 0, 0, 0, 0, 491, 518, 527, 485, 457, 514, 478, 506, 6, 0, 0, 0, 0, 0, 0, 0, 524, 499, 510, 532, 519, 494, 509, 506, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4162, 4185, 4051, 4141, 4054, 4100, 3982, 4093] avg=4096.0000 max=4185 max_rank=1 min=3982 min_rank=6 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16229 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[516, 499, 536, 514, 498, 535, 550, 512, 0, 0, 0, 0, 0, 0, 0, 0, 494, 474, 520, 496, 532, 490, 521, 507, 8, 1, 0, 0, 0, 0, 0, 0, 497, 528, 491, 498, 526, 502, 513, 499, 23, 20, 0, 0, 0, 0, 0, 0, 490, 537, 498, 493, 528, 490, 486, 518, 5, 4, 0, 0, 0, 0, 0, 0, 514, 536, 495, 520, 518, 504, 501, 508, 0, 0, 0, 0, 0, 0, 0, 0, 532, 516, 523, 515, 502, 484, 536, 504, 0, 0, 0, 0, 0, 0, 0, 0, 535, 480, 525, 479, 553, 537, 544, 480, 6, 0, 0, 0, 0, 0, 0, 0, 518, 494, 497, 515, 510, 543, 525, 470, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4160, 4043, 4097, 4049, 4096, 4112, 4139, 4072] avg=4096.0000 max=4160 max_rank=0 min=4043 min_rank=1 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16419 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[470, 519, 486, 516, 549, 493, 478, 452, 0, 0, 0, 0, 0, 0, 0, 0, 524, 511, 508, 552, 530, 489, 535, 538, 8, 1, 0, 0, 0, 0, 0, 0, 523, 529, 540, 544, 523, 507, 528, 518, 23, 20, 0, 0, 0, 0, 0, 0, 506, 509, 504, 491, 515, 509, 471, 506, 5, 4, 0, 0, 0, 0, 0, 0, 514, 520, 521, 508, 459, 506, 504, 550, 0, 0, 0, 0, 0, 0, 0, 0, 504, 505, 487, 515, 496, 526, 530, 506, 0, 0, 0, 0, 0, 0, 0, 0, 545, 527, 516, 531, 483, 491, 494, 507, 5, 0, 0, 0, 0, 0, 0, 0, 497, 496, 507, 518, 531, 519, 527, 489, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3963, 4196, 4255, 4020, 4082, 4069, 4099, 4084] avg=4096.0000 max=4255 max_rank=2 min=3963 min_rank=0 imbalance_ratio=1.038818 max_load_reduction=28513 max_load_reduction_pct=0.870148 moved_to_echo=16334 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[510, 561, 481, 517, 525, 552, 507, 526, 0, 0, 0, 0, 0, 0, 0, 0, 492, 472, 559, 513, 499, 487, 492, 503, 9, 1, 0, 0, 0, 0, 0, 0, 542, 507, 511, 502, 501, 457, 489, 490, 21, 20, 0, 0, 0, 0, 0, 0, 528, 530, 480, 562, 501, 512, 531, 480, 5, 4, 0, 0, 0, 0, 0, 0, 490, 523, 528, 509, 516, 535, 507, 501, 0, 0, 0, 0, 0, 0, 0, 0, 529, 509, 521, 487, 510, 531, 482, 534, 0, 0, 0, 0, 0, 0, 0, 0, 506, 517, 548, 504, 500, 513, 514, 526, 6, 0, 0, 0, 0, 0, 0, 0, 502, 501, 485, 529, 515, 490, 481, 540, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4179, 4027, 4040, 4133, 4109, 4103, 4134, 4043] avg=4096.0000 max=4179 max_rank=0 min=4027 min_rank=1 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[525, 505, 492, 475, 512, 534, 496, 521, 0, 0, 0, 0, 0, 0, 0, 0, 474, 504, 551, 520, 488, 506, 506, 521, 13, 8, 0, 0, 0, 0, 0, 0, 484, 473, 507, 538, 497, 459, 475, 492, 26, 23, 0, 0, 0, 0, 0, 0, 528, 526, 501, 497, 532, 545, 486, 518, 8, 9, 0, 0, 0, 0, 0, 0, 513, 518, 523, 544, 531, 524, 517, 481, 0, 0, 0, 0, 0, 0, 0, 0, 508, 500, 527, 498, 507, 482, 515, 542, 0, 0, 0, 0, 0, 0, 0, 0, 557, 498, 497, 550, 549, 513, 514, 499, 10, 0, 0, 0, 0, 0, 0, 0, 526, 500, 514, 507, 505, 487, 532, 505, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4091, 3974, 4150, 4151, 4079, 4187, 4076] avg=4096.0000 max=4187 max_rank=6 min=3974 min_rank=2 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16493 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[531, 523, 561, 509, 504, 465, 503, 511, 0, 0, 0, 0, 0, 0, 0, 0, 530, 523, 481, 510, 524, 526, 506, 530, 9, 1, 0, 0, 0, 0, 0, 0, 501, 549, 498, 543, 506, 493, 494, 508, 21, 21, 0, 0, 0, 0, 0, 0, 520, 481, 497, 517, 471, 491, 505, 504, 5, 4, 0, 0, 0, 0, 0, 0, 534, 498, 492, 522, 498, 519, 510, 500, 0, 0, 0, 0, 0, 0, 0, 0, 517, 535, 500, 485, 479, 478, 568, 543, 0, 0, 0, 0, 0, 0, 0, 0, 481, 539, 482, 527, 501, 486, 525, 535, 6, 0, 0, 0, 0, 0, 0, 0, 518, 520, 506, 501, 529, 535, 494, 529, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4140, 4134, 3995, 4073, 4105, 4082, 4132] avg=4096.0000 max=4140 max_rank=1 min=3995 min_rank=3 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[497, 477, 524, 500, 506, 500, 525, 488, 0, 0, 0, 0, 0, 0, 0, 0, 521, 525, 516, 493, 516, 473, 530, 496, 8, 1, 0, 0, 0, 0, 0, 0, 522, 482, 544, 539, 494, 480, 523, 519, 23, 21, 0, 0, 0, 0, 0, 0, 504, 526, 526, 489, 488, 533, 533, 517, 5, 4, 0, 0, 0, 0, 0, 0, 493, 519, 543, 537, 499, 486, 523, 515, 0, 0, 0, 0, 0, 0, 0, 0, 504, 471, 515, 513, 567, 509, 482, 520, 0, 0, 0, 0, 0, 0, 0, 0, 523, 488, 521, 456, 527, 517, 498, 502, 6, 0, 0, 0, 0, 0, 0, 0, 543, 514, 529, 498, 490, 537, 536, 519, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4017, 4079, 4147, 4125, 4115, 4081, 4038, 4166] avg=4096.0000 max=4166 max_rank=7 min=4017 min_rank=0 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16400 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[515, 488, 514, 545, 513, 521, 514, 510, 0, 0, 0, 0, 0, 0, 0, 0, 502, 483, 510, 501, 498, 516, 492, 496, 8, 1, 0, 0, 0, 0, 0, 0, 476, 523, 498, 501, 516, 490, 522, 503, 22, 19, 0, 0, 0, 0, 0, 0, 499, 531, 526, 520, 547, 471, 498, 554, 5, 4, 0, 0, 0, 0, 0, 0, 509, 513, 528, 497, 548, 494, 517, 482, 0, 0, 0, 0, 0, 0, 0, 0, 521, 508, 484, 505, 513, 504, 554, 530, 0, 0, 0, 0, 0, 0, 0, 0, 536, 511, 496, 534, 512, 495, 527, 490, 6, 0, 0, 0, 0, 0, 0, 0, 521, 466, 510, 481, 529, 510, 542, 543, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4007, 4070, 4155, 4088, 4119, 4107, 4102] avg=4096.0000 max=4155 max_rank=3 min=4007 min_rank=1 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16416 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 3968, 4096, 4096, 256, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[506, 487, 513, 514, 538, 524, 487, 505, 484, 514, 508, 553, 505, 517, 537, 554, 522, 488, 495, 517, 528, 508, 512, 544, 491, 522, 478, 503, 495, 492, 525, 511, 478, 515, 500, 535, 512, 510, 540, 493, 479, 483, 504, 500, 497, 525, 518, 511, 479, 510, 475, 521, 511, 552, 533, 531, 547, 539, 534, 503, 464, 489, 514, 589] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[496, 513, 480, 552, 567, 530, 533, 574, 519, 473, 499, 478, 499, 552, 521, 507, 507, 511, 525, 495, 517, 487, 476, 515, 500, 496, 496, 500, 550, 510, 490, 483, 512, 460, 531, 519, 513, 502, 495, 507, 499, 528, 496, 486, 507, 532, 523, 512, 520, 523, 559, 512, 498, 536, 480, 518, 506, 532, 549, 503, 522, 526, 515, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[536, 515, 528, 505, 531, 506, 508, 522, 514, 496, 492, 524, 505, 540, 500, 538, 517, 491, 489, 494, 503, 498, 480, 490, 520, 524, 519, 508, 478, 521, 513, 505, 509, 513, 522, 493, 515, 511, 493, 513, 510, 509, 544, 507, 502, 499, 499, 505, 562, 489, 557, 527, 491, 518, 534, 511, 544, 527, 511, 512, 508, 499, 505, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[502, 528, 490, 540, 507, 492, 552, 510, 519, 486, 500, 526, 521, 506, 523, 482, 508, 482, 489, 535, 491, 458, 550, 562, 523, 548, 491, 537, 499, 503, 497, 497, 514, 527, 493, 529, 507, 492, 525, 528, 520, 523, 527, 491, 556, 517, 501, 490, 504, 496, 523, 501, 524, 517, 475, 506, 495, 520, 537, 535, 536, 497, 500, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[476, 495, 510, 522, 551, 510, 553, 511, 511, 509, 523, 492, 498, 529, 535, 520, 522, 521, 548, 553, 513, 513, 496, 483, 526, 534, 536, 509, 474, 498, 503, 521, 491, 520, 532, 471, 510, 489, 501, 494, 537, 513, 500, 499, 518, 496, 520, 501, 537, 501, 516, 496, 516, 458, 481, 553, 514, 492, 529, 511, 525, 543, 490, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[504, 533, 501, 503, 508, 495, 521, 490, 496, 536, 531, 497, 483, 491, 521, 529, 501, 500, 504, 521, 520, 530, 538, 539, 514, 510, 518, 517, 518, 516, 522, 483, 503, 512, 512, 501, 529, 535, 487, 506, 529, 551, 533, 490, 534, 487, 496, 491, 491, 518, 527, 508, 545, 534, 517, 493, 509, 494, 499, 528, 511, 547, 508, 453] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[448, 525, 484, 506, 570, 501, 522, 514, 505, 516, 522, 484, 517, 509, 521, 488, 562, 517, 497, 506, 530, 550, 520, 542, 508, 503, 538, 534, 508, 555, 544, 519, 508, 498, 502, 493, 512, 482, 502, 508, 526, 455, 505, 487, 523, 537, 491, 535, 499, 486, 505, 520, 494, 508, 523, 494, 520, 485, 524, 517, 522, 541, 511, 490] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[540, 507, 500, 551, 535, 519, 517, 513, 535, 526, 520, 499, 525, 499, 504, 512, 492, 538, 508, 442, 506, 501, 496, 521, 495, 538, 543, 527, 516, 531, 516, 513, 523, 514, 480, 494, 459, 531, 564, 489, 520, 523, 502, 501, 512, 502, 503, 521, 511, 497, 489, 526, 497, 463, 509, 526, 523, 510, 545, 473, 557, 515, 527, 477] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[506, 487, 513, 514, 506, 524, 487, 505, 0, 0, 0, 0, 0, 0, 0, 0, 484, 514, 508, 553, 505, 517, 537, 554, 0, 0, 0, 0, 0, 0, 0, 0, 522, 488, 495, 517, 528, 508, 512, 544, 5, 1, 0, 0, 0, 0, 0, 0, 491, 518, 478, 503, 495, 492, 525, 511, 0, 0, 0, 0, 0, 0, 0, 0, 478, 515, 500, 535, 512, 510, 540, 493, 32, 6, 0, 0, 0, 0, 0, 0, 479, 483, 504, 500, 497, 525, 518, 511, 16, 0, 0, 0, 0, 0, 0, 0, 479, 510, 475, 521, 511, 552, 533, 531, 3, 0, 0, 0, 0, 0, 0, 0, 547, 539, 507, 503, 464, 489, 514, 589, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4042, 4172, 4120, 4013, 4121, 4033, 4115, 4152] avg=4096.0000 max=4172 max_rank=1 min=4013 min_rank=3 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16421 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[496, 513, 480, 552, 534, 530, 533, 574, 0, 0, 0, 0, 0, 0, 0, 0, 519, 473, 499, 478, 499, 552, 521, 507, 0, 0, 0, 0, 0, 0, 0, 0, 507, 511, 525, 495, 517, 487, 476, 515, 5, 1, 0, 0, 0, 0, 0, 0, 500, 492, 496, 500, 550, 510, 490, 483, 0, 0, 0, 0, 0, 0, 0, 0, 512, 460, 531, 519, 513, 502, 495, 507, 33, 6, 0, 0, 0, 0, 0, 0, 499, 528, 496, 486, 507, 532, 523, 512, 16, 0, 0, 0, 0, 0, 0, 0, 520, 523, 559, 512, 498, 536, 480, 518, 3, 0, 0, 0, 0, 0, 0, 0, 506, 532, 522, 503, 522, 526, 515, 496, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4212, 4048, 4039, 4021, 4078, 4099, 4149, 4122] avg=4096.0000 max=4212 max_rank=0 min=4021 min_rank=3 imbalance_ratio=1.028320 max_load_reduction=28556 max_load_reduction_pct=0.871460 moved_to_echo=16448 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[476, 495, 510, 522, 518, 510, 553, 511, 0, 0, 0, 0, 0, 0, 0, 0, 511, 509, 523, 492, 498, 529, 535, 520, 0, 0, 0, 0, 0, 0, 0, 0, 522, 521, 548, 553, 513, 513, 496, 483, 5, 1, 0, 0, 0, 0, 0, 0, 526, 530, 536, 509, 474, 498, 503, 521, 0, 0, 0, 0, 0, 0, 0, 0, 491, 520, 532, 471, 510, 489, 501, 494, 33, 6, 0, 0, 0, 0, 0, 0, 537, 513, 500, 499, 518, 496, 520, 501, 16, 0, 0, 0, 0, 0, 0, 0, 537, 501, 516, 496, 516, 458, 481, 553, 3, 0, 0, 0, 0, 0, 0, 0, 514, 492, 502, 511, 525, 543, 490, 519, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4117, 4155, 4097, 4047, 4100, 4061, 4096] avg=4096.0000 max=4155 max_rank=2 min=4047 min_rank=4 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16304 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[536, 515, 528, 505, 500, 506, 508, 522, 0, 0, 0, 0, 0, 0, 0, 0, 514, 496, 492, 524, 505, 540, 500, 538, 0, 0, 0, 0, 0, 0, 0, 0, 517, 491, 489, 494, 503, 498, 480, 490, 4, 1, 0, 0, 0, 0, 0, 0, 520, 520, 519, 508, 478, 521, 513, 505, 0, 0, 0, 0, 0, 0, 0, 0, 509, 513, 522, 493, 515, 511, 493, 513, 31, 6, 0, 0, 0, 0, 0, 0, 510, 509, 544, 507, 502, 499, 499, 505, 15, 0, 0, 0, 0, 0, 0, 0, 562, 489, 557, 527, 491, 518, 534, 511, 3, 0, 0, 0, 0, 0, 0, 0, 544, 527, 486, 512, 508, 499, 505, 519, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4109, 3967, 4084, 4106, 4090, 4192, 4100] avg=4096.0000 max=4192 max_rank=6 min=3967 min_rank=2 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16488 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[502, 528, 490, 540, 477, 492, 552, 510, 0, 0, 0, 0, 0, 0, 0, 0, 519, 486, 500, 526, 521, 506, 523, 482, 0, 0, 0, 0, 0, 0, 0, 0, 508, 482, 489, 535, 491, 458, 550, 562, 5, 1, 0, 0, 0, 0, 0, 0, 523, 544, 491, 537, 499, 503, 497, 497, 0, 0, 0, 0, 0, 0, 0, 0, 514, 527, 493, 529, 507, 492, 525, 528, 30, 6, 0, 0, 0, 0, 0, 0, 520, 523, 527, 491, 556, 517, 501, 490, 16, 0, 0, 0, 0, 0, 0, 0, 504, 496, 523, 501, 524, 517, 475, 506, 3, 0, 0, 0, 0, 0, 0, 0, 495, 520, 510, 535, 536, 497, 500, 508, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4063, 4081, 4091, 4151, 4141, 4049, 4101] avg=4096.0000 max=4151 max_rank=4 min=4049 min_rank=6 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16442 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[448, 525, 484, 506, 533, 501, 522, 514, 0, 0, 0, 0, 0, 0, 0, 0, 505, 516, 522, 484, 517, 509, 514, 488, 0, 0, 0, 0, 0, 0, 0, 0, 562, 517, 497, 506, 530, 550, 520, 542, 7, 7, 0, 0, 0, 0, 0, 0, 508, 487, 538, 534, 508, 555, 544, 519, 0, 0, 0, 0, 0, 0, 0, 0, 508, 498, 502, 493, 512, 482, 502, 508, 37, 10, 0, 0, 0, 0, 0, 0, 526, 455, 505, 487, 523, 537, 491, 535, 20, 0, 0, 0, 0, 0, 0, 0, 499, 486, 505, 520, 494, 508, 523, 494, 9, 7, 0, 0, 0, 0, 0, 0, 520, 485, 487, 517, 522, 541, 511, 490, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4033, 4055, 4238, 4193, 4052, 4079, 4045, 4073] avg=4096.0000 max=4238 max_rank=2 min=4033 min_rank=0 imbalance_ratio=1.034668 max_load_reduction=28530 max_load_reduction_pct=0.870667 moved_to_echo=16249 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[504, 533, 501, 503, 478, 495, 521, 490, 0, 0, 0, 0, 0, 0, 0, 0, 496, 536, 531, 497, 483, 491, 521, 529, 0, 0, 0, 0, 0, 0, 0, 0, 501, 500, 504, 521, 520, 530, 538, 539, 4, 1, 0, 0, 0, 0, 0, 0, 514, 506, 518, 517, 518, 516, 522, 483, 0, 0, 0, 0, 0, 0, 0, 0, 503, 512, 512, 501, 529, 535, 487, 506, 30, 6, 0, 0, 0, 0, 0, 0, 529, 551, 533, 490, 534, 487, 496, 491, 15, 0, 0, 0, 0, 0, 0, 0, 491, 518, 527, 508, 545, 534, 517, 493, 3, 0, 0, 0, 0, 0, 0, 0, 509, 494, 474, 528, 511, 547, 508, 453, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4025, 4084, 4158, 4094, 4121, 4126, 4136, 4024] avg=4096.0000 max=4158 max_rank=2 min=4024 min_rank=7 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16407 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[540, 507, 500, 551, 503, 519, 517, 513, 0, 0, 0, 0, 0, 0, 0, 0, 535, 526, 520, 499, 525, 499, 504, 512, 0, 0, 0, 0, 0, 0, 0, 0, 492, 538, 508, 442, 506, 501, 496, 521, 5, 1, 0, 0, 0, 0, 0, 0, 495, 534, 543, 527, 516, 531, 516, 513, 0, 0, 0, 0, 0, 0, 0, 0, 523, 514, 480, 494, 459, 531, 564, 489, 32, 6, 0, 0, 0, 0, 0, 0, 520, 523, 502, 501, 512, 502, 503, 521, 16, 0, 0, 0, 0, 0, 0, 0, 511, 497, 489, 526, 497, 463, 509, 526, 3, 0, 0, 0, 0, 0, 0, 0, 523, 510, 518, 473, 557, 515, 527, 477, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4150, 4120, 4010, 4175, 4092, 4100, 4021, 4100] avg=4096.0000 max=4175 max_rank=3 min=4010 min_rank=2 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16313 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 3968, 4224, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[520, 512, 517, 534, 490, 518, 520, 526, 522, 485, 493, 513, 523, 485, 507, 512, 503, 517, 484, 500, 522, 532, 517, 531, 507, 518, 518, 522, 486, 505, 505, 535, 559, 490, 536, 515, 470, 518, 533, 486, 512, 502, 469, 465, 499, 518, 485, 512, 555, 531, 522, 482, 546, 515, 526, 494, 468, 503, 520, 528, 553, 550, 500, 527] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[485, 494, 501, 509, 565, 512, 518, 529, 526, 499, 520, 518, 514, 518, 508, 567, 497, 514, 542, 510, 490, 596, 508, 534, 484, 545, 479, 497, 547, 518, 483, 478, 530, 501, 507, 525, 494, 531, 475, 486, 550, 516, 516, 520, 491, 515, 485, 483, 491, 499, 522, 501, 538, 511, 529, 500, 517, 497, 526, 501, 500, 509, 481, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[524, 488, 479, 549, 525, 494, 502, 492, 484, 508, 515, 494, 471, 533, 514, 520, 505, 529, 519, 457, 544, 495, 509, 529, 515, 508, 530, 502, 516, 498, 497, 492, 543, 466, 559, 491, 522, 508, 521, 567, 508, 463, 525, 530, 556, 518, 533, 506, 508, 494, 578, 503, 478, 500, 552, 503, 476, 514, 528, 560, 494, 519, 521, 487] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[519, 511, 500, 521, 514, 463, 516, 531, 523, 531, 544, 494, 527, 489, 517, 531, 461, 529, 519, 524, 505, 498, 544, 523, 542, 531, 528, 520, 486, 489, 497, 503, 507, 494, 548, 495, 462, 501, 527, 520, 513, 539, 482, 506, 506, 536, 508, 489, 522, 536, 506, 531, 524, 495, 484, 512, 535, 517, 506, 465, 546, 493, 526, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[492, 530, 512, 528, 494, 512, 560, 505, 494, 514, 531, 518, 527, 522, 531, 495, 473, 539, 479, 539, 518, 506, 517, 530, 524, 526, 497, 488, 502, 530, 490, 549, 539, 517, 509, 510, 522, 485, 498, 515, 483, 539, 517, 513, 507, 500, 493, 485, 542, 533, 524, 501, 493, 501, 512, 533, 510, 499, 497, 480, 515, 524, 511, 489] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[538, 517, 525, 522, 517, 503, 514, 518, 505, 504, 526, 494, 517, 578, 519, 522, 502, 523, 512, 515, 482, 501, 473, 510, 529, 490, 512, 481, 524, 505, 542, 504, 532, 532, 488, 488, 517, 504, 502, 549, 512, 457, 472, 503, 506, 494, 529, 518, 544, 537, 519, 510, 516, 515, 535, 521, 512, 496, 508, 525, 474, 494, 529, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[522, 518, 485, 543, 493, 530, 506, 467, 525, 486, 510, 512, 494, 500, 531, 521, 515, 561, 528, 518, 498, 523, 511, 544, 503, 524, 506, 508, 498, 491, 485, 529, 464, 529, 500, 480, 526, 510, 528, 510, 502, 521, 504, 515, 498, 498, 541, 495, 530, 507, 520, 546, 516, 478, 521, 490, 519, 523, 515, 540, 500, 523, 523, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[513, 526, 496, 526, 506, 516, 532, 499, 519, 518, 488, 548, 487, 540, 502, 483, 491, 488, 486, 529, 510, 490, 521, 481, 494, 511, 524, 522, 515, 493, 500, 543, 521, 523, 518, 519, 506, 477, 495, 531, 508, 510, 502, 547, 510, 524, 533, 501, 490, 530, 523, 512, 523, 521, 512, 526, 508, 504, 514, 536, 507, 540, 512, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[520, 512, 517, 526, 490, 518, 520, 526, 0, 0, 0, 0, 0, 0, 0, 0, 522, 485, 493, 513, 523, 474, 507, 512, 0, 0, 0, 0, 0, 0, 0, 0, 503, 514, 484, 500, 522, 532, 517, 531, 0, 0, 0, 0, 0, 0, 0, 0, 507, 518, 518, 522, 486, 505, 505, 535, 11, 3, 3, 0, 0, 0, 0, 0, 559, 490, 536, 515, 470, 518, 533, 486, 3, 0, 0, 0, 0, 0, 0, 0, 512, 502, 469, 465, 499, 518, 485, 512, 34, 0, 0, 0, 0, 0, 0, 0, 555, 531, 485, 482, 546, 515, 526, 494, 0, 0, 0, 0, 0, 0, 0, 0, 468, 503, 520, 528, 553, 550, 500, 527, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4129, 4029, 4103, 4113, 4110, 3996, 4134, 4154] avg=4096.0000 max=4154 max_rank=7 min=3996 min_rank=5 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16394 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[485, 494, 501, 502, 565, 512, 518, 529, 0, 0, 0, 0, 0, 0, 0, 0, 526, 499, 520, 518, 514, 506, 508, 567, 0, 0, 0, 0, 0, 0, 0, 0, 497, 511, 542, 510, 490, 596, 508, 534, 0, 0, 0, 0, 0, 0, 0, 0, 484, 545, 479, 497, 547, 518, 483, 478, 12, 3, 2, 0, 0, 0, 0, 0, 530, 501, 507, 525, 494, 531, 475, 486, 3, 0, 0, 0, 0, 0, 0, 0, 550, 516, 516, 520, 491, 515, 485, 483, 34, 0, 0, 0, 0, 0, 0, 0, 491, 499, 485, 501, 538, 511, 529, 500, 0, 0, 0, 0, 0, 0, 0, 0, 517, 497, 526, 501, 500, 509, 481, 516, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4158, 4188, 4048, 4052, 4110, 4054, 4052] avg=4096.0000 max=4188 max_rank=2 min=4048 min_rank=3 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16268 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[524, 488, 479, 541, 525, 494, 502, 492, 0, 0, 0, 0, 0, 0, 0, 0, 484, 508, 515, 494, 471, 521, 514, 520, 0, 0, 0, 0, 0, 0, 0, 0, 505, 525, 519, 457, 544, 495, 509, 529, 0, 0, 0, 0, 0, 0, 0, 0, 515, 508, 530, 502, 516, 498, 497, 492, 12, 3, 3, 0, 0, 0, 0, 0, 543, 466, 559, 491, 522, 508, 521, 567, 4, 0, 0, 0, 0, 0, 0, 0, 508, 463, 525, 530, 556, 518, 533, 506, 37, 0, 0, 0, 0, 0, 0, 0, 508, 494, 538, 503, 478, 500, 552, 503, 0, 0, 0, 0, 0, 0, 0, 0, 476, 514, 528, 560, 494, 519, 521, 487, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4045, 4027, 4083, 4076, 4181, 4176, 4076, 4104] avg=4096.0000 max=4181 max_rank=4 min=4027 min_rank=1 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16537 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[519, 511, 500, 514, 514, 463, 516, 531, 0, 0, 0, 0, 0, 0, 0, 0, 523, 531, 544, 494, 527, 478, 517, 531, 0, 0, 0, 0, 0, 0, 0, 0, 461, 525, 519, 524, 505, 498, 544, 523, 0, 0, 0, 0, 0, 0, 0, 0, 542, 531, 528, 520, 486, 489, 497, 503, 11, 3, 2, 0, 0, 0, 0, 0, 507, 494, 548, 495, 462, 501, 527, 520, 4, 0, 0, 0, 0, 0, 0, 0, 513, 539, 482, 506, 506, 536, 508, 489, 33, 0, 0, 0, 0, 0, 0, 0, 522, 536, 470, 531, 524, 495, 484, 512, 0, 0, 0, 0, 0, 0, 0, 0, 535, 517, 506, 465, 546, 493, 526, 507, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4145, 4099, 4112, 4058, 4112, 4074, 4100] avg=4096.0000 max=4145 max_rank=1 min=4058 min_rank=4 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16344 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[492, 530, 512, 505, 494, 512, 560, 505, 0, 0, 0, 0, 0, 0, 0, 0, 494, 514, 531, 518, 527, 506, 531, 495, 0, 0, 0, 0, 0, 0, 0, 0, 473, 531, 479, 539, 518, 506, 517, 530, 0, 0, 0, 0, 0, 0, 0, 0, 524, 526, 497, 488, 502, 530, 490, 549, 16, 5, 7, 0, 0, 0, 0, 0, 539, 517, 509, 510, 522, 485, 498, 515, 8, 5, 0, 0, 0, 0, 0, 0, 483, 539, 517, 513, 507, 500, 493, 485, 37, 0, 0, 0, 0, 0, 0, 0, 542, 533, 482, 501, 493, 501, 512, 533, 0, 0, 0, 0, 0, 0, 0, 0, 510, 499, 497, 480, 515, 524, 511, 489, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4116, 4093, 4134, 4108, 4074, 4097, 4036] avg=4096.0000 max=4134 max_rank=3 min=4036 min_rank=7 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16315 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[522, 518, 485, 535, 493, 530, 506, 467, 0, 0, 0, 0, 0, 0, 0, 0, 525, 486, 510, 512, 494, 489, 531, 521, 0, 0, 0, 0, 0, 0, 0, 0, 515, 557, 528, 518, 498, 523, 511, 544, 0, 0, 0, 0, 0, 0, 0, 0, 503, 524, 506, 508, 498, 491, 485, 529, 11, 3, 3, 0, 0, 0, 0, 0, 464, 529, 500, 480, 526, 510, 528, 510, 4, 0, 0, 0, 0, 0, 0, 0, 502, 521, 504, 515, 498, 498, 541, 495, 33, 0, 0, 0, 0, 0, 0, 0, 530, 507, 484, 546, 516, 478, 521, 490, 0, 0, 0, 0, 0, 0, 0, 0, 519, 523, 515, 540, 500, 523, 523, 511, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4056, 4068, 4194, 4061, 4051, 4107, 4072, 4159] avg=4096.0000 max=4194 max_rank=2 min=4051 min_rank=4 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[538, 517, 525, 515, 517, 503, 514, 518, 0, 0, 0, 0, 0, 0, 0, 0, 505, 504, 526, 494, 517, 565, 519, 522, 0, 0, 0, 0, 0, 0, 0, 0, 502, 520, 512, 515, 482, 501, 473, 510, 0, 0, 0, 0, 0, 0, 0, 0, 529, 490, 512, 481, 524, 505, 542, 504, 13, 3, 2, 0, 0, 0, 0, 0, 532, 532, 488, 488, 517, 504, 502, 549, 3, 0, 0, 0, 0, 0, 0, 0, 512, 457, 472, 503, 506, 494, 529, 518, 33, 0, 0, 0, 0, 0, 0, 0, 544, 537, 483, 510, 516, 515, 535, 521, 0, 0, 0, 0, 0, 0, 0, 0, 512, 496, 508, 525, 474, 494, 529, 506, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4147, 4152, 4015, 4105, 4115, 4024, 4161, 4049] avg=4096.0000 max=4161 max_rank=6 min=4015 min_rank=2 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16349 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[513, 526, 496, 519, 506, 516, 532, 499, 0, 0, 0, 0, 0, 0, 0, 0, 519, 518, 488, 548, 487, 528, 502, 483, 0, 0, 0, 0, 0, 0, 0, 0, 491, 485, 486, 529, 510, 490, 521, 481, 0, 0, 0, 0, 0, 0, 0, 0, 494, 511, 524, 522, 515, 493, 500, 543, 12, 3, 2, 0, 0, 0, 0, 0, 521, 523, 518, 519, 506, 477, 495, 531, 3, 0, 0, 0, 0, 0, 0, 0, 508, 510, 502, 547, 510, 524, 533, 501, 34, 0, 0, 0, 0, 0, 0, 0, 490, 530, 486, 512, 523, 521, 512, 526, 0, 0, 0, 0, 0, 0, 0, 0, 508, 504, 514, 536, 507, 540, 512, 488, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4073, 3993, 4119, 4093, 4169, 4100, 4114] avg=4096.0000 max=4169 max_rank=5 min=3993 min_rank=2 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16476 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 3968, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3968, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[511, 516, 523, 475, 499, 511, 509, 514, 523, 540, 486, 512, 514, 508, 531, 503, 512, 500, 505, 482, 525, 518, 553, 514, 498, 508, 506, 523, 504, 494, 525, 533, 504, 532, 493, 540, 497, 495, 525, 521, 513, 466, 529, 545, 498, 509, 507, 475, 510, 486, 487, 541, 509, 515, 534, 524, 497, 528, 500, 497, 524, 556, 526, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[505, 503, 494, 544, 535, 461, 546, 537, 505, 474, 505, 497, 495, 528, 519, 518, 533, 507, 485, 512, 532, 504, 534, 495, 550, 496, 507, 506, 475, 479, 522, 542, 505, 517, 505, 514, 508, 484, 543, 528, 526, 537, 482, 511, 495, 546, 529, 524, 544, 500, 532, 514, 522, 479, 506, 510, 491, 482, 521, 528, 495, 514, 543, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[499, 552, 465, 500, 509, 487, 535, 522, 515, 548, 503, 539, 532, 505, 520, 488, 496, 494, 531, 512, 516, 512, 519, 491, 548, 513, 497, 502, 512, 491, 496, 502, 552, 500, 485, 484, 523, 515, 494, 512, 529, 504, 526, 504, 512, 515, 510, 514, 518, 509, 539, 499, 515, 509, 523, 537, 497, 499, 516, 534, 527, 491, 512, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[501, 509, 505, 519, 523, 487, 482, 492, 514, 514, 479, 514, 522, 513, 534, 524, 475, 514, 510, 554, 522, 555, 484, 517, 508, 478, 530, 516, 490, 495, 510, 556, 497, 540, 508, 515, 505, 510, 530, 519, 503, 518, 498, 491, 536, 489, 534, 511, 490, 540, 491, 507, 511, 566, 514, 526, 512, 490, 510, 510, 470, 524, 538, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[511, 519, 478, 487, 530, 516, 538, 539, 505, 508, 540, 521, 513, 488, 511, 535, 522, 520, 514, 495, 545, 498, 542, 481, 541, 539, 496, 497, 466, 549, 518, 499, 528, 541, 505, 516, 524, 520, 510, 519, 493, 548, 504, 508, 495, 485, 502, 499, 479, 502, 487, 502, 480, 472, 502, 560, 536, 528, 531, 498, 517, 499, 519, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[517, 516, 505, 488, 516, 518, 503, 485, 512, 495, 542, 532, 477, 515, 486, 511, 522, 517, 509, 519, 502, 507, 486, 512, 508, 514, 485, 518, 535, 537, 525, 524, 524, 473, 570, 483, 515, 508, 525, 501, 502, 525, 488, 510, 513, 522, 516, 532, 483, 535, 531, 531, 491, 511, 531, 512, 517, 515, 515, 532, 527, 504, 480, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[501, 498, 479, 514, 510, 527, 469, 536, 520, 502, 529, 498, 530, 551, 520, 519, 494, 497, 510, 557, 503, 517, 496, 535, 488, 534, 499, 513, 534, 503, 518, 505, 500, 542, 483, 533, 496, 486, 495, 472, 529, 539, 515, 511, 492, 506, 514, 500, 535, 501, 528, 549, 539, 472, 499, 521, 493, 522, 515, 502, 477, 543, 548, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[506, 488, 529, 540, 502, 481, 517, 515, 515, 514, 509, 507, 526, 522, 490, 454, 509, 487, 536, 542, 539, 499, 534, 484, 481, 509, 505, 507, 535, 508, 521, 541, 501, 507, 501, 501, 540, 505, 560, 503, 547, 526, 524, 523, 511, 502, 501, 483, 494, 504, 531, 516, 545, 504, 514, 531, 518, 510, 493, 496, 531, 507, 480, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[505, 503, 494, 544, 535, 461, 546, 537, 15, 9, 1, 0, 0, 0, 0, 0, 505, 474, 505, 497, 495, 523, 519, 518, 0, 0, 0, 0, 0, 0, 0, 0, 533, 507, 485, 512, 523, 504, 534, 495, 0, 0, 0, 0, 0, 0, 0, 0, 550, 496, 507, 506, 475, 479, 522, 542, 0, 0, 0, 0, 0, 0, 0, 0, 505, 517, 505, 514, 508, 484, 542, 528, 0, 0, 0, 0, 0, 0, 0, 0, 526, 537, 482, 511, 495, 546, 529, 524, 1, 0, 0, 0, 0, 0, 0, 0, 544, 500, 532, 514, 522, 479, 506, 495, 0, 0, 0, 0, 0, 0, 0, 0, 491, 482, 521, 528, 495, 514, 543, 488, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4150, 4036, 4093, 4077, 4103, 4151, 4092, 4066] avg=4096.0000 max=4151 max_rank=5 min=4036 min_rank=1 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16412 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[511, 516, 523, 475, 499, 511, 509, 514, 16, 9, 1, 0, 0, 0, 0, 0, 523, 540, 486, 512, 514, 503, 531, 503, 0, 0, 0, 0, 0, 0, 0, 0, 512, 500, 505, 482, 516, 518, 553, 514, 0, 0, 0, 0, 0, 0, 0, 0, 498, 508, 506, 523, 504, 494, 525, 533, 0, 0, 0, 0, 0, 0, 0, 0, 504, 532, 493, 540, 497, 495, 524, 521, 0, 0, 0, 0, 0, 0, 0, 0, 513, 466, 529, 545, 498, 509, 507, 475, 1, 0, 0, 0, 0, 0, 0, 0, 510, 486, 487, 541, 509, 515, 534, 508, 0, 0, 0, 0, 0, 0, 0, 0, 497, 528, 500, 497, 524, 556, 526, 510, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4112, 4100, 4091, 4106, 4043, 4090, 4142] avg=4096.0000 max=4142 max_rank=7 min=4043 min_rank=5 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[511, 519, 478, 487, 530, 516, 538, 539, 17, 9, 1, 0, 0, 0, 0, 0, 505, 508, 540, 521, 513, 483, 511, 535, 0, 0, 0, 0, 0, 0, 0, 0, 522, 520, 514, 495, 536, 498, 542, 481, 0, 0, 0, 0, 0, 0, 0, 0, 541, 539, 496, 497, 466, 549, 518, 499, 0, 0, 0, 0, 0, 0, 0, 0, 528, 541, 505, 516, 524, 520, 509, 519, 0, 0, 0, 0, 0, 0, 0, 0, 493, 548, 504, 508, 495, 485, 502, 499, 1, 0, 0, 0, 0, 0, 0, 0, 479, 502, 487, 502, 480, 472, 502, 543, 0, 0, 0, 0, 0, 0, 0, 0, 536, 528, 531, 498, 517, 499, 519, 498, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4145, 4116, 4108, 4105, 4162, 4035, 3967, 4130] avg=4096.0000 max=4162 max_rank=4 min=3967 min_rank=6 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16294 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[499, 552, 465, 500, 509, 487, 535, 522, 16, 9, 1, 0, 0, 0, 0, 0, 515, 548, 503, 539, 532, 500, 520, 488, 0, 0, 0, 0, 0, 0, 0, 0, 496, 494, 531, 512, 507, 512, 519, 491, 0, 0, 0, 0, 0, 0, 0, 0, 548, 513, 497, 502, 512, 491, 496, 502, 0, 0, 0, 0, 0, 0, 0, 0, 552, 500, 485, 484, 523, 515, 493, 512, 0, 0, 0, 0, 0, 0, 0, 0, 529, 504, 526, 504, 512, 515, 510, 514, 1, 0, 0, 0, 0, 0, 0, 0, 518, 509, 539, 499, 515, 509, 523, 521, 0, 0, 0, 0, 0, 0, 0, 0, 497, 499, 516, 534, 527, 491, 512, 513, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4145, 4062, 4061, 4064, 4115, 4133, 4093] avg=4096.0000 max=4145 max_rank=1 min=4061 min_rank=3 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16405 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[501, 509, 505, 519, 523, 487, 482, 492, 20, 13, 7, 0, 0, 0, 0, 0, 514, 514, 479, 514, 522, 497, 534, 524, 0, 0, 0, 0, 0, 0, 0, 0, 475, 514, 510, 554, 509, 555, 484, 517, 0, 0, 0, 0, 0, 0, 0, 0, 508, 478, 530, 516, 490, 495, 510, 556, 4, 0, 0, 0, 0, 0, 0, 0, 497, 540, 508, 515, 505, 510, 519, 519, 0, 0, 0, 0, 0, 0, 0, 0, 503, 518, 498, 491, 536, 489, 534, 511, 7, 2, 0, 0, 0, 0, 0, 0, 490, 540, 491, 507, 511, 566, 514, 506, 0, 0, 0, 0, 0, 0, 0, 0, 512, 490, 510, 510, 470, 524, 538, 519, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4058, 4098, 4118, 4087, 4113, 4089, 4125, 4080] avg=4096.0000 max=4125 max_rank=6 min=4058 min_rank=0 imbalance_ratio=1.007080 max_load_reduction=28643 max_load_reduction_pct=0.874115 moved_to_echo=16407 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[517, 516, 505, 488, 516, 518, 503, 485, 15, 8, 1, 0, 0, 0, 0, 0, 512, 495, 542, 532, 477, 510, 486, 511, 0, 0, 0, 0, 0, 0, 0, 0, 522, 517, 509, 519, 494, 507, 486, 512, 0, 0, 0, 0, 0, 0, 0, 0, 508, 514, 485, 518, 535, 537, 525, 524, 0, 0, 0, 0, 0, 0, 0, 0, 524, 473, 570, 483, 515, 508, 524, 501, 0, 0, 0, 0, 0, 0, 0, 0, 502, 525, 488, 510, 513, 522, 516, 532, 1, 0, 0, 0, 0, 0, 0, 0, 483, 535, 531, 531, 491, 511, 531, 497, 0, 0, 0, 0, 0, 0, 0, 0, 517, 515, 515, 532, 527, 504, 480, 508, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4065, 4066, 4146, 4098, 4109, 4110, 4102] avg=4096.0000 max=4146 max_rank=3 min=4065 min_rank=1 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16419 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[506, 488, 529, 540, 502, 481, 517, 515, 16, 9, 1, 0, 0, 0, 0, 0, 515, 514, 509, 507, 526, 517, 490, 454, 0, 0, 0, 0, 0, 0, 0, 0, 509, 487, 536, 542, 530, 499, 534, 484, 0, 0, 0, 0, 0, 0, 0, 0, 481, 509, 505, 507, 535, 508, 521, 541, 0, 0, 0, 0, 0, 0, 0, 0, 501, 507, 501, 501, 540, 505, 558, 503, 0, 0, 0, 0, 0, 0, 0, 0, 547, 526, 524, 523, 511, 502, 501, 483, 2, 0, 0, 0, 0, 0, 0, 0, 494, 504, 531, 516, 545, 504, 514, 515, 0, 0, 0, 0, 0, 0, 0, 0, 518, 510, 493, 496, 531, 507, 480, 507, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4032, 4121, 4107, 4116, 4119, 4123, 4046] avg=4096.0000 max=4123 max_rank=6 min=4032 min_rank=1 imbalance_ratio=1.006592 max_load_reduction=28645 max_load_reduction_pct=0.874176 moved_to_echo=16404 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[501, 498, 479, 514, 510, 527, 469, 536, 16, 9, 1, 0, 0, 0, 0, 0, 520, 502, 529, 498, 530, 546, 520, 519, 0, 0, 0, 0, 0, 0, 0, 0, 494, 497, 510, 557, 494, 517, 496, 535, 0, 0, 0, 0, 0, 0, 0, 0, 488, 534, 499, 513, 534, 503, 518, 505, 0, 0, 0, 0, 0, 0, 0, 0, 500, 542, 483, 533, 496, 486, 494, 472, 0, 0, 0, 0, 0, 0, 0, 0, 529, 539, 515, 511, 492, 506, 514, 500, 1, 0, 0, 0, 0, 0, 0, 0, 535, 501, 528, 549, 539, 472, 499, 505, 0, 0, 0, 0, 0, 0, 0, 0, 493, 522, 515, 502, 477, 543, 548, 505, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4164, 4100, 4094, 4006, 4107, 4128, 4109] avg=4096.0000 max=4164 max_rank=1 min=4006 min_rank=4 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16350 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33792 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[493, 530, 533, 514, 487, 521, 530, 554, 523, 511, 542, 505, 513, 521, 537, 505, 479, 505, 531, 493, 523, 516, 518, 540, 513, 498, 504, 497, 536, 515, 515, 500, 531, 503, 517, 505, 509, 525, 469, 535, 496, 510, 541, 541, 515, 488, 479, 489, 530, 516, 489, 507, 501, 524, 521, 489, 534, 501, 474, 499, 509, 498, 506, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[481, 503, 502, 536, 553, 505, 524, 501, 498, 497, 515, 505, 521, 485, 510, 492, 529, 488, 507, 541, 546, 560, 463, 526, 496, 505, 500, 503, 539, 486, 539, 490, 519, 522, 501, 482, 515, 506, 502, 515, 536, 499, 514, 499, 518, 545, 479, 523, 561, 500, 546, 494, 536, 470, 526, 517, 532, 510, 522, 521, 530, 487, 452, 543] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[497, 495, 518, 525, 490, 507, 545, 543, 484, 511, 485, 510, 480, 473, 530, 531, 512, 494, 505, 523, 535, 486, 510, 503, 563, 507, 503, 566, 521, 514, 526, 509, 516, 518, 546, 481, 500, 515, 533, 510, 505, 535, 487, 518, 500, 498, 517, 523, 551, 501, 490, 498, 520, 546, 517, 512, 513, 505, 466, 521, 507, 507, 530, 481] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[501, 542, 536, 518, 546, 510, 513, 542, 537, 515, 513, 526, 504, 493, 507, 542, 487, 513, 499, 491, 540, 520, 559, 502, 529, 495, 532, 543, 528, 484, 519, 496, 518, 474, 509, 500, 526, 507, 471, 503, 487, 531, 498, 510, 538, 529, 477, 527, 511, 498, 491, 503, 521, 502, 488, 508, 506, 502, 506, 517, 498, 502, 533, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[537, 524, 537, 546, 518, 518, 523, 477, 536, 522, 489, 488, 493, 522, 476, 488, 533, 491, 506, 541, 504, 514, 527, 486, 525, 511, 513, 507, 513, 529, 481, 499, 536, 498, 544, 515, 500, 541, 518, 525, 559, 506, 458, 523, 504, 500, 517, 503, 502, 496, 558, 485, 510, 491, 493, 516, 511, 502, 523, 525, 469, 530, 507, 529] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[537, 514, 535, 515, 523, 522, 526, 511, 504, 497, 517, 521, 544, 498, 497, 532, 513, 491, 516, 496, 499, 521, 515, 533, 491, 526, 512, 558, 482, 478, 498, 507, 507, 478, 497, 524, 504, 497, 511, 531, 514, 521, 515, 507, 485, 507, 513, 512, 535, 546, 510, 507, 481, 517, 492, 545, 521, 486, 525, 561, 549, 509, 447, 485] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[478, 492, 525, 534, 556, 528, 507, 493, 512, 536, 486, 537, 522, 531, 498, 510, 526, 490, 493, 507, 505, 520, 492, 502, 487, 524, 498, 513, 534, 535, 469, 501, 531, 535, 530, 514, 492, 507, 490, 513, 503, 499, 518, 534, 537, 516, 495, 541, 505, 510, 524, 508, 521, 461, 490, 495, 489, 533, 552, 523, 516, 508, 537, 500] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[527, 506, 537, 494, 535, 515, 510, 525, 534, 503, 523, 498, 489, 535, 500, 523, 526, 518, 482, 538, 514, 519, 555, 488, 492, 508, 507, 524, 442, 496, 504, 542, 515, 518, 491, 505, 565, 559, 482, 495, 534, 470, 510, 494, 495, 504, 509, 514, 530, 524, 533, 510, 544, 520, 529, 497, 508, 489, 499, 496, 486, 500, 489, 545] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[497, 495, 464, 525, 490, 507, 545, 543, 0, 0, 0, 0, 0, 0, 0, 0, 484, 511, 485, 510, 480, 473, 530, 531, 7, 2, 0, 0, 0, 0, 0, 0, 512, 494, 505, 523, 531, 486, 510, 503, 0, 0, 0, 0, 0, 0, 0, 0, 563, 507, 503, 566, 521, 514, 526, 509, 11, 0, 0, 0, 0, 0, 0, 0, 516, 518, 546, 481, 500, 515, 533, 510, 2, 0, 0, 0, 0, 0, 0, 0, 505, 535, 487, 518, 500, 498, 517, 523, 12, 0, 0, 0, 0, 0, 0, 0, 551, 501, 490, 498, 520, 546, 517, 512, 0, 0, 0, 0, 0, 0, 0, 0, 513, 505, 466, 521, 507, 507, 530, 481, 24, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 4013, 4064, 4220, 4121, 4095, 4135, 4054] avg=4096.0000 max=4220 max_rank=3 min=4013 min_rank=1 imbalance_ratio=1.030273 max_load_reduction=28548 max_load_reduction_pct=0.871216 moved_to_echo=16405 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[493, 530, 478, 514, 487, 521, 530, 554, 0, 0, 0, 0, 0, 0, 0, 0, 523, 511, 542, 505, 513, 521, 537, 505, 8, 2, 0, 0, 0, 0, 0, 0, 479, 505, 531, 493, 519, 516, 518, 540, 0, 0, 0, 0, 0, 0, 0, 0, 513, 498, 504, 497, 536, 515, 515, 500, 11, 0, 0, 0, 0, 0, 0, 0, 531, 503, 517, 505, 509, 525, 469, 535, 2, 0, 0, 0, 0, 0, 0, 0, 496, 510, 541, 541, 515, 488, 479, 489, 12, 0, 0, 0, 0, 0, 0, 0, 530, 516, 489, 507, 501, 524, 521, 489, 0, 0, 0, 0, 0, 0, 0, 0, 534, 501, 474, 499, 509, 498, 506, 515, 24, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4167, 4101, 4089, 4096, 4071, 4077, 4060] avg=4096.0000 max=4167 max_rank=1 min=4060 min_rank=7 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16304 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[481, 503, 451, 536, 553, 505, 524, 501, 0, 0, 0, 0, 0, 0, 0, 0, 498, 497, 515, 505, 521, 485, 510, 492, 7, 2, 0, 0, 0, 0, 0, 0, 529, 488, 507, 541, 542, 560, 463, 526, 0, 0, 0, 0, 0, 0, 0, 0, 496, 505, 500, 503, 539, 486, 539, 490, 10, 0, 0, 0, 0, 0, 0, 0, 519, 522, 501, 482, 515, 506, 502, 515, 2, 0, 0, 0, 0, 0, 0, 0, 536, 499, 514, 499, 518, 545, 479, 523, 11, 0, 0, 0, 0, 0, 0, 0, 561, 500, 546, 494, 536, 470, 526, 517, 0, 0, 0, 0, 0, 0, 0, 0, 532, 510, 522, 521, 530, 487, 452, 543, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4054, 4032, 4156, 4068, 4064, 4124, 4150, 4120] avg=4096.0000 max=4156 max_rank=2 min=4032 min_rank=1 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16458 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[537, 524, 467, 546, 518, 518, 523, 477, 0, 0, 0, 0, 0, 0, 0, 0, 536, 522, 489, 488, 493, 522, 476, 488, 11, 7, 0, 0, 0, 0, 0, 0, 533, 491, 506, 541, 492, 514, 527, 486, 0, 0, 0, 0, 0, 0, 0, 0, 525, 511, 513, 507, 513, 529, 481, 499, 15, 0, 0, 0, 0, 0, 0, 0, 536, 498, 544, 515, 500, 541, 518, 525, 5, 1, 0, 0, 0, 0, 0, 0, 559, 506, 458, 523, 504, 500, 517, 503, 16, 0, 0, 0, 0, 0, 0, 0, 501, 496, 558, 485, 510, 491, 493, 516, 0, 0, 0, 0, 0, 0, 0, 0, 511, 502, 523, 525, 469, 530, 507, 529, 28, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4032, 4090, 4093, 4183, 4086, 4050, 4124] avg=4096.0000 max=4183 max_rank=4 min=4032 min_rank=1 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16443 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[501, 542, 480, 518, 546, 510, 513, 542, 0, 0, 0, 0, 0, 0, 0, 0, 537, 515, 513, 526, 504, 493, 507, 542, 8, 2, 0, 0, 0, 0, 0, 0, 487, 513, 499, 491, 536, 520, 559, 502, 0, 0, 0, 0, 0, 0, 0, 0, 529, 495, 532, 543, 528, 484, 519, 496, 11, 0, 0, 0, 0, 0, 0, 0, 518, 474, 509, 500, 526, 507, 471, 503, 2, 0, 0, 0, 0, 0, 0, 0, 487, 531, 498, 510, 538, 529, 477, 527, 12, 0, 0, 0, 0, 0, 0, 0, 511, 498, 491, 503, 521, 502, 488, 508, 0, 0, 0, 0, 0, 0, 0, 0, 506, 502, 506, 517, 498, 502, 533, 495, 25, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4152, 4147, 4107, 4137, 4010, 4109, 4022, 4084] avg=4096.0000 max=4152 max_rank=0 min=4010 min_rank=4 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16225 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[478, 492, 470, 534, 556, 528, 507, 493, 0, 0, 0, 0, 0, 0, 0, 0, 512, 536, 486, 537, 522, 531, 498, 510, 8, 2, 0, 0, 0, 0, 0, 0, 526, 490, 493, 507, 502, 520, 492, 502, 0, 0, 0, 0, 0, 0, 0, 0, 487, 524, 498, 513, 534, 535, 469, 501, 11, 0, 0, 0, 0, 0, 0, 0, 531, 535, 530, 514, 492, 507, 490, 513, 1, 0, 0, 0, 0, 0, 0, 0, 503, 499, 518, 534, 537, 516, 495, 541, 12, 0, 0, 0, 0, 0, 0, 0, 505, 510, 524, 508, 521, 461, 490, 495, 0, 0, 0, 0, 0, 0, 0, 0, 489, 533, 552, 523, 516, 508, 537, 500, 24, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4058, 4142, 4032, 4072, 4113, 4155, 4014, 4182] avg=4096.0000 max=4182 max_rank=7 min=4014 min_rank=6 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16464 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[537, 514, 480, 515, 523, 522, 526, 511, 0, 0, 0, 0, 0, 0, 0, 0, 504, 497, 517, 521, 544, 498, 497, 532, 8, 2, 0, 0, 0, 0, 0, 0, 513, 491, 516, 496, 496, 521, 515, 533, 0, 0, 0, 0, 0, 0, 0, 0, 491, 526, 512, 558, 482, 478, 498, 507, 11, 0, 0, 0, 0, 0, 0, 0, 507, 478, 497, 524, 504, 497, 511, 531, 1, 0, 0, 0, 0, 0, 0, 0, 514, 521, 515, 507, 485, 507, 513, 512, 12, 0, 0, 0, 0, 0, 0, 0, 535, 546, 510, 507, 481, 517, 492, 545, 0, 0, 0, 0, 0, 0, 0, 0, 521, 486, 525, 561, 549, 509, 447, 485, 24, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4120, 4081, 4063, 4050, 4086, 4133, 4107] avg=4096.0000 max=4133 max_rank=6 min=4050 min_rank=4 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16376 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[527, 506, 481, 494, 535, 515, 510, 525, 0, 0, 0, 0, 0, 0, 0, 0, 534, 503, 523, 498, 489, 535, 500, 523, 8, 2, 0, 0, 0, 0, 0, 0, 526, 518, 482, 538, 511, 519, 555, 488, 0, 0, 0, 0, 0, 0, 0, 0, 492, 508, 507, 524, 442, 496, 504, 542, 11, 0, 0, 0, 0, 0, 0, 0, 515, 518, 491, 505, 565, 559, 482, 495, 1, 0, 0, 0, 0, 0, 0, 0, 534, 470, 510, 494, 495, 504, 509, 514, 12, 0, 0, 0, 0, 0, 0, 0, 530, 524, 533, 510, 544, 520, 529, 497, 0, 0, 0, 0, 0, 0, 0, 0, 508, 489, 499, 496, 486, 500, 489, 545, 25, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4115, 4137, 4026, 4131, 4042, 4187, 4037] avg=4096.0000 max=4187 max_rank=6 min=4026 min_rank=3 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 3840, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[518, 524, 527, 523, 506, 525, 562, 467, 532, 547, 531, 514, 527, 523, 497, 538, 496, 527, 522, 508, 510, 495, 508, 534, 471, 521, 530, 511, 536, 485, 522, 482, 509, 522, 495, 484, 510, 516, 502, 505, 507, 497, 492, 509, 497, 492, 485, 505, 549, 492, 488, 544, 544, 524, 545, 466, 521, 497, 490, 504, 534, 521, 516, 487] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[513, 495, 550, 506, 506, 495, 523, 514, 493, 523, 502, 505, 512, 503, 510, 513, 510, 533, 523, 486, 512, 502, 510, 519, 516, 518, 520, 534, 515, 519, 499, 474, 523, 487, 541, 525, 473, 491, 515, 510, 509, 515, 551, 506, 536, 527, 519, 500, 519, 498, 509, 517, 522, 525, 528, 519, 481, 481, 527, 512, 511, 501, 520, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[522, 539, 514, 531, 495, 519, 526, 516, 458, 529, 516, 526, 503, 515, 541, 514, 478, 498, 496, 550, 529, 514, 462, 484, 547, 493, 502, 512, 499, 466, 510, 485, 522, 505, 518, 519, 472, 516, 502, 503, 539, 530, 507, 538, 530, 497, 535, 527, 530, 528, 492, 555, 513, 505, 522, 504, 490, 509, 534, 522, 485, 514, 495, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[562, 507, 536, 521, 562, 496, 501, 523, 538, 517, 513, 544, 561, 509, 531, 509, 495, 517, 522, 513, 546, 500, 469, 498, 517, 491, 494, 512, 495, 472, 533, 497, 518, 541, 499, 501, 542, 528, 522, 540, 523, 494, 523, 520, 473, 507, 457, 506, 479, 528, 488, 480, 519, 492, 514, 502, 519, 520, 527, 503, 499, 497, 501, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[527, 507, 503, 519, 515, 503, 507, 534, 490, 507, 532, 538, 517, 529, 516, 504, 531, 476, 509, 504, 528, 528, 543, 534, 480, 478, 478, 485, 487, 496, 509, 543, 541, 506, 492, 510, 559, 530, 495, 506, 524, 520, 505, 501, 486, 511, 515, 489, 513, 507, 514, 513, 496, 509, 525, 507, 533, 516, 518, 513, 514, 505, 505, 533] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[487, 571, 501, 492, 507, 545, 520, 518, 520, 493, 569, 501, 559, 520, 521, 509, 507, 526, 534, 487, 519, 521, 529, 486, 514, 494, 518, 535, 505, 508, 540, 513, 522, 517, 489, 530, 511, 517, 533, 468, 491, 515, 490, 513, 501, 499, 499, 496, 496, 507, 513, 493, 518, 557, 533, 503, 486, 518, 496, 479, 522, 468, 492, 527] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[513, 495, 508, 517, 532, 487, 518, 524, 524, 484, 526, 540, 497, 505, 545, 493, 556, 496, 527, 527, 520, 515, 499, 494, 505, 507, 528, 494, 503, 486, 520, 550, 583, 508, 557, 515, 506, 516, 506, 475, 506, 554, 499, 536, 482, 479, 506, 506, 501, 496, 502, 484, 522, 521, 504, 493, 519, 521, 514, 490, 507, 525, 513, 487] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[531, 504, 515, 480, 518, 553, 511, 510, 498, 489, 463, 524, 473, 472, 525, 513, 525, 506, 527, 545, 517, 531, 529, 493, 495, 555, 511, 495, 547, 540, 462, 537, 549, 552, 488, 484, 477, 508, 498, 478, 529, 514, 517, 509, 488, 515, 494, 528, 517, 479, 518, 521, 548, 494, 522, 512, 522, 516, 510, 491, 536, 500, 553, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[479, 524, 527, 523, 506, 525, 562, 467, 0, 0, 0, 0, 0, 0, 0, 0, 532, 547, 531, 488, 527, 523, 497, 538, 0, 0, 0, 0, 0, 0, 0, 0, 496, 527, 522, 508, 502, 495, 508, 534, 0, 0, 0, 0, 0, 0, 0, 0, 471, 521, 530, 511, 536, 485, 522, 482, 33, 0, 0, 0, 0, 0, 0, 0, 508, 522, 495, 484, 510, 516, 502, 505, 0, 0, 0, 0, 0, 0, 0, 0, 507, 497, 492, 509, 497, 492, 485, 505, 17, 6, 0, 0, 0, 0, 0, 0, 549, 492, 488, 544, 544, 524, 544, 466, 0, 0, 0, 0, 0, 0, 0, 0, 521, 497, 490, 504, 534, 521, 516, 487, 9, 8, 1, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4183, 4092, 4091, 4042, 4007, 4151, 4089] avg=4096.0000 max=4183 max_rank=1 min=4007 min_rank=5 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16289 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[481, 539, 514, 531, 495, 519, 526, 516, 0, 0, 0, 0, 0, 0, 0, 0, 458, 529, 516, 499, 503, 515, 541, 514, 0, 0, 0, 0, 0, 0, 0, 0, 478, 498, 496, 550, 521, 514, 462, 484, 0, 0, 0, 0, 0, 0, 0, 0, 547, 493, 502, 512, 499, 466, 510, 485, 34, 0, 0, 0, 0, 0, 0, 0, 521, 505, 518, 519, 472, 516, 502, 503, 0, 0, 0, 0, 0, 0, 0, 0, 539, 530, 507, 538, 530, 497, 535, 527, 17, 7, 0, 0, 0, 0, 0, 0, 530, 528, 492, 555, 513, 505, 521, 504, 0, 0, 0, 0, 0, 0, 0, 0, 490, 509, 534, 522, 485, 514, 495, 521, 10, 8, 1, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4075, 4003, 4048, 4056, 4227, 4148, 4090] avg=4096.0000 max=4227 max_rank=5 min=4003 min_rank=2 imbalance_ratio=1.031982 max_load_reduction=28541 max_load_reduction_pct=0.871002 moved_to_echo=16521 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[519, 507, 536, 521, 562, 496, 501, 523, 0, 0, 0, 0, 0, 0, 0, 0, 538, 517, 513, 516, 561, 509, 531, 509, 0, 0, 0, 0, 0, 0, 0, 0, 495, 517, 522, 513, 538, 500, 469, 498, 0, 0, 0, 0, 0, 0, 0, 0, 517, 491, 494, 512, 495, 472, 533, 497, 36, 0, 0, 0, 0, 0, 0, 0, 517, 541, 499, 501, 542, 528, 522, 540, 0, 0, 0, 0, 0, 0, 0, 0, 523, 494, 523, 520, 473, 507, 457, 506, 18, 7, 0, 0, 0, 0, 0, 0, 479, 528, 488, 480, 519, 492, 513, 502, 0, 0, 0, 0, 0, 0, 0, 0, 519, 520, 527, 503, 499, 497, 501, 505, 10, 8, 1, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4165, 4194, 4052, 4047, 4190, 4028, 4001, 4091] avg=4096.0000 max=4194 max_rank=1 min=4001 min_rank=6 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16310 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[490, 504, 515, 480, 518, 553, 511, 510, 0, 0, 0, 0, 0, 0, 0, 0, 498, 489, 463, 497, 473, 472, 525, 513, 0, 0, 0, 0, 0, 0, 0, 0, 525, 506, 527, 545, 509, 531, 529, 493, 0, 0, 0, 0, 0, 0, 0, 0, 495, 555, 511, 495, 547, 540, 462, 537, 34, 0, 0, 0, 0, 0, 0, 0, 548, 552, 488, 484, 477, 508, 498, 478, 0, 0, 0, 0, 0, 0, 0, 0, 529, 514, 517, 509, 488, 515, 494, 528, 17, 7, 0, 0, 0, 0, 0, 0, 517, 479, 518, 521, 548, 494, 521, 512, 0, 0, 0, 0, 0, 0, 0, 0, 522, 516, 510, 491, 536, 500, 553, 507, 10, 8, 1, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 3930, 4165, 4176, 4033, 4118, 4110, 4155] avg=4096.0000 max=4176 max_rank=3 min=3930 min_rank=1 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16416 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[442, 571, 501, 492, 507, 545, 520, 518, 0, 0, 0, 0, 0, 0, 0, 0, 520, 493, 569, 469, 559, 520, 521, 509, 0, 0, 0, 0, 0, 0, 0, 0, 507, 526, 534, 487, 508, 521, 529, 486, 0, 0, 0, 0, 0, 0, 0, 0, 514, 494, 518, 535, 505, 508, 540, 513, 35, 0, 0, 0, 0, 0, 0, 0, 515, 517, 489, 530, 511, 517, 533, 468, 0, 0, 0, 0, 0, 0, 0, 0, 491, 515, 490, 513, 501, 499, 499, 496, 20, 10, 0, 0, 0, 0, 0, 0, 496, 507, 513, 493, 518, 557, 530, 503, 0, 0, 0, 0, 0, 0, 0, 0, 486, 518, 496, 479, 522, 468, 492, 527, 12, 11, 7, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4160, 4098, 4162, 4080, 4034, 4117, 4021] avg=4096.0000 max=4162 max_rank=3 min=4021 min_rank=7 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16252 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[486, 507, 503, 519, 515, 503, 507, 534, 0, 0, 0, 0, 0, 0, 0, 0, 490, 507, 532, 510, 517, 529, 516, 504, 0, 0, 0, 0, 0, 0, 0, 0, 531, 476, 509, 504, 520, 528, 543, 534, 0, 0, 0, 0, 0, 0, 0, 0, 480, 478, 478, 485, 487, 496, 509, 543, 34, 0, 0, 0, 0, 0, 0, 0, 540, 506, 492, 510, 559, 530, 495, 506, 0, 0, 0, 0, 0, 0, 0, 0, 524, 520, 505, 501, 486, 511, 515, 489, 18, 7, 0, 0, 0, 0, 0, 0, 513, 507, 514, 513, 496, 509, 524, 507, 0, 0, 0, 0, 0, 0, 0, 0, 533, 516, 518, 513, 514, 505, 505, 533, 10, 8, 1, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4105, 4145, 3990, 4138, 4076, 4083, 4157] avg=4096.0000 max=4157 max_rank=7 min=3990 min_rank=3 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16454 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[474, 495, 508, 517, 532, 487, 518, 524, 0, 0, 0, 0, 0, 0, 0, 0, 524, 484, 526, 512, 497, 505, 545, 493, 0, 0, 0, 0, 0, 0, 0, 0, 556, 496, 527, 527, 512, 515, 499, 494, 0, 0, 0, 0, 0, 0, 0, 0, 505, 507, 528, 494, 503, 486, 520, 550, 33, 0, 0, 0, 0, 0, 0, 0, 582, 508, 557, 515, 506, 516, 506, 475, 0, 0, 0, 0, 0, 0, 0, 0, 506, 554, 499, 536, 482, 479, 506, 506, 18, 6, 0, 0, 0, 0, 0, 0, 501, 496, 502, 484, 522, 521, 503, 493, 0, 0, 0, 0, 0, 0, 0, 0, 519, 521, 514, 490, 507, 525, 513, 487, 10, 8, 1, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4055, 4086, 4126, 4126, 4165, 4092, 4022, 4096] avg=4096.0000 max=4165 max_rank=4 min=4022 min_rank=6 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[474, 495, 550, 506, 506, 495, 523, 514, 0, 0, 0, 0, 0, 0, 0, 0, 493, 523, 502, 479, 512, 503, 510, 513, 0, 0, 0, 0, 0, 0, 0, 0, 510, 533, 523, 486, 504, 502, 510, 519, 0, 0, 0, 0, 0, 0, 0, 0, 516, 518, 520, 534, 515, 519, 499, 474, 33, 0, 0, 0, 0, 0, 0, 0, 522, 487, 541, 525, 473, 491, 515, 510, 0, 0, 0, 0, 0, 0, 0, 0, 509, 515, 551, 506, 536, 527, 519, 500, 17, 6, 0, 0, 0, 0, 0, 0, 519, 498, 509, 517, 522, 525, 527, 519, 0, 0, 0, 0, 0, 0, 0, 0, 481, 481, 527, 512, 511, 501, 520, 517, 9, 8, 1, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4035, 4087, 4128, 4064, 4186, 4136, 4069] avg=4096.0000 max=4186 max_rank=5 min=4035 min_rank=1 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16455 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 128, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4352, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3968, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[486, 489, 506, 521, 516, 524, 502, 521, 488, 463, 499, 498, 500, 535, 522, 504, 512, 496, 509, 510, 511, 525, 506, 502, 493, 532, 484, 496, 526, 504, 559, 503, 536, 529, 497, 508, 571, 511, 486, 493, 542, 519, 531, 531, 528, 482, 520, 483, 496, 497, 512, 507, 549, 549, 554, 521, 488, 519, 492, 519, 512, 545, 483, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[513, 536, 523, 488, 508, 499, 541, 469, 555, 527, 518, 486, 491, 524, 530, 467, 509, 525, 497, 531, 531, 513, 522, 542, 486, 474, 547, 541, 558, 505, 495, 501, 489, 488, 524, 529, 562, 513, 515, 497, 509, 507, 518, 548, 508, 495, 498, 552, 525, 503, 496, 482, 514, 495, 523, 499, 485, 525, 486, 508, 508, 514, 499, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[503, 522, 532, 495, 530, 554, 502, 524, 507, 504, 513, 538, 504, 511, 542, 509, 464, 513, 543, 522, 552, 508, 510, 496, 545, 530, 498, 475, 497, 551, 516, 521, 486, 532, 495, 517, 519, 520, 505, 516, 509, 527, 504, 482, 477, 511, 474, 510, 513, 509, 512, 483, 506, 529, 518, 515, 508, 496, 494, 502, 485, 531, 519, 533] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[499, 535, 511, 508, 543, 518, 511, 517, 491, 529, 523, 490, 462, 478, 537, 481, 522, 500, 526, 509, 489, 505, 500, 553, 531, 539, 523, 511, 534, 512, 532, 502, 513, 528, 496, 506, 490, 496, 525, 503, 529, 502, 521, 489, 539, 450, 530, 521, 500, 522, 495, 511, 499, 509, 548, 570, 506, 473, 481, 506, 530, 519, 489, 551] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[527, 535, 523, 503, 511, 500, 525, 521, 529, 535, 483, 512, 555, 546, 519, 487, 474, 508, 523, 520, 471, 500, 503, 533, 485, 525, 496, 509, 476, 535, 554, 523, 513, 500, 520, 506, 509, 535, 475, 500, 540, 486, 512, 532, 521, 494, 513, 548, 498, 524, 494, 531, 535, 455, 498, 506, 510, 518, 530, 517, 488, 512, 488, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[493, 522, 521, 485, 560, 459, 497, 498, 478, 509, 505, 474, 511, 539, 492, 541, 521, 513, 502, 539, 518, 494, 484, 513, 518, 558, 509, 560, 539, 498, 495, 504, 531, 528, 487, 537, 527, 527, 536, 514, 514, 532, 534, 526, 539, 520, 542, 542, 474, 493, 462, 504, 504, 510, 494, 530, 463, 526, 518, 486, 527, 481, 508, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[505, 528, 496, 514, 502, 508, 500, 502, 470, 489, 519, 511, 518, 521, 502, 531, 507, 546, 510, 518, 512, 525, 528, 529, 504, 547, 506, 456, 544, 522, 508, 495, 546, 492, 516, 490, 505, 468, 530, 515, 495, 541, 556, 486, 531, 513, 528, 497, 518, 528, 484, 523, 491, 543, 498, 512, 518, 515, 516, 524, 482, 505, 500, 529] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[486, 512, 488, 505, 550, 497, 480, 487, 550, 472, 468, 514, 506, 513, 497, 502, 517, 511, 507, 542, 537, 529, 524, 511, 518, 494, 491, 539, 549, 477, 484, 474, 489, 544, 530, 496, 541, 513, 510, 513, 517, 516, 493, 503, 520, 484, 534, 526, 475, 478, 526, 554, 517, 543, 509, 538, 536, 494, 545, 543, 521, 492, 518, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[503, 522, 532, 495, 530, 554, 502, 524, 8, 1, 0, 0, 0, 0, 0, 0, 507, 504, 513, 538, 504, 511, 542, 509, 25, 3, 0, 0, 0, 0, 0, 0, 464, 513, 543, 504, 552, 508, 510, 496, 0, 0, 0, 0, 0, 0, 0, 0, 545, 530, 498, 475, 481, 551, 516, 521, 0, 0, 0, 0, 0, 0, 0, 0, 486, 532, 495, 517, 511, 520, 505, 516, 0, 0, 0, 0, 0, 0, 0, 0, 509, 527, 504, 482, 477, 511, 474, 485, 0, 0, 0, 0, 0, 0, 0, 0, 513, 509, 512, 483, 506, 529, 518, 515, 7, 0, 0, 0, 0, 0, 0, 0, 508, 496, 494, 502, 485, 531, 519, 533, 15, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4171, 4156, 4090, 4117, 4082, 3969, 4092, 4091] avg=4096.0000 max=4171 max_rank=0 min=3969 min_rank=5 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16234 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[486, 489, 506, 521, 516, 524, 502, 521, 8, 1, 0, 0, 0, 0, 0, 0, 488, 463, 499, 498, 500, 535, 522, 504, 24, 3, 0, 0, 0, 0, 0, 0, 512, 496, 509, 493, 511, 525, 506, 502, 0, 0, 0, 0, 0, 0, 0, 0, 493, 532, 484, 496, 509, 504, 559, 503, 0, 0, 0, 0, 0, 0, 0, 0, 536, 529, 497, 508, 563, 511, 486, 493, 0, 0, 0, 0, 0, 0, 0, 0, 542, 519, 531, 531, 528, 482, 520, 459, 0, 0, 0, 0, 0, 0, 0, 0, 496, 497, 512, 507, 549, 549, 554, 521, 7, 0, 0, 0, 0, 0, 0, 0, 488, 519, 492, 519, 512, 545, 483, 516, 14, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4036, 4054, 4080, 4123, 4112, 4192, 4097] avg=4096.0000 max=4192 max_rank=6 min=4036 min_rank=1 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16524 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[513, 536, 523, 488, 508, 499, 541, 469, 9, 1, 0, 0, 0, 0, 0, 0, 555, 527, 518, 486, 491, 524, 530, 467, 28, 3, 0, 0, 0, 0, 0, 0, 509, 525, 497, 513, 531, 513, 522, 542, 0, 0, 0, 0, 0, 0, 0, 0, 486, 474, 547, 541, 540, 505, 495, 501, 0, 0, 0, 0, 0, 0, 0, 0, 489, 488, 524, 529, 554, 513, 515, 497, 0, 0, 0, 0, 0, 0, 0, 0, 509, 507, 518, 548, 508, 495, 498, 524, 0, 0, 0, 0, 0, 0, 0, 0, 525, 503, 496, 482, 514, 495, 523, 499, 7, 0, 0, 0, 0, 0, 0, 0, 485, 525, 486, 508, 508, 514, 499, 502, 15, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4129, 4152, 4089, 4109, 4107, 4044, 4051] avg=4096.0000 max=4152 max_rank=2 min=4044 min_rank=6 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16311 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[527, 535, 523, 503, 511, 500, 525, 521, 11, 5, 0, 0, 0, 0, 0, 0, 529, 535, 483, 512, 555, 546, 519, 487, 32, 9, 0, 0, 0, 0, 0, 0, 474, 508, 523, 491, 471, 500, 503, 533, 0, 0, 0, 0, 0, 0, 0, 0, 485, 525, 496, 509, 452, 535, 554, 523, 0, 0, 0, 0, 0, 0, 0, 0, 513, 500, 520, 506, 493, 535, 475, 500, 0, 0, 0, 0, 0, 0, 0, 0, 540, 486, 512, 532, 521, 494, 513, 516, 0, 0, 0, 0, 0, 0, 0, 0, 498, 524, 494, 531, 535, 455, 498, 506, 11, 0, 0, 0, 0, 0, 0, 0, 510, 518, 530, 517, 488, 512, 488, 514, 20, 13, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4161, 4207, 4003, 4079, 4042, 4114, 4052, 4110] avg=4096.0000 max=4207 max_rank=1 min=4003 min_rank=2 imbalance_ratio=1.027100 max_load_reduction=28561 max_load_reduction_pct=0.871613 moved_to_echo=16318 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[493, 522, 521, 485, 560, 459, 497, 498, 8, 1, 0, 0, 0, 0, 0, 0, 478, 509, 505, 474, 511, 539, 492, 541, 27, 3, 0, 0, 0, 0, 0, 0, 521, 513, 502, 521, 518, 494, 484, 513, 0, 0, 0, 0, 0, 0, 0, 0, 518, 558, 509, 560, 522, 498, 495, 504, 0, 0, 0, 0, 0, 0, 0, 0, 531, 528, 487, 537, 519, 527, 536, 514, 0, 0, 0, 0, 0, 0, 0, 0, 514, 532, 534, 526, 539, 520, 542, 515, 0, 0, 0, 0, 0, 0, 0, 0, 474, 493, 462, 504, 504, 510, 494, 530, 7, 0, 0, 0, 0, 0, 0, 0, 463, 526, 518, 486, 527, 481, 508, 503, 15, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4044, 4079, 4066, 4164, 4179, 4222, 3978, 4036] avg=4096.0000 max=4222 max_rank=5 min=3978 min_rank=6 imbalance_ratio=1.030762 max_load_reduction=28546 max_load_reduction_pct=0.871155 moved_to_echo=16415 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[499, 535, 511, 508, 543, 518, 511, 517, 8, 1, 0, 0, 0, 0, 0, 0, 491, 529, 523, 490, 462, 478, 537, 481, 26, 3, 0, 0, 0, 0, 0, 0, 522, 500, 526, 492, 489, 505, 500, 553, 0, 0, 0, 0, 0, 0, 0, 0, 531, 539, 523, 511, 517, 512, 532, 502, 0, 0, 0, 0, 0, 0, 0, 0, 513, 528, 496, 506, 483, 496, 525, 503, 0, 0, 0, 0, 0, 0, 0, 0, 529, 502, 521, 489, 539, 450, 530, 495, 0, 0, 0, 0, 0, 0, 0, 0, 500, 522, 495, 511, 499, 509, 548, 570, 6, 0, 0, 0, 0, 0, 0, 0, 506, 473, 481, 506, 530, 519, 489, 551, 14, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4151, 4020, 4087, 4167, 4050, 4055, 4160, 4078] avg=4096.0000 max=4167 max_rank=3 min=4020 min_rank=1 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16343 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[505, 528, 496, 514, 502, 508, 500, 502, 9, 1, 0, 0, 0, 0, 0, 0, 470, 489, 519, 511, 518, 521, 502, 531, 25, 3, 0, 0, 0, 0, 0, 0, 507, 546, 510, 500, 512, 525, 528, 529, 0, 0, 0, 0, 0, 0, 0, 0, 504, 547, 506, 456, 526, 522, 508, 495, 0, 0, 0, 0, 0, 0, 0, 0, 546, 492, 516, 490, 498, 468, 530, 515, 0, 0, 0, 0, 0, 0, 0, 0, 495, 541, 556, 486, 531, 513, 528, 472, 0, 0, 0, 0, 0, 0, 0, 0, 518, 528, 484, 523, 491, 543, 498, 512, 6, 0, 0, 0, 0, 0, 0, 0, 518, 515, 516, 524, 482, 505, 500, 529, 15, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4065, 4089, 4157, 4064, 4055, 4122, 4103, 4113] avg=4096.0000 max=4157 max_rank=2 min=4055 min_rank=4 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[486, 512, 488, 505, 550, 497, 480, 487, 9, 1, 0, 0, 0, 0, 0, 0, 550, 472, 468, 514, 506, 513, 497, 502, 26, 4, 0, 0, 0, 0, 0, 0, 517, 511, 507, 523, 537, 529, 524, 511, 0, 0, 0, 0, 0, 0, 0, 0, 518, 494, 491, 539, 531, 477, 484, 474, 0, 0, 0, 0, 0, 0, 0, 0, 489, 544, 530, 496, 533, 513, 510, 513, 0, 0, 0, 0, 0, 0, 0, 0, 517, 516, 493, 503, 520, 484, 534, 500, 0, 0, 0, 0, 0, 0, 0, 0, 475, 478, 526, 554, 517, 543, 509, 538, 7, 0, 0, 0, 0, 0, 0, 0, 536, 494, 545, 543, 521, 492, 518, 519, 15, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4015, 4052, 4159, 4008, 4128, 4067, 4147, 4192] avg=4096.0000 max=4192 max_rank=7 min=4008 min_rank=3 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16534 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 3968, 4224, 3968, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[525, 495, 497, 494, 536, 489, 519, 523, 545, 504, 515, 517, 537, 531, 499, 494, 517, 494, 507, 480, 493, 499, 519, 520, 504, 496, 527, 526, 473, 495, 487, 517, 502, 513, 515, 485, 505, 535, 533, 532, 529, 531, 534, 521, 493, 493, 511, 519, 540, 529, 507, 509, 509, 490, 536, 527, 510, 491, 530, 504, 521, 495, 494, 551] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[495, 485, 552, 488, 520, 536, 478, 518, 515, 538, 469, 512, 524, 531, 533, 514, 542, 527, 499, 485, 520, 518, 468, 574, 502, 523, 451, 520, 502, 520, 550, 541, 484, 512, 520, 551, 544, 538, 508, 473, 516, 539, 465, 537, 494, 498, 534, 503, 556, 491, 503, 501, 527, 508, 476, 485, 513, 512, 530, 496, 469, 530, 514, 491] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[487, 518, 547, 468, 520, 487, 527, 519, 537, 502, 520, 529, 532, 507, 546, 521, 506, 495, 539, 480, 516, 507, 517, 526, 497, 528, 539, 528, 512, 487, 557, 468, 466, 533, 504, 545, 532, 525, 538, 473, 488, 494, 501, 514, 502, 521, 515, 536, 516, 497, 484, 539, 543, 500, 491, 514, 495, 523, 512, 510, 494, 506, 504, 484] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[500, 500, 501, 517, 506, 471, 504, 557, 532, 516, 468, 529, 495, 525, 515, 520, 520, 514, 522, 477, 484, 480, 507, 520, 519, 480, 518, 521, 521, 512, 524, 505, 543, 567, 497, 498, 543, 516, 504, 528, 490, 522, 524, 498, 486, 515, 473, 532, 491, 469, 532, 487, 511, 523, 522, 517, 525, 545, 527, 518, 517, 516, 528, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[531, 497, 491, 521, 544, 503, 489, 500, 532, 525, 504, 502, 495, 494, 501, 508, 510, 505, 546, 503, 530, 520, 535, 520, 527, 471, 521, 535, 460, 506, 489, 499, 547, 502, 548, 509, 493, 515, 534, 499, 498, 530, 557, 501, 508, 532, 487, 494, 486, 522, 546, 491, 510, 477, 555, 483, 522, 502, 571, 500, 510, 520, 494, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[492, 496, 509, 558, 529, 526, 508, 473, 501, 524, 492, 473, 517, 542, 490, 513, 485, 502, 483, 492, 488, 500, 516, 499, 517, 526, 539, 506, 506, 491, 490, 519, 506, 549, 530, 502, 519, 507, 511, 500, 529, 537, 528, 518, 532, 488, 522, 482, 494, 486, 553, 495, 528, 529, 542, 531, 494, 523, 535, 549, 499, 512, 506, 530] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[516, 494, 523, 492, 515, 503, 469, 544, 522, 531, 504, 502, 539, 520, 518, 496, 503, 484, 525, 474, 528, 509, 530, 512, 522, 513, 484, 538, 491, 534, 517, 500, 534, 477, 504, 508, 495, 521, 512, 540, 484, 530, 526, 574, 502, 535, 555, 497, 474, 510, 502, 490, 539, 526, 526, 509, 485, 493, 559, 497, 490, 529, 507, 486] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[519, 500, 498, 508, 497, 494, 507, 461, 537, 528, 519, 544, 492, 506, 521, 500, 486, 514, 481, 496, 517, 505, 510, 514, 525, 518, 526, 511, 524, 537, 552, 484, 524, 473, 506, 532, 521, 498, 515, 524, 546, 526, 536, 520, 508, 506, 507, 531, 518, 482, 507, 548, 495, 547, 492, 529, 516, 499, 489, 502, 503, 518, 523, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[525, 495, 497, 494, 536, 489, 519, 523, 31, 4, 0, 0, 0, 0, 0, 0, 520, 504, 515, 517, 537, 531, 499, 494, 0, 0, 0, 0, 0, 0, 0, 0, 517, 494, 507, 480, 493, 499, 519, 520, 21, 14, 0, 0, 0, 0, 0, 0, 504, 496, 527, 526, 473, 495, 487, 517, 5, 0, 0, 0, 0, 0, 0, 0, 502, 513, 515, 485, 505, 535, 502, 532, 0, 0, 0, 0, 0, 0, 0, 0, 529, 508, 534, 521, 493, 493, 511, 519, 0, 0, 0, 0, 0, 0, 0, 0, 540, 529, 507, 509, 509, 490, 536, 527, 2, 0, 0, 0, 0, 0, 0, 0, 510, 491, 530, 504, 521, 495, 494, 551, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4117, 4064, 4030, 4089, 4108, 4149, 4098] avg=4096.0000 max=4149 max_rank=6 min=4030 min_rank=3 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16444 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[500, 500, 501, 517, 506, 471, 504, 557, 30, 4, 0, 0, 0, 0, 0, 0, 508, 516, 468, 529, 495, 525, 515, 520, 0, 0, 0, 0, 0, 0, 0, 0, 520, 514, 522, 477, 484, 480, 507, 520, 20, 13, 0, 0, 0, 0, 0, 0, 519, 480, 518, 521, 521, 512, 524, 505, 4, 0, 0, 0, 0, 0, 0, 0, 543, 567, 497, 498, 543, 516, 474, 528, 0, 0, 0, 0, 0, 0, 0, 0, 490, 502, 524, 498, 486, 515, 473, 532, 0, 0, 0, 0, 0, 0, 0, 0, 491, 469, 532, 487, 511, 523, 522, 517, 1, 0, 0, 0, 0, 0, 0, 0, 525, 545, 527, 518, 517, 516, 528, 524, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4076, 4057, 4104, 4166, 4020, 4053, 4202] avg=4096.0000 max=4202 max_rank=7 min=4020 min_rank=5 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16441 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[487, 518, 547, 468, 520, 487, 527, 519, 32, 4, 0, 0, 0, 0, 0, 0, 513, 502, 520, 529, 532, 507, 546, 521, 0, 0, 0, 0, 0, 0, 0, 0, 506, 495, 539, 480, 516, 507, 517, 526, 20, 13, 0, 0, 0, 0, 0, 0, 497, 528, 539, 528, 512, 487, 557, 468, 4, 0, 0, 0, 0, 0, 0, 0, 466, 533, 504, 545, 532, 525, 506, 473, 0, 0, 0, 0, 0, 0, 0, 0, 488, 474, 501, 514, 502, 521, 515, 536, 0, 0, 0, 0, 0, 0, 0, 0, 516, 497, 484, 539, 543, 500, 491, 514, 1, 0, 0, 0, 0, 0, 0, 0, 495, 523, 512, 510, 494, 506, 504, 484, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4170, 4119, 4120, 4084, 4051, 4085, 4030] avg=4096.0000 max=4170 max_rank=1 min=4030 min_rank=7 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16250 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[495, 485, 552, 488, 520, 536, 478, 518, 34, 6, 0, 0, 0, 0, 0, 0, 486, 538, 469, 512, 524, 531, 533, 514, 0, 0, 0, 0, 0, 0, 0, 0, 542, 527, 499, 485, 520, 518, 468, 574, 23, 18, 0, 0, 0, 0, 0, 0, 502, 523, 451, 520, 502, 520, 550, 541, 8, 0, 0, 0, 0, 0, 0, 0, 484, 512, 520, 551, 544, 538, 474, 473, 0, 0, 0, 0, 0, 0, 0, 0, 516, 503, 465, 537, 494, 498, 534, 503, 0, 0, 0, 0, 0, 0, 0, 0, 556, 491, 503, 501, 527, 508, 476, 485, 5, 0, 0, 0, 0, 0, 0, 0, 513, 512, 530, 496, 469, 530, 514, 491, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4107, 4174, 4117, 4096, 4050, 4052, 4060] avg=4096.0000 max=4174 max_rank=2 min=4050 min_rank=5 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16258 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[492, 496, 509, 558, 529, 526, 508, 473, 30, 3, 0, 0, 0, 0, 0, 0, 479, 524, 492, 473, 517, 542, 490, 513, 0, 0, 0, 0, 0, 0, 0, 0, 485, 502, 483, 492, 488, 500, 516, 499, 19, 14, 0, 0, 0, 0, 0, 0, 517, 526, 539, 506, 506, 491, 490, 519, 5, 0, 0, 0, 0, 0, 0, 0, 506, 549, 530, 502, 519, 507, 481, 500, 0, 0, 0, 0, 0, 0, 0, 0, 529, 514, 528, 518, 532, 488, 522, 482, 0, 0, 0, 0, 0, 0, 0, 0, 494, 486, 553, 495, 528, 529, 542, 531, 2, 0, 0, 0, 0, 0, 0, 0, 494, 523, 535, 549, 499, 512, 506, 530, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4124, 4030, 3998, 4099, 4094, 4113, 4160, 4150] avg=4096.0000 max=4160 max_rank=6 min=3998 min_rank=2 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16517 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[531, 497, 491, 521, 544, 503, 489, 500, 32, 4, 0, 0, 0, 0, 0, 0, 508, 525, 504, 502, 495, 494, 501, 508, 0, 0, 0, 0, 0, 0, 0, 0, 510, 505, 546, 503, 530, 520, 535, 520, 20, 13, 0, 0, 0, 0, 0, 0, 527, 471, 521, 535, 460, 506, 489, 499, 5, 0, 0, 0, 0, 0, 0, 0, 547, 502, 548, 509, 493, 515, 502, 499, 0, 0, 0, 0, 0, 0, 0, 0, 498, 508, 557, 501, 508, 532, 487, 494, 0, 0, 0, 0, 0, 0, 0, 0, 486, 522, 546, 491, 510, 477, 555, 483, 2, 0, 0, 0, 0, 0, 0, 0, 522, 502, 571, 500, 510, 520, 494, 511, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4037, 4202, 4013, 4115, 4085, 4072, 4132] avg=4096.0000 max=4202 max_rank=2 min=4013 min_rank=3 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16404 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[516, 494, 523, 492, 515, 503, 469, 544, 30, 4, 0, 0, 0, 0, 0, 0, 498, 531, 504, 502, 539, 520, 518, 496, 0, 0, 0, 0, 0, 0, 0, 0, 503, 484, 525, 474, 528, 509, 530, 512, 20, 13, 0, 0, 0, 0, 0, 0, 522, 513, 484, 538, 491, 534, 517, 500, 5, 0, 0, 0, 0, 0, 0, 0, 534, 477, 504, 508, 495, 521, 482, 540, 0, 0, 0, 0, 0, 0, 0, 0, 484, 508, 526, 574, 502, 535, 555, 497, 0, 0, 0, 0, 0, 0, 0, 0, 474, 510, 502, 490, 539, 526, 526, 509, 2, 0, 0, 0, 0, 0, 0, 0, 485, 493, 559, 497, 490, 529, 507, 486, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4108, 4098, 4104, 4061, 4181, 4078, 4048] avg=4096.0000 max=4181 max_rank=5 min=4048 min_rank=7 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[519, 500, 498, 508, 497, 494, 507, 461, 30, 4, 0, 0, 0, 0, 0, 0, 513, 528, 519, 544, 492, 506, 521, 500, 0, 0, 0, 0, 0, 0, 0, 0, 486, 514, 481, 496, 517, 505, 510, 514, 20, 13, 0, 0, 0, 0, 0, 0, 525, 518, 526, 511, 524, 537, 552, 484, 4, 0, 0, 0, 0, 0, 0, 0, 524, 473, 506, 532, 521, 498, 485, 524, 0, 0, 0, 0, 0, 0, 0, 0, 546, 506, 536, 520, 508, 506, 507, 531, 0, 0, 0, 0, 0, 0, 0, 0, 518, 482, 507, 548, 495, 547, 492, 529, 1, 0, 0, 0, 0, 0, 0, 0, 516, 499, 489, 502, 503, 518, 523, 496, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4018, 4123, 4056, 4181, 4063, 4160, 4119, 4048] avg=4096.0000 max=4181 max_rank=3 min=4018 min_rank=0 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4224, 3968, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4352, 4096, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 3968, 4096, 4096, 4224, 4224, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[490, 514, 533, 519, 495, 555, 506, 510, 524, 527, 506, 535, 528, 479, 554, 522, 502, 498, 500, 515, 523, 498, 508, 508, 505, 505, 516, 525, 504, 499, 508, 490, 497, 526, 512, 495, 502, 532, 508, 534, 508, 548, 467, 509, 502, 554, 508, 506, 527, 493, 508, 520, 518, 469, 501, 502, 519, 536, 496, 502, 486, 506, 514, 562] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[518, 490, 517, 517, 521, 504, 506, 493, 485, 511, 501, 550, 493, 515, 521, 485, 535, 516, 509, 540, 528, 530, 514, 525, 553, 490, 505, 509, 516, 505, 464, 490, 504, 478, 523, 542, 517, 512, 541, 502, 514, 517, 541, 544, 482, 489, 474, 531, 526, 548, 504, 532, 521, 489, 515, 486, 532, 474, 518, 516, 509, 521, 466, 544] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[487, 510, 487, 509, 480, 526, 506, 491, 545, 524, 492, 508, 483, 510, 485, 501, 508, 560, 509, 470, 521, 537, 537, 496, 475, 503, 520, 538, 538, 507, 528, 521, 548, 535, 508, 558, 504, 498, 522, 525, 493, 477, 467, 578, 512, 518, 510, 506, 536, 522, 511, 501, 522, 507, 547, 503, 489, 499, 500, 516, 524, 504, 503, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[499, 516, 483, 491, 518, 499, 502, 509, 542, 477, 529, 482, 537, 495, 548, 478, 525, 519, 473, 499, 497, 541, 519, 477, 540, 505, 506, 496, 501, 501, 518, 476, 515, 501, 519, 491, 534, 499, 505, 485, 524, 519, 547, 503, 490, 535, 593, 535, 492, 509, 522, 496, 496, 524, 521, 502, 517, 513, 535, 572, 551, 520, 503, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[497, 525, 511, 545, 553, 558, 529, 486, 535, 517, 570, 521, 508, 507, 487, 486, 541, 508, 514, 508, 507, 477, 526, 534, 537, 525, 511, 515, 499, 507, 513, 494, 493, 511, 529, 472, 547, 499, 531, 503, 532, 536, 476, 514, 482, 480, 512, 518, 498, 490, 499, 503, 503, 533, 488, 514, 529, 487, 515, 513, 546, 483, 486, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[561, 540, 533, 521, 561, 502, 477, 544, 485, 530, 506, 530, 500, 469, 501, 524, 516, 493, 499, 512, 523, 516, 521, 509, 542, 535, 523, 498, 492, 485, 515, 480, 505, 500, 496, 536, 530, 526, 480, 453, 519, 506, 511, 476, 503, 526, 519, 542, 507, 505, 465, 574, 532, 534, 545, 462, 505, 492, 514, 502, 538, 484, 531, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[514, 531, 500, 518, 475, 537, 500, 528, 521, 488, 509, 494, 524, 539, 531, 515, 501, 507, 522, 516, 505, 547, 505, 525, 507, 534, 489, 501, 491, 547, 497, 491, 538, 528, 510, 521, 467, 520, 534, 506, 496, 529, 527, 520, 528, 479, 496, 528, 506, 474, 515, 511, 528, 518, 529, 484, 510, 497, 501, 528, 510, 493, 497, 531] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[505, 520, 497, 539, 473, 504, 518, 509, 511, 547, 450, 518, 553, 536, 513, 492, 498, 520, 522, 498, 506, 514, 550, 496, 525, 527, 535, 534, 493, 510, 518, 541, 506, 471, 512, 497, 504, 484, 488, 522, 511, 545, 466, 498, 530, 562, 517, 509, 517, 502, 518, 500, 529, 517, 504, 521, 496, 507, 511, 478, 524, 505, 524, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[490, 514, 533, 519, 495, 550, 506, 510, 0, 0, 0, 0, 0, 0, 0, 0, 522, 527, 506, 535, 528, 479, 554, 522, 0, 0, 0, 0, 0, 0, 0, 0, 502, 498, 500, 515, 523, 498, 496, 508, 0, 0, 0, 0, 0, 0, 0, 0, 505, 505, 516, 525, 504, 499, 508, 490, 13, 0, 0, 0, 0, 0, 0, 0, 497, 526, 512, 495, 502, 532, 508, 534, 6, 3, 0, 0, 0, 0, 0, 0, 508, 532, 467, 509, 502, 554, 508, 506, 0, 0, 0, 0, 0, 0, 0, 0, 527, 493, 508, 520, 518, 469, 501, 502, 6, 2, 0, 0, 0, 0, 0, 0, 519, 536, 496, 502, 486, 506, 514, 562, 3, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4117, 4173, 4040, 4065, 4115, 4086, 4046, 4126] avg=4096.0000 max=4173 max_rank=1 min=4040 min_rank=2 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[487, 510, 487, 509, 480, 521, 506, 491, 0, 0, 0, 0, 0, 0, 0, 0, 543, 524, 492, 508, 483, 510, 485, 501, 0, 0, 0, 0, 0, 0, 0, 0, 508, 560, 509, 470, 521, 537, 525, 496, 0, 0, 0, 0, 0, 0, 0, 0, 475, 503, 520, 538, 538, 507, 528, 521, 11, 0, 0, 0, 0, 0, 0, 0, 548, 535, 508, 558, 504, 498, 522, 525, 6, 2, 0, 0, 0, 0, 0, 0, 493, 464, 467, 578, 512, 518, 510, 506, 0, 0, 0, 0, 0, 0, 0, 0, 536, 522, 511, 501, 522, 507, 547, 503, 6, 2, 0, 0, 0, 0, 0, 0, 489, 499, 500, 516, 524, 504, 503, 513, 3, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3991, 4046, 4126, 4141, 4206, 4048, 4157, 4053] avg=4096.0000 max=4206 max_rank=4 min=3991 min_rank=0 imbalance_ratio=1.026855 max_load_reduction=28562 max_load_reduction_pct=0.871643 moved_to_echo=16464 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[518, 490, 517, 517, 521, 499, 506, 493, 0, 0, 0, 0, 0, 0, 0, 0, 483, 511, 501, 550, 493, 515, 521, 485, 0, 0, 0, 0, 0, 0, 0, 0, 535, 516, 509, 540, 528, 530, 502, 525, 0, 0, 0, 0, 0, 0, 0, 0, 553, 490, 505, 509, 516, 505, 464, 490, 12, 0, 0, 0, 0, 0, 0, 0, 504, 478, 523, 542, 517, 512, 541, 502, 6, 3, 0, 0, 0, 0, 0, 0, 514, 502, 541, 544, 482, 489, 474, 531, 0, 0, 0, 0, 0, 0, 0, 0, 526, 548, 504, 532, 521, 489, 515, 486, 6, 2, 0, 0, 0, 0, 0, 0, 532, 474, 518, 516, 509, 521, 466, 544, 3, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4059, 4185, 4044, 4128, 4077, 4129, 4085] avg=4096.0000 max=4185 max_rank=2 min=4044 min_rank=3 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16419 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[499, 516, 483, 491, 518, 495, 502, 509, 0, 0, 0, 0, 0, 0, 0, 0, 540, 477, 529, 482, 537, 495, 548, 478, 0, 0, 0, 0, 0, 0, 0, 0, 525, 519, 473, 499, 497, 541, 507, 477, 0, 0, 0, 0, 0, 0, 0, 0, 540, 505, 506, 496, 501, 501, 518, 476, 12, 0, 0, 0, 0, 0, 0, 0, 515, 501, 519, 491, 534, 499, 505, 485, 6, 3, 0, 0, 0, 0, 0, 0, 524, 504, 547, 503, 490, 535, 593, 535, 0, 0, 0, 0, 0, 0, 0, 0, 492, 509, 522, 496, 496, 524, 521, 502, 6, 2, 0, 0, 0, 0, 0, 0, 517, 513, 535, 572, 551, 520, 503, 502, 2, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4013, 4086, 4038, 4055, 4058, 4231, 4070, 4217] avg=4096.0000 max=4231 max_rank=5 min=4013 min_rank=0 imbalance_ratio=1.032959 max_load_reduction=28537 max_load_reduction_pct=0.870880 moved_to_echo=16576 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[561, 540, 533, 521, 561, 498, 477, 544, 0, 0, 0, 0, 0, 0, 0, 0, 483, 530, 506, 530, 500, 469, 501, 524, 0, 0, 0, 0, 0, 0, 0, 0, 516, 493, 499, 512, 523, 516, 509, 509, 0, 0, 0, 0, 0, 0, 0, 0, 542, 535, 523, 498, 492, 485, 515, 480, 12, 0, 0, 0, 0, 0, 0, 0, 505, 500, 496, 536, 530, 526, 480, 453, 6, 3, 0, 0, 0, 0, 0, 0, 519, 491, 511, 476, 503, 526, 519, 542, 0, 0, 0, 0, 0, 0, 0, 0, 507, 505, 465, 574, 532, 534, 545, 462, 6, 2, 0, 0, 0, 0, 0, 0, 505, 492, 514, 502, 538, 484, 531, 507, 2, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4235, 4043, 4077, 4082, 4035, 4087, 4132, 4077] avg=4096.0000 max=4235 max_rank=0 min=4035 min_rank=4 imbalance_ratio=1.033936 max_load_reduction=28533 max_load_reduction_pct=0.870758 moved_to_echo=16331 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[497, 525, 511, 545, 553, 553, 529, 486, 0, 0, 0, 0, 0, 0, 0, 0, 533, 517, 570, 521, 508, 507, 487, 486, 0, 0, 0, 0, 0, 0, 0, 0, 541, 508, 514, 508, 507, 477, 514, 534, 0, 0, 0, 0, 0, 0, 0, 0, 537, 525, 511, 515, 499, 507, 513, 494, 12, 0, 0, 0, 0, 0, 0, 0, 493, 511, 529, 472, 547, 499, 531, 503, 6, 3, 0, 0, 0, 0, 0, 0, 532, 521, 476, 514, 482, 480, 512, 518, 0, 0, 0, 0, 0, 0, 0, 0, 498, 490, 499, 503, 503, 533, 488, 514, 6, 2, 0, 0, 0, 0, 0, 0, 529, 487, 515, 513, 546, 483, 486, 495, 3, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4199, 4129, 4103, 4113, 4094, 4035, 4036, 4059] avg=4096.0000 max=4199 max_rank=0 min=4035 min_rank=5 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16224 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[505, 520, 497, 539, 473, 493, 518, 509, 0, 0, 0, 0, 0, 0, 0, 0, 504, 547, 450, 518, 553, 536, 513, 492, 0, 0, 0, 0, 0, 0, 0, 0, 498, 520, 522, 498, 506, 514, 529, 496, 0, 0, 0, 0, 0, 0, 0, 0, 525, 527, 535, 534, 493, 510, 518, 541, 16, 0, 0, 0, 0, 0, 0, 0, 506, 471, 512, 497, 504, 484, 488, 522, 9, 6, 0, 0, 0, 0, 0, 0, 511, 523, 466, 498, 530, 562, 517, 509, 0, 0, 0, 0, 0, 0, 0, 0, 517, 502, 518, 500, 529, 517, 504, 521, 12, 5, 0, 0, 0, 0, 0, 0, 496, 507, 511, 478, 524, 505, 524, 521, 6, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4054, 4113, 4083, 4199, 3999, 4116, 4125, 4079] avg=4096.0000 max=4199 max_rank=3 min=3999 min_rank=4 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16319 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[514, 531, 500, 518, 475, 532, 500, 528, 0, 0, 0, 0, 0, 0, 0, 0, 519, 488, 509, 494, 524, 539, 531, 515, 0, 0, 0, 0, 0, 0, 0, 0, 501, 507, 522, 516, 505, 547, 493, 525, 0, 0, 0, 0, 0, 0, 0, 0, 507, 534, 489, 501, 491, 547, 497, 491, 12, 0, 0, 0, 0, 0, 0, 0, 538, 528, 510, 521, 467, 520, 534, 506, 6, 3, 0, 0, 0, 0, 0, 0, 496, 514, 527, 520, 528, 479, 496, 528, 0, 0, 0, 0, 0, 0, 0, 0, 506, 474, 515, 511, 528, 518, 529, 484, 6, 2, 0, 0, 0, 0, 0, 0, 510, 497, 501, 528, 510, 493, 497, 531, 3, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4119, 4116, 4069, 4133, 4088, 4073, 4072] avg=4096.0000 max=4133 max_rank=4 min=4069 min_rank=3 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[514, 520, 531, 503, 547, 512, 518, 509, 525, 488, 474, 527, 513, 526, 530, 525, 527, 526, 491, 494, 489, 511, 488, 537, 520, 545, 476, 510, 518, 503, 510, 506, 522, 517, 528, 525, 571, 512, 498, 505, 525, 503, 507, 473, 532, 516, 512, 486, 488, 504, 499, 522, 531, 487, 504, 493, 551, 491, 510, 532, 500, 535, 493, 483] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[508, 483, 557, 496, 496, 501, 498, 514, 518, 488, 500, 472, 497, 550, 540, 519, 522, 547, 503, 538, 493, 497, 543, 539, 521, 504, 519, 535, 502, 520, 505, 505, 518, 503, 520, 499, 506, 492, 480, 507, 498, 512, 481, 523, 524, 483, 486, 502, 528, 523, 494, 526, 494, 548, 494, 513, 533, 542, 524, 541, 508, 506, 524, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[552, 517, 504, 523, 522, 491, 538, 465, 532, 521, 485, 479, 503, 506, 520, 498, 537, 496, 543, 518, 515, 496, 545, 513, 537, 542, 487, 487, 535, 493, 509, 515, 529, 528, 503, 485, 541, 517, 522, 491, 488, 515, 502, 499, 529, 496, 511, 499, 515, 527, 477, 497, 526, 518, 497, 545, 479, 499, 540, 537, 541, 492, 512, 487] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[511, 522, 501, 535, 520, 512, 512, 511, 525, 522, 491, 529, 510, 480, 505, 490, 492, 515, 506, 531, 482, 496, 561, 516, 539, 520, 501, 540, 529, 510, 489, 503, 542, 567, 472, 527, 514, 559, 455, 499, 521, 544, 534, 488, 530, 523, 512, 504, 502, 516, 556, 512, 523, 495, 458, 480, 521, 483, 522, 508, 520, 478, 510, 487] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[495, 503, 497, 477, 522, 525, 523, 506, 517, 521, 505, 549, 517, 498, 550, 478, 508, 504, 491, 483, 503, 521, 504, 487, 538, 506, 514, 536, 524, 495, 526, 542, 511, 493, 539, 522, 519, 473, 511, 546, 539, 522, 507, 497, 502, 451, 487, 518, 523, 514, 494, 523, 489, 521, 512, 505, 545, 534, 542, 466, 578, 490, 537, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[512, 491, 507, 511, 523, 537, 532, 523, 477, 509, 503, 501, 518, 529, 511, 522, 509, 523, 493, 486, 477, 545, 517, 531, 523, 509, 528, 512, 477, 493, 500, 529, 505, 483, 530, 525, 504, 492, 534, 518, 497, 502, 501, 526, 515, 534, 586, 511, 501, 543, 530, 499, 487, 521, 500, 499, 514, 491, 518, 534, 524, 479, 514, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[493, 539, 528, 522, 516, 498, 511, 504, 535, 463, 531, 517, 515, 503, 532, 530, 494, 495, 507, 487, 522, 505, 485, 501, 555, 512, 494, 520, 482, 503, 541, 556, 505, 525, 508, 489, 517, 515, 534, 518, 485, 511, 525, 539, 524, 492, 521, 538, 458, 475, 558, 543, 511, 552, 467, 524, 513, 497, 507, 495, 501, 513, 510, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[496, 478, 536, 554, 534, 537, 518, 518, 537, 526, 517, 563, 477, 500, 485, 514, 534, 511, 548, 529, 506, 505, 503, 554, 505, 532, 479, 491, 528, 545, 508, 496, 521, 483, 515, 510, 482, 514, 524, 510, 526, 502, 491, 485, 502, 525, 503, 508, 526, 481, 488, 532, 513, 528, 497, 502, 536, 521, 480, 508, 496, 471, 504, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[514, 520, 531, 503, 529, 512, 518, 509, 0, 0, 0, 0, 0, 0, 0, 0, 525, 488, 474, 527, 513, 526, 530, 525, 3, 0, 0, 0, 0, 0, 0, 0, 527, 526, 491, 494, 489, 511, 488, 537, 0, 0, 0, 0, 0, 0, 0, 0, 501, 545, 476, 510, 518, 503, 510, 506, 0, 0, 0, 0, 0, 0, 0, 0, 522, 517, 528, 525, 563, 512, 498, 505, 0, 0, 0, 0, 0, 0, 0, 0, 525, 503, 507, 473, 532, 516, 512, 486, 17, 0, 0, 0, 0, 0, 0, 0, 488, 504, 499, 522, 531, 487, 504, 493, 19, 0, 0, 0, 0, 0, 0, 0, 551, 491, 510, 532, 500, 535, 493, 483, 5, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4136, 4111, 4063, 4069, 4170, 4071, 4047, 4101] avg=4096.0000 max=4170 max_rank=4 min=4047 min_rank=6 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[511, 522, 501, 535, 503, 512, 512, 511, 0, 0, 0, 0, 0, 0, 0, 0, 525, 522, 491, 529, 510, 480, 505, 490, 2, 0, 0, 0, 0, 0, 0, 0, 492, 515, 506, 531, 482, 496, 561, 516, 0, 0, 0, 0, 0, 0, 0, 0, 519, 520, 501, 540, 529, 510, 489, 503, 0, 0, 0, 0, 0, 0, 0, 0, 542, 567, 472, 527, 508, 559, 455, 499, 0, 0, 0, 0, 0, 0, 0, 0, 521, 544, 534, 488, 530, 523, 512, 504, 16, 0, 0, 0, 0, 0, 0, 0, 502, 516, 556, 512, 523, 495, 458, 480, 20, 0, 0, 0, 0, 0, 0, 0, 521, 483, 522, 508, 520, 478, 510, 487, 4, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4054, 4099, 4111, 4129, 4172, 4062, 4034] avg=4096.0000 max=4172 max_rank=5 min=4034 min_rank=7 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[508, 483, 557, 496, 480, 501, 498, 514, 0, 0, 0, 0, 0, 0, 0, 0, 518, 488, 500, 472, 497, 550, 540, 519, 2, 0, 0, 0, 0, 0, 0, 0, 522, 547, 503, 538, 493, 497, 543, 539, 0, 0, 0, 0, 0, 0, 0, 0, 502, 504, 519, 535, 502, 520, 505, 505, 0, 0, 0, 0, 0, 0, 0, 0, 518, 503, 520, 499, 500, 492, 480, 507, 0, 0, 0, 0, 0, 0, 0, 0, 498, 512, 481, 523, 524, 483, 486, 502, 15, 0, 0, 0, 0, 0, 0, 0, 528, 523, 494, 526, 494, 548, 494, 513, 19, 0, 0, 0, 0, 0, 0, 0, 533, 542, 524, 541, 508, 506, 524, 506, 4, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4037, 4086, 4182, 4092, 4019, 4024, 4139, 4189] avg=4096.0000 max=4189 max_rank=7 min=4019 min_rank=4 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16371 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[552, 517, 504, 523, 505, 491, 538, 465, 0, 0, 0, 0, 0, 0, 0, 0, 532, 521, 485, 479, 503, 506, 520, 498, 2, 0, 0, 0, 0, 0, 0, 0, 537, 496, 543, 518, 515, 496, 545, 513, 0, 0, 0, 0, 0, 0, 0, 0, 517, 542, 487, 487, 535, 493, 509, 515, 0, 0, 0, 0, 0, 0, 0, 0, 529, 528, 503, 485, 535, 517, 522, 491, 0, 0, 0, 0, 0, 0, 0, 0, 488, 515, 502, 499, 529, 496, 511, 499, 16, 0, 0, 0, 0, 0, 0, 0, 515, 527, 477, 497, 526, 518, 497, 545, 20, 0, 0, 0, 0, 0, 0, 0, 479, 499, 540, 537, 541, 492, 512, 487, 4, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4046, 4163, 4085, 4110, 4055, 4122, 4092] avg=4096.0000 max=4163 max_rank=2 min=4046 min_rank=1 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[493, 539, 528, 522, 499, 498, 511, 504, 0, 0, 0, 0, 0, 0, 0, 0, 535, 463, 531, 517, 515, 503, 532, 530, 2, 0, 0, 0, 0, 0, 0, 0, 494, 495, 507, 487, 522, 505, 485, 501, 0, 0, 0, 0, 0, 0, 0, 0, 535, 512, 494, 520, 482, 503, 541, 556, 0, 0, 0, 0, 0, 0, 0, 0, 505, 525, 508, 489, 511, 515, 534, 518, 0, 0, 0, 0, 0, 0, 0, 0, 485, 511, 525, 539, 524, 492, 521, 538, 16, 0, 0, 0, 0, 0, 0, 0, 458, 475, 558, 543, 511, 552, 467, 524, 20, 0, 0, 0, 0, 0, 0, 0, 513, 497, 507, 495, 501, 513, 510, 502, 4, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4128, 3996, 4143, 4105, 4151, 4108, 4043] avg=4096.0000 max=4151 max_rank=5 min=3996 min_rank=2 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16407 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[495, 503, 497, 477, 500, 525, 523, 506, 0, 0, 0, 0, 0, 0, 0, 0, 517, 521, 505, 549, 517, 498, 550, 478, 8, 7, 0, 0, 0, 0, 0, 0, 508, 504, 491, 483, 503, 521, 504, 480, 0, 0, 0, 0, 0, 0, 0, 0, 508, 506, 514, 536, 524, 495, 526, 542, 0, 0, 0, 0, 0, 0, 0, 0, 511, 493, 539, 522, 502, 473, 511, 546, 0, 0, 0, 0, 0, 0, 0, 0, 539, 522, 507, 497, 502, 451, 487, 518, 20, 6, 0, 0, 0, 0, 0, 0, 523, 514, 494, 523, 489, 521, 512, 505, 24, 0, 0, 0, 0, 0, 0, 0, 545, 534, 542, 466, 578, 490, 537, 493, 9, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4026, 4150, 3994, 4151, 4097, 4049, 4105, 4196] avg=4096.0000 max=4196 max_rank=7 min=3994 min_rank=2 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16447 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[512, 491, 507, 511, 506, 537, 532, 523, 0, 0, 0, 0, 0, 0, 0, 0, 477, 509, 503, 501, 518, 529, 511, 522, 2, 0, 0, 0, 0, 0, 0, 0, 509, 523, 493, 486, 477, 545, 517, 531, 0, 0, 0, 0, 0, 0, 0, 0, 504, 509, 528, 512, 477, 493, 500, 529, 0, 0, 0, 0, 0, 0, 0, 0, 505, 483, 530, 525, 498, 492, 534, 518, 0, 0, 0, 0, 0, 0, 0, 0, 497, 502, 501, 526, 515, 534, 586, 511, 16, 0, 0, 0, 0, 0, 0, 0, 501, 543, 530, 499, 487, 521, 500, 499, 19, 0, 0, 0, 0, 0, 0, 0, 514, 491, 518, 534, 524, 479, 514, 493, 4, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4072, 4081, 4052, 4085, 4188, 4099, 4072] avg=4096.0000 max=4188 max_rank=5 min=4052 min_rank=3 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16444 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[496, 478, 536, 554, 517, 537, 518, 518, 0, 0, 0, 0, 0, 0, 0, 0, 537, 526, 517, 563, 477, 500, 485, 514, 2, 0, 0, 0, 0, 0, 0, 0, 534, 511, 548, 529, 506, 505, 503, 554, 0, 0, 0, 0, 0, 0, 0, 0, 486, 532, 479, 491, 528, 545, 508, 496, 0, 0, 0, 0, 0, 0, 0, 0, 521, 483, 515, 510, 476, 514, 524, 510, 0, 0, 0, 0, 0, 0, 0, 0, 526, 502, 491, 485, 502, 525, 503, 508, 16, 0, 0, 0, 0, 0, 0, 0, 526, 481, 488, 532, 513, 528, 497, 502, 19, 0, 0, 0, 0, 0, 0, 0, 536, 521, 480, 508, 496, 471, 504, 520, 4, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4154, 4121, 4190, 4065, 4053, 4058, 4086, 4041] avg=4096.0000 max=4190 max_rank=2 min=4041 min_rank=7 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16238 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 3968, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 3968, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[497, 537, 472, 506, 516, 494, 499, 519, 498, 535, 506, 524, 480, 533, 531, 544, 496, 539, 465, 500, 500, 508, 532, 501, 496, 491, 486, 522, 506, 519, 532, 495, 532, 525, 515, 545, 499, 533, 564, 498, 524, 542, 493, 518, 512, 548, 473, 514, 523, 518, 476, 505, 527, 529, 555, 496, 503, 517, 502, 507, 475, 508, 514, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[516, 560, 512, 491, 512, 504, 464, 486, 518, 548, 531, 532, 525, 501, 545, 493, 533, 521, 517, 511, 508, 525, 486, 556, 529, 497, 526, 517, 497, 540, 521, 493, 492, 544, 509, 485, 475, 542, 511, 523, 497, 499, 544, 545, 523, 485, 521, 504, 498, 498, 506, 521, 487, 505, 498, 521, 527, 519, 526, 490, 508, 481, 489, 480] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[492, 487, 473, 541, 514, 526, 493, 512, 485, 475, 518, 501, 540, 529, 491, 524, 498, 533, 525, 488, 520, 495, 512, 527, 496, 494, 500, 505, 499, 529, 544, 518, 531, 526, 499, 498, 526, 519, 553, 537, 499, 529, 549, 500, 516, 542, 528, 530, 546, 508, 504, 489, 485, 519, 540, 497, 504, 484, 488, 489, 496, 539, 519, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[510, 478, 517, 528, 501, 524, 520, 540, 483, 534, 520, 519, 510, 553, 493, 489, 521, 520, 535, 528, 480, 539, 509, 509, 513, 510, 529, 521, 521, 527, 513, 483, 537, 486, 509, 515, 516, 514, 511, 536, 485, 481, 496, 494, 490, 481, 541, 529, 478, 525, 494, 552, 473, 465, 532, 469, 517, 490, 517, 551, 512, 517, 536, 542] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[507, 494, 533, 527, 525, 484, 510, 513, 520, 478, 532, 492, 497, 527, 489, 538, 512, 505, 500, 502, 494, 487, 518, 518, 490, 527, 523, 524, 561, 565, 518, 517, 527, 488, 524, 475, 522, 543, 487, 501, 524, 525, 484, 522, 528, 542, 502, 533, 543, 541, 491, 519, 520, 470, 528, 484, 539, 478, 526, 498, 493, 499, 471, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[500, 519, 515, 506, 509, 477, 534, 514, 572, 489, 515, 510, 526, 530, 502, 519, 468, 502, 523, 524, 511, 560, 541, 519, 533, 477, 499, 506, 467, 503, 495, 534, 514, 468, 520, 547, 531, 489, 517, 514, 484, 530, 484, 514, 478, 525, 510, 508, 512, 522, 526, 490, 481, 525, 549, 518, 529, 501, 507, 554, 508, 516, 486, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[508, 548, 519, 505, 547, 526, 525, 542, 481, 530, 556, 495, 499, 518, 514, 531, 538, 441, 521, 505, 551, 512, 522, 490, 456, 533, 509, 524, 530, 548, 513, 526, 489, 494, 478, 510, 498, 513, 509, 549, 508, 503, 515, 489, 504, 533, 529, 509, 526, 483, 520, 473, 550, 493, 509, 513, 504, 534, 493, 468, 500, 515, 519, 475] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[502, 511, 522, 522, 532, 506, 539, 480, 531, 555, 525, 528, 518, 520, 484, 459, 538, 531, 528, 555, 488, 494, 502, 474, 536, 533, 536, 529, 538, 534, 486, 528, 534, 508, 502, 459, 446, 530, 537, 506, 527, 492, 487, 488, 480, 532, 520, 505, 549, 476, 503, 486, 490, 526, 536, 500, 534, 513, 527, 524, 477, 521, 490, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[497, 537, 472, 506, 516, 494, 499, 519, 2, 0, 0, 0, 0, 0, 0, 0, 498, 535, 506, 524, 480, 506, 531, 544, 0, 0, 0, 0, 0, 0, 0, 0, 496, 539, 465, 500, 500, 508, 528, 501, 0, 0, 0, 0, 0, 0, 0, 0, 496, 491, 486, 522, 506, 495, 532, 495, 0, 0, 0, 0, 0, 0, 0, 0, 532, 525, 515, 545, 499, 533, 556, 498, 0, 0, 0, 0, 0, 0, 0, 0, 524, 542, 493, 518, 512, 548, 473, 514, 2, 0, 0, 0, 0, 0, 0, 0, 523, 518, 476, 505, 527, 529, 555, 496, 14, 8, 0, 0, 0, 0, 0, 0, 503, 517, 502, 507, 475, 508, 514, 499, 27, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4042, 4124, 4037, 4023, 4203, 4126, 4151, 4062] avg=4096.0000 max=4203 max_rank=4 min=4023 min_rank=3 imbalance_ratio=1.026123 max_load_reduction=28565 max_load_reduction_pct=0.871735 moved_to_echo=16542 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[516, 560, 512, 491, 512, 504, 464, 486, 2, 0, 0, 0, 0, 0, 0, 0, 518, 548, 531, 532, 525, 475, 545, 493, 0, 0, 0, 0, 0, 0, 0, 0, 533, 521, 517, 511, 508, 525, 482, 556, 0, 0, 0, 0, 0, 0, 0, 0, 529, 497, 526, 517, 497, 516, 521, 493, 0, 0, 0, 0, 0, 0, 0, 0, 492, 544, 509, 485, 475, 542, 504, 523, 0, 0, 0, 0, 0, 0, 0, 0, 497, 499, 544, 545, 523, 485, 521, 504, 2, 0, 0, 0, 0, 0, 0, 0, 498, 498, 506, 521, 487, 505, 498, 521, 14, 7, 0, 0, 0, 0, 0, 0, 527, 519, 526, 490, 508, 481, 489, 480, 26, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4047, 4167, 4153, 4096, 4074, 4120, 4055, 4056] avg=4096.0000 max=4167 max_rank=1 min=4047 min_rank=0 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16305 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[492, 487, 473, 541, 514, 526, 493, 512, 2, 0, 0, 0, 0, 0, 0, 0, 485, 475, 518, 501, 540, 502, 491, 524, 0, 0, 0, 0, 0, 0, 0, 0, 498, 533, 525, 488, 520, 495, 508, 527, 0, 0, 0, 0, 0, 0, 0, 0, 496, 494, 500, 505, 499, 505, 544, 518, 0, 0, 0, 0, 0, 0, 0, 0, 531, 526, 499, 498, 526, 519, 545, 537, 0, 0, 0, 0, 0, 0, 0, 0, 499, 529, 549, 500, 516, 542, 528, 530, 2, 0, 0, 0, 0, 0, 0, 0, 546, 508, 504, 489, 485, 519, 540, 497, 14, 8, 0, 0, 0, 0, 0, 0, 504, 484, 488, 489, 496, 539, 519, 495, 27, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4040, 4036, 4094, 4061, 4181, 4195, 4110, 4051] avg=4096.0000 max=4195 max_rank=5 min=4036 min_rank=1 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16537 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[510, 478, 517, 528, 501, 524, 520, 540, 2, 0, 0, 0, 0, 0, 0, 0, 483, 534, 520, 519, 510, 525, 493, 489, 0, 0, 0, 0, 0, 0, 0, 0, 521, 520, 535, 528, 480, 539, 505, 509, 0, 0, 0, 0, 0, 0, 0, 0, 513, 510, 529, 521, 521, 503, 513, 483, 0, 0, 0, 0, 0, 0, 0, 0, 537, 486, 509, 515, 516, 514, 504, 536, 0, 0, 0, 0, 0, 0, 0, 0, 485, 481, 496, 494, 490, 481, 541, 529, 2, 0, 0, 0, 0, 0, 0, 0, 478, 525, 494, 552, 473, 465, 532, 469, 14, 7, 0, 0, 0, 0, 0, 0, 517, 490, 517, 551, 512, 517, 536, 542, 28, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4073, 4137, 4093, 4117, 3999, 4009, 4220] avg=4096.0000 max=4220 max_rank=7 min=3999 min_rank=5 imbalance_ratio=1.030273 max_load_reduction=28548 max_load_reduction_pct=0.871216 moved_to_echo=16345 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[508, 548, 519, 505, 547, 526, 525, 542, 2, 0, 0, 0, 0, 0, 0, 0, 481, 530, 556, 495, 499, 491, 514, 531, 0, 0, 0, 0, 0, 0, 0, 0, 538, 441, 521, 505, 551, 512, 518, 490, 0, 0, 0, 0, 0, 0, 0, 0, 456, 533, 509, 524, 530, 524, 513, 526, 0, 0, 0, 0, 0, 0, 0, 0, 489, 494, 478, 510, 498, 513, 502, 549, 0, 0, 0, 0, 0, 0, 0, 0, 508, 503, 515, 489, 504, 533, 529, 509, 2, 0, 0, 0, 0, 0, 0, 0, 526, 483, 520, 473, 550, 493, 509, 513, 14, 7, 0, 0, 0, 0, 0, 0, 504, 534, 493, 468, 500, 515, 519, 475, 27, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4222, 4097, 4076, 4115, 4033, 4092, 4088, 4045] avg=4096.0000 max=4222 max_rank=0 min=4033 min_rank=4 imbalance_ratio=1.030762 max_load_reduction=28546 max_load_reduction_pct=0.871155 moved_to_echo=16258 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[507, 494, 533, 527, 525, 484, 510, 513, 2, 0, 0, 0, 0, 0, 0, 0, 520, 478, 532, 492, 497, 500, 489, 538, 0, 0, 0, 0, 0, 0, 0, 0, 512, 505, 500, 502, 494, 487, 514, 518, 0, 0, 0, 0, 0, 0, 0, 0, 490, 527, 523, 524, 561, 540, 518, 517, 0, 0, 0, 0, 0, 0, 0, 0, 527, 488, 524, 475, 522, 543, 480, 501, 0, 0, 0, 0, 0, 0, 0, 0, 524, 525, 484, 522, 528, 542, 502, 533, 2, 0, 0, 0, 0, 0, 0, 0, 543, 541, 491, 519, 520, 470, 528, 484, 15, 7, 0, 0, 0, 0, 0, 0, 539, 478, 526, 498, 493, 499, 471, 514, 27, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4046, 4032, 4200, 4060, 4162, 4118, 4055] avg=4096.0000 max=4200 max_rank=3 min=4032 min_rank=2 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16395 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[500, 519, 515, 506, 509, 477, 534, 514, 7, 5, 0, 0, 0, 0, 0, 0, 572, 489, 515, 510, 526, 499, 502, 519, 0, 0, 0, 0, 0, 0, 0, 0, 468, 502, 523, 524, 511, 560, 526, 519, 0, 0, 0, 0, 0, 0, 0, 0, 533, 477, 499, 506, 467, 473, 495, 534, 0, 0, 0, 0, 0, 0, 0, 0, 514, 468, 520, 547, 531, 489, 502, 514, 0, 0, 0, 0, 0, 0, 0, 0, 484, 530, 484, 514, 478, 525, 510, 508, 8, 0, 0, 0, 0, 0, 0, 0, 512, 522, 526, 490, 481, 525, 549, 518, 17, 10, 0, 0, 0, 0, 0, 0, 529, 501, 507, 554, 508, 516, 486, 512, 31, 13, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4132, 4133, 3984, 4085, 4041, 4150, 4157] avg=4096.0000 max=4157 max_rank=7 min=3984 min_rank=3 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16433 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[502, 511, 522, 522, 532, 506, 539, 480, 2, 0, 0, 0, 0, 0, 0, 0, 531, 555, 525, 528, 518, 493, 484, 459, 0, 0, 0, 0, 0, 0, 0, 0, 538, 531, 528, 555, 488, 494, 498, 474, 0, 0, 0, 0, 0, 0, 0, 0, 536, 533, 536, 529, 538, 510, 486, 528, 0, 0, 0, 0, 0, 0, 0, 0, 534, 508, 502, 459, 446, 530, 530, 506, 0, 0, 0, 0, 0, 0, 0, 0, 527, 492, 487, 488, 480, 532, 520, 505, 2, 0, 0, 0, 0, 0, 0, 0, 549, 476, 503, 486, 490, 526, 536, 500, 14, 7, 0, 0, 0, 0, 0, 0, 534, 513, 527, 524, 477, 521, 490, 499, 27, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4093, 4106, 4196, 4015, 4033, 4087, 4122] avg=4096.0000 max=4196 max_rank=3 min=4015 min_rank=4 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16257 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4352, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[485, 501, 514, 526, 524, 502, 502, 513, 522, 534, 492, 513, 535, 514, 519, 478, 512, 518, 529, 502, 541, 503, 517, 518, 512, 563, 469, 525, 515, 535, 492, 493, 503, 522, 476, 541, 519, 552, 489, 483, 503, 527, 510, 493, 514, 526, 483, 522, 533, 491, 483, 556, 510, 550, 522, 514, 477, 508, 487, 506, 502, 528, 487, 533] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[536, 479, 504, 539, 543, 504, 525, 513, 520, 519, 525, 525, 500, 505, 513, 502, 525, 528, 528, 544, 512, 539, 553, 495, 534, 468, 493, 522, 473, 501, 527, 541, 511, 513, 499, 500, 504, 523, 488, 518, 513, 490, 527, 487, 533, 511, 501, 515, 532, 525, 526, 531, 476, 504, 513, 490, 471, 509, 456, 502, 514, 466, 554, 531] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[519, 485, 510, 529, 506, 526, 508, 527, 484, 519, 527, 528, 530, 494, 537, 510, 508, 493, 542, 523, 508, 501, 522, 520, 484, 509, 483, 510, 510, 532, 494, 516, 510, 494, 563, 474, 516, 558, 501, 478, 494, 534, 499, 508, 537, 510, 491, 531, 530, 531, 480, 559, 483, 520, 501, 519, 524, 519, 501, 528, 481, 502, 507, 491] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[513, 520, 517, 516, 547, 538, 506, 502, 540, 514, 556, 530, 506, 505, 523, 469, 490, 507, 495, 497, 531, 505, 489, 521, 501, 481, 544, 517, 506, 522, 516, 503, 516, 551, 510, 478, 490, 486, 501, 514, 498, 507, 542, 509, 516, 536, 504, 516, 533, 525, 499, 512, 491, 465, 496, 500, 504, 502, 523, 525, 542, 497, 536, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[492, 548, 547, 519, 535, 512, 553, 461, 471, 533, 561, 528, 501, 540, 496, 504, 558, 548, 500, 490, 516, 522, 491, 548, 506, 512, 493, 491, 505, 564, 465, 553, 472, 488, 505, 495, 493, 513, 500, 569, 513, 530, 517, 516, 499, 510, 534, 532, 500, 532, 476, 490, 520, 490, 524, 498, 521, 501, 514, 487, 469, 520, 469, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[524, 527, 514, 507, 570, 504, 514, 514, 524, 496, 500, 531, 520, 500, 471, 496, 501, 490, 517, 478, 526, 528, 528, 504, 511, 487, 489, 548, 521, 535, 497, 530, 531, 528, 538, 497, 512, 490, 507, 458, 507, 526, 514, 501, 462, 478, 548, 526, 525, 483, 494, 535, 528, 522, 537, 493, 514, 501, 536, 531, 489, 504, 525, 526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[508, 505, 539, 497, 531, 526, 507, 445, 516, 541, 545, 526, 479, 494, 511, 538, 491, 515, 546, 517, 485, 535, 504, 519, 520, 531, 531, 531, 521, 513, 517, 510, 519, 523, 490, 500, 485, 526, 518, 502, 530, 513, 492, 519, 527, 489, 522, 493, 512, 518, 485, 484, 480, 461, 549, 482, 491, 513, 523, 542, 532, 503, 530, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[506, 523, 495, 518, 500, 484, 485, 520, 525, 520, 510, 532, 542, 510, 488, 557, 504, 529, 526, 487, 510, 506, 477, 530, 508, 511, 517, 505, 520, 508, 516, 497, 535, 519, 518, 520, 484, 509, 507, 502, 514, 541, 486, 491, 509, 517, 544, 528, 513, 512, 513, 506, 541, 492, 510, 517, 512, 479, 519, 519, 515, 501, 488, 541] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[485, 501, 514, 526, 508, 502, 502, 513, 0, 0, 0, 0, 0, 0, 0, 0, 522, 534, 466, 513, 535, 514, 519, 478, 0, 0, 0, 0, 0, 0, 0, 0, 512, 518, 508, 502, 541, 503, 517, 518, 0, 0, 0, 0, 0, 0, 0, 0, 512, 563, 469, 525, 515, 535, 492, 493, 1, 0, 0, 0, 0, 0, 0, 0, 503, 522, 476, 541, 519, 552, 489, 483, 26, 1, 0, 0, 0, 0, 0, 0, 503, 522, 510, 493, 514, 526, 483, 522, 0, 0, 0, 0, 0, 0, 0, 0, 533, 491, 483, 556, 510, 550, 522, 514, 13, 4, 0, 0, 0, 0, 0, 0, 477, 508, 487, 506, 502, 528, 487, 533, 20, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4051, 4081, 4119, 4105, 4112, 4073, 4176, 4051] avg=4096.0000 max=4176 max_rank=6 min=4051 min_rank=0 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16412 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[519, 485, 510, 529, 490, 526, 508, 527, 0, 0, 0, 0, 0, 0, 0, 0, 484, 519, 499, 528, 530, 494, 537, 510, 0, 0, 0, 0, 0, 0, 0, 0, 508, 493, 520, 523, 508, 501, 522, 520, 0, 0, 0, 0, 0, 0, 0, 0, 484, 509, 483, 510, 510, 532, 494, 516, 1, 0, 0, 0, 0, 0, 0, 0, 510, 494, 563, 474, 516, 558, 501, 478, 28, 1, 0, 0, 0, 0, 0, 0, 494, 529, 499, 508, 537, 510, 491, 531, 0, 0, 0, 0, 0, 0, 0, 0, 530, 531, 480, 559, 483, 520, 501, 519, 13, 4, 0, 0, 0, 0, 0, 0, 524, 519, 501, 528, 481, 502, 507, 491, 21, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4101, 4095, 4039, 4123, 4099, 4140, 4077] avg=4096.0000 max=4140 max_rank=6 min=4039 min_rank=3 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16439 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[513, 520, 517, 516, 530, 538, 506, 502, 0, 0, 0, 0, 0, 0, 0, 0, 540, 514, 527, 530, 506, 505, 523, 469, 0, 0, 0, 0, 0, 0, 0, 0, 490, 507, 476, 497, 531, 505, 489, 521, 0, 0, 0, 0, 0, 0, 0, 0, 501, 481, 544, 517, 506, 522, 516, 503, 1, 0, 0, 0, 0, 0, 0, 0, 516, 551, 510, 478, 490, 486, 501, 514, 29, 0, 0, 0, 0, 0, 0, 0, 498, 502, 542, 509, 516, 536, 504, 516, 0, 0, 0, 0, 0, 0, 0, 0, 533, 525, 499, 512, 491, 465, 496, 500, 14, 4, 0, 0, 0, 0, 0, 0, 504, 502, 523, 525, 542, 497, 536, 517, 19, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4142, 4114, 4016, 4091, 4075, 4123, 4039, 4168] avg=4096.0000 max=4168 max_rank=7 min=4016 min_rank=2 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16405 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[508, 505, 539, 497, 514, 526, 507, 445, 0, 0, 0, 0, 0, 0, 0, 0, 516, 541, 516, 526, 479, 494, 511, 538, 0, 0, 0, 0, 0, 0, 0, 0, 491, 515, 524, 517, 485, 535, 504, 519, 0, 0, 0, 0, 0, 0, 0, 0, 520, 531, 531, 531, 521, 513, 517, 510, 1, 0, 0, 0, 0, 0, 0, 0, 519, 523, 490, 500, 485, 526, 518, 502, 29, 1, 0, 0, 0, 0, 0, 0, 530, 508, 492, 519, 527, 489, 522, 493, 0, 0, 0, 0, 0, 0, 0, 0, 512, 518, 485, 484, 480, 461, 549, 482, 14, 4, 0, 0, 0, 0, 0, 0, 491, 513, 523, 542, 532, 503, 530, 521, 21, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4041, 4121, 4090, 4175, 4093, 4080, 3989, 4179] avg=4096.0000 max=4179 max_rank=7 min=3989 min_rank=6 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16341 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[536, 479, 504, 539, 526, 504, 525, 513, 0, 0, 0, 0, 0, 0, 0, 0, 520, 519, 497, 525, 500, 505, 513, 502, 0, 0, 0, 0, 0, 0, 0, 0, 525, 528, 507, 544, 512, 539, 553, 495, 0, 0, 0, 0, 0, 0, 0, 0, 534, 468, 493, 522, 473, 501, 527, 541, 1, 0, 0, 0, 0, 0, 0, 0, 511, 513, 499, 500, 504, 523, 488, 518, 28, 1, 0, 0, 0, 0, 0, 0, 513, 486, 527, 487, 533, 511, 501, 515, 0, 0, 0, 0, 0, 0, 0, 0, 532, 525, 526, 531, 476, 504, 513, 490, 14, 3, 0, 0, 0, 0, 0, 0, 471, 509, 456, 502, 514, 466, 554, 531, 20, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4081, 4203, 4060, 4085, 4073, 4114, 4026] avg=4096.0000 max=4203 max_rank=2 min=4026 min_rank=7 imbalance_ratio=1.026123 max_load_reduction=28565 max_load_reduction_pct=0.871735 moved_to_echo=16298 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[492, 548, 547, 519, 511, 512, 553, 461, 0, 0, 0, 0, 0, 0, 0, 0, 471, 533, 528, 528, 501, 540, 496, 504, 0, 0, 0, 0, 0, 0, 0, 0, 558, 548, 472, 490, 516, 522, 491, 548, 0, 0, 0, 0, 0, 0, 0, 0, 506, 512, 493, 491, 505, 564, 465, 553, 7, 0, 0, 0, 0, 0, 0, 0, 472, 488, 505, 495, 493, 513, 500, 569, 33, 3, 0, 0, 0, 0, 0, 0, 513, 517, 517, 516, 499, 510, 534, 532, 0, 0, 0, 0, 0, 0, 0, 0, 500, 532, 476, 490, 520, 490, 524, 498, 17, 6, 0, 0, 0, 0, 0, 0, 521, 501, 514, 487, 469, 520, 469, 508, 25, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4143, 4101, 4145, 4096, 4071, 4138, 4053, 4021] avg=4096.0000 max=4145 max_rank=2 min=4021 min_rank=7 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16283 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[524, 527, 514, 507, 552, 504, 514, 514, 0, 0, 0, 0, 0, 0, 0, 0, 524, 496, 474, 531, 520, 500, 471, 496, 0, 0, 0, 0, 0, 0, 0, 0, 501, 490, 497, 478, 526, 528, 528, 504, 0, 0, 0, 0, 0, 0, 0, 0, 511, 487, 489, 548, 521, 535, 497, 530, 1, 0, 0, 0, 0, 0, 0, 0, 531, 528, 538, 497, 512, 490, 507, 458, 26, 0, 0, 0, 0, 0, 0, 0, 507, 521, 514, 501, 462, 478, 548, 526, 0, 0, 0, 0, 0, 0, 0, 0, 525, 483, 494, 535, 528, 522, 537, 493, 15, 4, 0, 0, 0, 0, 0, 0, 514, 501, 536, 531, 489, 504, 525, 526, 20, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4156, 4012, 4052, 4119, 4087, 4057, 4136, 4149] avg=4096.0000 max=4156 max_rank=0 min=4012 min_rank=1 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[506, 523, 495, 518, 484, 484, 485, 520, 0, 0, 0, 0, 0, 0, 0, 0, 525, 520, 483, 532, 542, 510, 488, 557, 0, 0, 0, 0, 0, 0, 0, 0, 504, 529, 505, 487, 510, 506, 477, 530, 0, 0, 0, 0, 0, 0, 0, 0, 508, 511, 517, 505, 520, 508, 516, 497, 1, 0, 0, 0, 0, 0, 0, 0, 535, 519, 518, 520, 484, 509, 507, 502, 27, 1, 0, 0, 0, 0, 0, 0, 514, 536, 486, 491, 509, 517, 544, 528, 0, 0, 0, 0, 0, 0, 0, 0, 513, 512, 513, 506, 541, 492, 510, 517, 13, 4, 0, 0, 0, 0, 0, 0, 512, 479, 519, 519, 515, 501, 488, 541, 20, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4015, 4157, 4048, 4083, 4122, 4125, 4121, 4097] avg=4096.0000 max=4157 max_rank=1 min=4015 min_rank=0 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16465 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 3968, 4224, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[485, 535, 538, 502, 479, 505, 522, 562, 536, 509, 504, 523, 520, 515, 538, 504, 481, 506, 543, 516, 495, 508, 501, 465, 516, 502, 494, 544, 507, 519, 484, 557, 521, 493, 497, 514, 488, 508, 531, 501, 537, 513, 497, 519, 510, 536, 536, 520, 509, 518, 544, 446, 500, 519, 509, 521, 517, 519, 495, 520, 511, 498, 508, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[532, 519, 502, 481, 503, 502, 517, 478, 537, 511, 518, 531, 507, 525, 507, 494, 524, 471, 544, 557, 521, 541, 499, 515, 511, 470, 490, 503, 516, 542, 559, 507, 497, 505, 555, 507, 516, 490, 551, 502, 476, 532, 491, 541, 512, 502, 539, 548, 498, 518, 525, 497, 505, 501, 490, 488, 483, 491, 521, 509, 511, 512, 504, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[507, 553, 505, 528, 485, 508, 502, 514, 522, 472, 484, 484, 508, 473, 516, 558, 479, 533, 482, 498, 514, 527, 537, 487, 505, 510, 511, 506, 544, 506, 519, 549, 504, 522, 496, 515, 538, 516, 489, 500, 491, 517, 522, 526, 489, 507, 497, 513, 491, 510, 522, 517, 499, 503, 572, 551, 504, 527, 525, 526, 487, 500, 516, 550] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[519, 511, 566, 481, 524, 458, 492, 488, 519, 545, 481, 531, 508, 521, 497, 525, 530, 536, 509, 505, 483, 527, 533, 516, 527, 522, 495, 504, 538, 526, 503, 484, 527, 492, 475, 513, 455, 531, 517, 494, 514, 510, 502, 545, 499, 467, 500, 550, 509, 493, 488, 497, 534, 523, 539, 527, 550, 500, 515, 545, 504, 502, 512, 535] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[513, 470, 494, 506, 491, 506, 524, 523, 528, 518, 530, 522, 533, 500, 515, 470, 502, 540, 507, 540, 536, 523, 520, 502, 482, 502, 544, 510, 506, 540, 547, 486, 515, 508, 494, 524, 489, 516, 543, 500, 552, 523, 488, 499, 505, 483, 536, 536, 493, 496, 484, 510, 487, 520, 495, 512, 494, 504, 507, 522, 522, 518, 517, 546] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[528, 472, 520, 508, 510, 493, 505, 501, 530, 503, 512, 503, 528, 504, 524, 561, 503, 507, 527, 531, 499, 487, 517, 529, 509, 526, 502, 503, 516, 507, 497, 522, 520, 504, 507, 522, 521, 501, 499, 490, 538, 508, 526, 494, 483, 508, 457, 536, 532, 511, 537, 549, 497, 509, 449, 496, 515, 529, 510, 538, 507, 511, 538, 542] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[505, 487, 502, 512, 520, 530, 488, 543, 463, 491, 522, 503, 521, 529, 529, 514, 521, 491, 471, 545, 529, 519, 500, 543, 532, 523, 475, 537, 525, 499, 469, 553, 518, 507, 534, 497, 525, 514, 545, 503, 505, 522, 493, 500, 521, 501, 496, 476, 535, 542, 534, 518, 505, 522, 498, 523, 492, 525, 519, 491, 491, 524, 499, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[485, 491, 487, 509, 509, 470, 520, 516, 532, 487, 542, 507, 546, 500, 553, 502, 479, 527, 501, 471, 493, 521, 495, 509, 516, 538, 556, 499, 512, 487, 473, 508, 481, 485, 493, 504, 516, 476, 546, 477, 538, 508, 537, 506, 500, 525, 560, 517, 492, 519, 551, 505, 521, 518, 546, 527, 485, 514, 521, 506, 504, 541, 553, 555] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[532, 519, 502, 481, 503, 502, 517, 478, 22, 17, 0, 0, 0, 0, 0, 0, 537, 511, 518, 531, 507, 525, 486, 494, 0, 0, 0, 0, 0, 0, 0, 0, 524, 471, 544, 557, 521, 541, 499, 515, 0, 0, 0, 0, 0, 0, 0, 0, 511, 470, 490, 503, 516, 542, 559, 495, 0, 0, 0, 0, 0, 0, 0, 0, 497, 505, 555, 507, 516, 490, 551, 502, 12, 8, 4, 3, 0, 0, 0, 0, 476, 532, 491, 541, 512, 502, 539, 540, 0, 0, 0, 0, 0, 0, 0, 0, 498, 518, 522, 497, 505, 501, 490, 488, 0, 0, 0, 0, 0, 0, 0, 0, 483, 491, 521, 509, 511, 512, 504, 495, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4109, 4172, 4086, 4150, 4133, 4019, 4026] avg=4096.0000 max=4172 max_rank=2 min=4019 min_rank=6 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16328 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[485, 535, 538, 502, 479, 505, 522, 562, 21, 18, 0, 0, 0, 0, 0, 0, 536, 509, 504, 523, 520, 515, 516, 504, 0, 0, 0, 0, 0, 0, 0, 0, 481, 506, 543, 516, 495, 508, 501, 465, 0, 0, 0, 0, 0, 0, 0, 0, 516, 502, 494, 544, 507, 519, 484, 544, 0, 0, 0, 0, 0, 0, 0, 0, 521, 493, 497, 514, 488, 508, 531, 501, 13, 8, 4, 3, 0, 0, 0, 0, 537, 513, 497, 519, 510, 536, 536, 512, 0, 0, 0, 0, 0, 0, 0, 0, 509, 518, 541, 446, 500, 519, 509, 521, 0, 0, 0, 0, 0, 0, 0, 0, 517, 519, 495, 520, 511, 498, 508, 477, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4167, 4127, 4015, 4110, 4081, 4160, 4063, 4045] avg=4096.0000 max=4167 max_rank=0 min=4015 min_rank=2 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16349 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[528, 472, 520, 508, 510, 493, 505, 501, 23, 17, 0, 0, 0, 0, 0, 0, 530, 503, 512, 503, 528, 504, 503, 561, 0, 0, 0, 0, 0, 0, 0, 0, 503, 507, 527, 531, 499, 487, 517, 529, 0, 0, 0, 0, 0, 0, 0, 0, 509, 526, 502, 503, 516, 507, 497, 510, 0, 0, 0, 0, 0, 0, 0, 0, 520, 504, 507, 522, 521, 501, 499, 490, 12, 8, 4, 3, 0, 0, 0, 0, 538, 508, 526, 494, 483, 508, 457, 528, 0, 0, 0, 0, 0, 0, 0, 0, 532, 511, 534, 549, 497, 509, 449, 496, 0, 0, 0, 0, 0, 0, 0, 0, 515, 529, 510, 538, 507, 511, 538, 519, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4144, 4100, 4070, 4091, 4042, 4077, 4167] avg=4096.0000 max=4167 max_rank=7 min=4042 min_rank=5 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16377 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[507, 553, 505, 528, 485, 508, 502, 514, 29, 21, 0, 0, 0, 0, 0, 0, 522, 472, 484, 484, 508, 473, 487, 558, 0, 0, 0, 0, 0, 0, 0, 0, 479, 533, 482, 498, 514, 527, 537, 487, 0, 0, 0, 0, 0, 0, 0, 0, 505, 510, 511, 506, 544, 506, 519, 531, 0, 0, 0, 0, 0, 0, 0, 0, 504, 522, 496, 515, 538, 516, 489, 500, 18, 12, 8, 7, 0, 0, 0, 0, 491, 517, 522, 526, 489, 507, 497, 501, 0, 0, 0, 0, 0, 0, 0, 0, 491, 510, 515, 517, 499, 503, 572, 551, 0, 0, 0, 0, 0, 0, 0, 0, 504, 527, 525, 526, 487, 500, 516, 521, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4152, 3988, 4057, 4132, 4125, 4050, 4158, 4106] avg=4096.0000 max=4158 max_rank=6 min=3988 min_rank=1 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16439 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[505, 487, 502, 512, 520, 530, 488, 543, 21, 17, 0, 0, 0, 0, 0, 0, 463, 491, 522, 503, 521, 529, 508, 514, 0, 0, 0, 0, 0, 0, 0, 0, 521, 491, 471, 545, 529, 519, 500, 543, 0, 0, 0, 0, 0, 0, 0, 0, 532, 523, 475, 537, 525, 499, 469, 540, 0, 0, 0, 0, 0, 0, 0, 0, 518, 507, 534, 497, 525, 514, 545, 503, 13, 7, 4, 3, 0, 0, 0, 0, 505, 522, 493, 500, 521, 501, 496, 469, 0, 0, 0, 0, 0, 0, 0, 0, 535, 542, 531, 518, 505, 522, 498, 523, 0, 0, 0, 0, 0, 0, 0, 0, 492, 525, 519, 491, 491, 524, 499, 481, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4051, 4119, 4100, 4170, 4007, 4174, 4022] avg=4096.0000 max=4174 max_rank=6 min=4007 min_rank=5 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16373 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[519, 511, 566, 481, 524, 458, 492, 488, 23, 16, 0, 0, 0, 0, 0, 0, 519, 545, 481, 531, 508, 521, 477, 525, 0, 0, 0, 0, 0, 0, 0, 0, 530, 536, 509, 505, 483, 527, 533, 516, 0, 0, 0, 0, 0, 0, 0, 0, 527, 522, 495, 504, 538, 526, 503, 473, 0, 0, 0, 0, 0, 0, 0, 0, 527, 492, 475, 513, 455, 531, 517, 494, 11, 8, 4, 3, 0, 0, 0, 0, 514, 510, 502, 545, 499, 467, 500, 542, 0, 0, 0, 0, 0, 0, 0, 0, 509, 493, 485, 497, 534, 523, 539, 527, 0, 0, 0, 0, 0, 0, 0, 0, 550, 500, 515, 545, 504, 502, 512, 512, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4107, 4139, 4088, 4030, 4079, 4107, 4140] avg=4096.0000 max=4140 max_rank=7 min=4030 min_rank=4 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16356 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[513, 470, 494, 506, 491, 506, 524, 523, 23, 17, 0, 0, 0, 0, 0, 0, 528, 518, 530, 522, 533, 500, 494, 470, 0, 0, 0, 0, 0, 0, 0, 0, 502, 540, 507, 540, 536, 523, 520, 502, 0, 0, 0, 0, 0, 0, 0, 0, 482, 502, 544, 510, 506, 540, 547, 474, 0, 0, 0, 0, 0, 0, 0, 0, 515, 508, 494, 524, 489, 516, 543, 500, 12, 8, 4, 3, 0, 0, 0, 0, 552, 523, 488, 499, 505, 483, 536, 528, 0, 0, 0, 0, 0, 0, 0, 0, 493, 496, 481, 510, 487, 520, 495, 512, 0, 0, 0, 0, 0, 0, 0, 0, 494, 504, 507, 522, 522, 518, 517, 523, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4095, 4170, 4105, 4116, 4114, 3994, 4107] avg=4096.0000 max=4170 max_rank=2 min=3994 min_rank=6 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16331 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[485, 491, 487, 509, 509, 470, 520, 516, 24, 18, 0, 0, 0, 0, 0, 0, 532, 487, 542, 507, 546, 500, 531, 502, 0, 0, 0, 0, 0, 0, 0, 0, 479, 527, 501, 471, 493, 521, 495, 509, 0, 0, 0, 0, 0, 0, 0, 0, 516, 538, 556, 499, 512, 487, 473, 496, 0, 0, 0, 0, 0, 0, 0, 0, 481, 485, 493, 504, 516, 476, 546, 477, 12, 8, 4, 3, 0, 0, 0, 0, 538, 508, 537, 506, 500, 525, 560, 509, 0, 0, 0, 0, 0, 0, 0, 0, 492, 519, 548, 505, 521, 518, 546, 527, 0, 0, 0, 0, 0, 0, 0, 0, 485, 514, 521, 506, 504, 541, 553, 531, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4029, 4147, 3996, 4077, 4005, 4183, 4176, 4155] avg=4096.0000 max=4183 max_rank=5 min=3996 min_rank=2 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16519 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 256, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4224, 3968, 128, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True + [2026-09-21 08:26:50] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 272.1 | throughput per GPU (TFLOP/s/GPU): 118.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.772961E+00 | loss scale: 1.0 | grad norm: 3.879 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[502, 512, 523, 557, 460, 481, 478, 495, 511, 536, 498, 472, 504, 566, 484, 501, 477, 549, 522, 539, 483, 502, 504, 473, 509, 540, 488, 520, 517, 486, 555, 492, 507, 520, 468, 511, 527, 528, 524, 477, 515, 515, 555, 553, 552, 520, 543, 527, 515, 496, 516, 521, 511, 517, 509, 485, 491, 527, 485, 520, 550, 539, 531, 477] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[524, 542, 546, 505, 516, 511, 498, 517, 504, 516, 485, 507, 537, 510, 502, 500, 463, 532, 519, 458, 484, 479, 512, 505, 504, 496, 529, 507, 508, 501, 491, 512, 554, 535, 518, 529, 529, 505, 484, 510, 480, 530, 518, 542, 478, 496, 530, 564, 491, 536, 500, 500, 498, 501, 523, 517, 511, 559, 504, 497, 505, 550, 498, 556] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[505, 524, 511, 478, 544, 504, 536, 529, 512, 496, 479, 520, 535, 521, 491, 469, 483, 560, 488, 523, 493, 496, 501, 511, 515, 508, 534, 529, 500, 502, 505, 552, 552, 491, 519, 483, 510, 516, 492, 485, 496, 486, 486, 520, 516, 569, 528, 505, 502, 508, 533, 506, 500, 521, 536, 526, 543, 479, 517, 530, 538, 489, 518, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[524, 508, 533, 518, 521, 496, 509, 499, 504, 533, 523, 518, 482, 508, 498, 556, 494, 500, 494, 528, 532, 518, 520, 522, 530, 485, 485, 525, 529, 481, 554, 540, 504, 552, 530, 535, 502, 486, 496, 511, 519, 496, 465, 488, 476, 502, 527, 496, 518, 508, 537, 525, 543, 534, 505, 559, 509, 506, 505, 493, 485, 504, 505, 480] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[507, 519, 495, 500, 518, 492, 547, 477, 577, 520, 524, 498, 476, 487, 533, 542, 509, 534, 480, 510, 497, 532, 550, 515, 547, 522, 496, 503, 485, 515, 531, 469, 533, 522, 537, 513, 509, 483, 513, 443, 539, 528, 509, 521, 471, 543, 505, 558, 494, 505, 543, 486, 497, 514, 498, 489, 497, 541, 522, 516, 531, 509, 512, 480] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[517, 504, 514, 507, 541, 508, 518, 518, 541, 484, 479, 504, 492, 506, 523, 516, 498, 518, 522, 529, 538, 491, 532, 495, 537, 492, 515, 514, 483, 471, 492, 530, 517, 512, 541, 505, 520, 524, 555, 520, 510, 497, 519, 501, 515, 478, 483, 503, 487, 528, 531, 500, 470, 524, 509, 519, 524, 509, 491, 510, 527, 543, 534, 533] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[502, 476, 546, 525, 546, 517, 494, 468, 532, 501, 453, 512, 534, 542, 487, 518, 525, 482, 508, 541, 539, 548, 512, 511, 548, 500, 523, 472, 496, 517, 546, 505, 535, 548, 509, 492, 514, 522, 509, 477, 512, 517, 514, 527, 487, 531, 521, 519, 539, 475, 526, 533, 509, 508, 469, 527, 467, 549, 489, 515, 493, 508, 524, 477] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[501, 496, 529, 540, 537, 518, 501, 506, 494, 519, 514, 515, 490, 536, 486, 490, 498, 541, 527, 509, 530, 520, 470, 459, 513, 513, 520, 527, 529, 498, 504, 487, 546, 504, 534, 489, 510, 501, 550, 547, 517, 525, 508, 482, 480, 537, 564, 509, 503, 497, 547, 521, 501, 522, 457, 557, 504, 496, 477, 524, 526, 524, 495, 497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[505, 524, 509, 478, 544, 504, 536, 529, 0, 0, 0, 0, 0, 0, 0, 0, 512, 496, 479, 520, 535, 521, 491, 469, 20, 1, 0, 0, 0, 0, 0, 0, 483, 560, 488, 523, 493, 496, 501, 511, 16, 0, 0, 0, 0, 0, 0, 0, 515, 508, 534, 529, 500, 502, 505, 552, 2, 1, 0, 0, 0, 0, 0, 0, 532, 491, 519, 483, 510, 516, 492, 485, 0, 0, 0, 0, 0, 0, 0, 0, 496, 486, 486, 520, 516, 569, 510, 505, 0, 0, 0, 0, 0, 0, 0, 0, 502, 508, 532, 506, 500, 521, 536, 526, 0, 0, 0, 0, 0, 0, 0, 0, 543, 479, 517, 530, 538, 489, 518, 514, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4129, 4044, 4071, 4148, 4028, 4088, 4131, 4129] avg=4096.0000 max=4148 max_rank=3 min=4028 min_rank=4 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16376 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[502, 512, 521, 557, 460, 481, 478, 495, 0, 0, 0, 0, 0, 0, 0, 0, 511, 536, 498, 472, 504, 566, 484, 501, 18, 1, 0, 0, 0, 0, 0, 0, 477, 549, 522, 539, 483, 502, 504, 473, 17, 0, 0, 0, 0, 0, 0, 0, 509, 540, 488, 520, 517, 486, 555, 492, 2, 1, 0, 0, 0, 0, 0, 0, 489, 520, 468, 511, 527, 528, 524, 477, 0, 0, 0, 0, 0, 0, 0, 0, 515, 515, 555, 553, 552, 520, 524, 527, 0, 0, 0, 0, 0, 0, 0, 0, 515, 496, 515, 521, 511, 517, 509, 485, 0, 0, 0, 0, 0, 0, 0, 0, 491, 527, 485, 520, 550, 539, 531, 477, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4006, 4091, 4066, 4110, 4044, 4261, 4069, 4121] avg=4096.0000 max=4261 max_rank=5 min=4006 min_rank=0 imbalance_ratio=1.040283 max_load_reduction=28507 max_load_reduction_pct=0.869965 moved_to_echo=16495 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[524, 542, 544, 505, 516, 511, 498, 517, 0, 0, 0, 0, 0, 0, 0, 0, 504, 516, 485, 507, 537, 510, 502, 500, 20, 1, 0, 0, 0, 0, 0, 0, 463, 532, 519, 458, 484, 479, 512, 505, 16, 0, 0, 0, 0, 0, 0, 0, 504, 496, 529, 507, 508, 501, 491, 512, 2, 1, 0, 0, 0, 0, 0, 0, 534, 535, 518, 529, 529, 505, 484, 510, 0, 0, 0, 0, 0, 0, 0, 0, 480, 530, 518, 542, 478, 496, 512, 564, 0, 0, 0, 0, 0, 0, 0, 0, 491, 536, 499, 500, 498, 501, 523, 517, 0, 0, 0, 0, 0, 0, 0, 0, 511, 559, 504, 497, 505, 550, 498, 556, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4157, 4082, 3968, 4051, 4144, 4120, 4065, 4181] avg=4096.0000 max=4181 max_rank=7 min=3968 min_rank=2 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16510 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[524, 508, 531, 518, 521, 496, 509, 499, 0, 0, 0, 0, 0, 0, 0, 0, 504, 533, 523, 518, 482, 508, 498, 556, 18, 1, 0, 0, 0, 0, 0, 0, 494, 500, 494, 528, 532, 518, 520, 522, 16, 0, 0, 0, 0, 0, 0, 0, 530, 485, 485, 525, 529, 481, 554, 540, 2, 1, 0, 0, 0, 0, 0, 0, 486, 552, 530, 535, 502, 486, 496, 511, 0, 0, 0, 0, 0, 0, 0, 0, 519, 496, 465, 488, 476, 502, 509, 496, 0, 0, 0, 0, 0, 0, 0, 0, 518, 508, 536, 525, 543, 534, 505, 559, 0, 0, 0, 0, 0, 0, 0, 0, 509, 506, 505, 493, 485, 504, 505, 480, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4141, 4124, 4132, 4098, 3951, 4228, 3988] avg=4096.0000 max=4228 max_rank=6 min=3951 min_rank=5 imbalance_ratio=1.032227 max_load_reduction=28540 max_load_reduction_pct=0.870972 moved_to_echo=16265 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[507, 519, 493, 500, 518, 492, 547, 477, 0, 0, 0, 0, 0, 0, 0, 0, 577, 520, 524, 498, 476, 487, 533, 542, 19, 1, 0, 0, 0, 0, 0, 0, 509, 534, 480, 510, 497, 532, 550, 515, 16, 0, 0, 0, 0, 0, 0, 0, 547, 522, 496, 503, 485, 515, 531, 469, 2, 1, 0, 0, 0, 0, 0, 0, 514, 522, 537, 513, 509, 483, 513, 443, 0, 0, 0, 0, 0, 0, 0, 0, 539, 528, 509, 521, 471, 543, 487, 558, 0, 0, 0, 0, 0, 0, 0, 0, 494, 505, 542, 486, 497, 514, 498, 489, 0, 0, 0, 0, 0, 0, 0, 0, 497, 541, 522, 516, 531, 509, 512, 480, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4053, 4177, 4143, 4071, 4034, 4156, 4025, 4109] avg=4096.0000 max=4177 max_rank=1 min=4025 min_rank=6 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16324 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[517, 504, 506, 507, 541, 508, 518, 518, 0, 0, 0, 0, 0, 0, 0, 0, 541, 484, 479, 504, 492, 506, 523, 516, 22, 2, 0, 0, 0, 0, 0, 0, 498, 518, 522, 529, 538, 491, 532, 495, 20, 0, 0, 0, 0, 0, 0, 0, 537, 492, 515, 514, 483, 471, 492, 530, 8, 5, 5, 0, 0, 0, 0, 0, 495, 512, 541, 505, 520, 524, 555, 520, 0, 0, 0, 0, 0, 0, 0, 0, 510, 497, 519, 501, 515, 478, 456, 503, 0, 0, 0, 0, 0, 0, 0, 0, 487, 528, 524, 500, 470, 524, 509, 519, 0, 0, 0, 0, 0, 0, 0, 0, 524, 509, 491, 510, 527, 543, 534, 533, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4069, 4143, 4052, 4172, 3979, 4061, 4173] avg=4096.0000 max=4173 max_rank=7 min=3979 min_rank=5 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[502, 476, 544, 525, 546, 517, 494, 468, 0, 0, 0, 0, 0, 0, 0, 0, 532, 501, 453, 512, 534, 542, 487, 518, 19, 1, 0, 0, 0, 0, 0, 0, 525, 482, 508, 541, 539, 548, 512, 511, 16, 0, 0, 0, 0, 0, 0, 0, 548, 500, 523, 472, 496, 517, 546, 505, 2, 1, 0, 0, 0, 0, 0, 0, 516, 548, 509, 492, 514, 522, 509, 477, 0, 0, 0, 0, 0, 0, 0, 0, 512, 517, 514, 527, 487, 531, 503, 519, 0, 0, 0, 0, 0, 0, 0, 0, 539, 475, 525, 533, 509, 508, 469, 527, 0, 0, 0, 0, 0, 0, 0, 0, 467, 549, 489, 515, 493, 508, 524, 477, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4099, 4182, 4110, 4087, 4110, 4085, 4023] avg=4096.0000 max=4182 max_rank=2 min=4023 min_rank=7 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16305 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[501, 496, 527, 540, 537, 518, 501, 506, 0, 0, 0, 0, 0, 0, 0, 0, 494, 519, 514, 515, 490, 536, 486, 490, 20, 1, 0, 0, 0, 0, 0, 0, 498, 541, 527, 509, 530, 520, 470, 459, 17, 0, 0, 0, 0, 0, 0, 0, 513, 513, 520, 527, 529, 498, 504, 487, 2, 1, 0, 0, 0, 0, 0, 0, 526, 504, 534, 489, 510, 501, 550, 547, 0, 0, 0, 0, 0, 0, 0, 0, 517, 525, 508, 482, 480, 537, 545, 509, 0, 0, 0, 0, 0, 0, 0, 0, 503, 497, 546, 521, 501, 522, 457, 557, 0, 0, 0, 0, 0, 0, 0, 0, 504, 496, 477, 524, 526, 524, 495, 497, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4065, 4071, 4094, 4161, 4103, 4104, 4044] avg=4096.0000 max=4161 max_rank=4 min=4044 min_rank=7 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16412 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3968, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 3968, 4096, 4096, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[545, 480, 466, 527, 536, 539, 483, 514, 552, 495, 483, 524, 482, 548, 502, 533, 536, 498, 561, 555, 532, 544, 500, 494, 481, 538, 502, 506, 512, 523, 532, 495, 521, 502, 476, 515, 497, 485, 501, 506, 505, 494, 520, 497, 492, 486, 507, 498, 513, 519, 529, 501, 495, 533, 518, 550, 515, 529, 531, 519, 512, 481, 486, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[507, 493, 522, 496, 512, 500, 499, 526, 541, 487, 523, 493, 513, 532, 473, 525, 510, 511, 493, 499, 507, 546, 513, 524, 482, 500, 555, 530, 501, 486, 514, 526, 504, 522, 511, 535, 515, 531, 511, 543, 538, 487, 474, 526, 505, 508, 517, 471, 533, 517, 489, 504, 570, 467, 558, 496, 531, 509, 508, 533, 516, 518, 487, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[512, 493, 538, 504, 533, 499, 478, 522, 522, 480, 475, 499, 527, 482, 503, 515, 523, 486, 507, 502, 539, 513, 505, 489, 492, 504, 512, 524, 523, 510, 527, 480, 505, 526, 497, 518, 531, 529, 510, 521, 516, 520, 500, 486, 543, 535, 499, 500, 504, 505, 489, 512, 525, 539, 505, 537, 528, 520, 504, 529, 527, 542, 524, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[510, 535, 521, 520, 522, 494, 487, 461, 508, 496, 504, 529, 502, 529, 533, 531, 496, 516, 533, 507, 494, 532, 499, 512, 501, 485, 524, 514, 489, 533, 501, 549, 504, 557, 502, 470, 509, 525, 509, 574, 542, 505, 513, 512, 475, 507, 491, 490, 521, 521, 540, 524, 507, 505, 493, 511, 521, 511, 508, 496, 538, 501, 535, 484] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[535, 520, 496, 557, 513, 553, 543, 504, 482, 522, 511, 547, 534, 492, 501, 524, 534, 478, 512, 516, 494, 477, 523, 489, 472, 493, 469, 540, 504, 535, 521, 538, 462, 487, 549, 542, 536, 510, 501, 514, 457, 509, 500, 524, 528, 556, 475, 520, 524, 523, 547, 522, 501, 507, 525, 510, 498, 530, 483, 489, 468, 526, 488, 528] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[519, 488, 528, 514, 536, 519, 532, 508, 516, 502, 520, 490, 508, 500, 503, 511, 514, 522, 530, 523, 500, 539, 516, 503, 512, 503, 502, 545, 517, 550, 479, 520, 480, 522, 486, 526, 501, 524, 519, 505, 561, 519, 482, 493, 500, 503, 498, 512, 497, 515, 537, 498, 511, 538, 546, 514, 503, 484, 517, 513, 518, 501, 493, 483] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[497, 523, 561, 503, 541, 497, 562, 499, 510, 508, 511, 476, 527, 492, 562, 535, 515, 497, 510, 530, 512, 514, 501, 525, 492, 528, 518, 513, 521, 521, 493, 494, 521, 530, 534, 507, 527, 472, 493, 511, 507, 501, 517, 551, 516, 495, 508, 486, 479, 503, 530, 459, 527, 546, 522, 504, 497, 532, 466, 479, 524, 515, 509, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[544, 516, 529, 559, 462, 492, 528, 501, 488, 507, 530, 498, 500, 512, 523, 542, 503, 507, 521, 533, 490, 535, 548, 538, 522, 461, 490, 538, 495, 501, 502, 556, 504, 482, 474, 503, 520, 480, 526, 509, 520, 563, 513, 480, 513, 500, 544, 494, 523, 486, 521, 524, 489, 497, 535, 519, 513, 524, 482, 507, 493, 538, 491, 530] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[545, 480, 466, 506, 536, 539, 483, 514, 0, 0, 0, 0, 0, 0, 0, 0, 552, 495, 483, 524, 482, 548, 502, 533, 5, 0, 0, 0, 0, 0, 0, 0, 536, 498, 561, 555, 532, 525, 500, 494, 0, 0, 0, 0, 0, 0, 0, 0, 481, 538, 502, 506, 512, 523, 532, 495, 8, 0, 0, 0, 0, 0, 0, 0, 521, 502, 476, 515, 497, 485, 501, 506, 6, 0, 0, 0, 0, 0, 0, 0, 505, 494, 520, 497, 492, 486, 507, 498, 29, 2, 0, 0, 0, 0, 0, 0, 513, 519, 529, 501, 495, 533, 489, 550, 0, 0, 0, 0, 0, 0, 0, 0, 515, 529, 531, 519, 512, 481, 486, 517, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4124, 4201, 4097, 4009, 4030, 4129, 4109] avg=4096.0000 max=4201 max_rank=2 min=4009 min_rank=4 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16277 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[507, 493, 522, 476, 512, 500, 499, 526, 0, 0, 0, 0, 0, 0, 0, 0, 541, 487, 523, 493, 513, 532, 473, 525, 5, 0, 0, 0, 0, 0, 0, 0, 510, 511, 493, 499, 507, 527, 513, 524, 0, 0, 0, 0, 0, 0, 0, 0, 482, 500, 555, 530, 501, 486, 514, 526, 8, 0, 0, 0, 0, 0, 0, 0, 504, 522, 511, 535, 515, 531, 511, 543, 6, 0, 0, 0, 0, 0, 0, 0, 538, 487, 474, 526, 505, 508, 517, 471, 32, 2, 0, 0, 0, 0, 0, 0, 533, 517, 489, 504, 570, 467, 526, 496, 0, 0, 0, 0, 0, 0, 0, 0, 531, 509, 508, 533, 516, 518, 487, 495, 18, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4035, 4092, 4084, 4102, 4178, 4060, 4102, 4115] avg=4096.0000 max=4178 max_rank=4 min=4035 min_rank=0 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16455 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[497, 523, 561, 483, 541, 497, 562, 499, 0, 0, 0, 0, 0, 0, 0, 0, 510, 508, 511, 476, 527, 492, 562, 535, 5, 0, 0, 0, 0, 0, 0, 0, 515, 497, 510, 530, 512, 497, 501, 525, 0, 0, 0, 0, 0, 0, 0, 0, 492, 528, 518, 513, 521, 521, 493, 494, 7, 0, 0, 0, 0, 0, 0, 0, 521, 530, 534, 507, 527, 472, 493, 511, 5, 0, 0, 0, 0, 0, 0, 0, 507, 501, 517, 551, 516, 495, 508, 486, 29, 2, 0, 0, 0, 0, 0, 0, 479, 503, 530, 459, 527, 546, 493, 504, 0, 0, 0, 0, 0, 0, 0, 0, 497, 532, 466, 479, 524, 515, 509, 512, 18, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4163, 4126, 4087, 4087, 4100, 4112, 4041, 4052] avg=4096.0000 max=4163 max_rank=0 min=4041 min_rank=6 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16305 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[512, 493, 538, 484, 533, 499, 478, 522, 0, 0, 0, 0, 0, 0, 0, 0, 522, 480, 475, 499, 527, 482, 503, 515, 5, 0, 0, 0, 0, 0, 0, 0, 523, 486, 507, 502, 539, 496, 505, 489, 0, 0, 0, 0, 0, 0, 0, 0, 492, 504, 512, 524, 523, 510, 527, 480, 7, 0, 0, 0, 0, 0, 0, 0, 505, 526, 497, 518, 531, 529, 510, 521, 5, 0, 0, 0, 0, 0, 0, 0, 516, 520, 500, 486, 543, 535, 499, 500, 28, 2, 0, 0, 0, 0, 0, 0, 504, 505, 489, 512, 525, 539, 477, 537, 0, 0, 0, 0, 0, 0, 0, 0, 528, 520, 504, 529, 527, 542, 524, 524, 18, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4059, 4008, 4047, 4079, 4142, 4129, 4088, 4216] avg=4096.0000 max=4216 max_rank=7 min=4008 min_rank=1 imbalance_ratio=1.029297 max_load_reduction=28552 max_load_reduction_pct=0.871338 moved_to_echo=16575 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[544, 516, 529, 536, 462, 492, 528, 501, 0, 0, 0, 0, 0, 0, 0, 0, 488, 507, 530, 498, 500, 512, 523, 542, 5, 0, 0, 0, 0, 0, 0, 0, 503, 507, 521, 533, 490, 516, 548, 538, 0, 0, 0, 0, 0, 0, 0, 0, 522, 461, 490, 538, 495, 501, 502, 556, 8, 0, 0, 0, 0, 0, 0, 0, 504, 482, 474, 503, 520, 480, 526, 509, 6, 0, 0, 0, 0, 0, 0, 0, 520, 563, 513, 480, 513, 500, 544, 494, 30, 3, 0, 0, 0, 0, 0, 0, 523, 486, 521, 524, 489, 497, 505, 519, 0, 0, 0, 0, 0, 0, 0, 0, 513, 524, 482, 507, 493, 538, 491, 530, 20, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4105, 4156, 4073, 4004, 4160, 4064, 4098] avg=4096.0000 max=4160 max_rank=5 min=4004 min_rank=4 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16326 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[510, 535, 521, 499, 522, 494, 487, 461, 0, 0, 0, 0, 0, 0, 0, 0, 508, 496, 504, 529, 502, 529, 533, 531, 5, 0, 0, 0, 0, 0, 0, 0, 496, 516, 533, 507, 494, 513, 499, 512, 0, 0, 0, 0, 0, 0, 0, 0, 501, 485, 524, 514, 489, 533, 501, 549, 8, 0, 0, 0, 0, 0, 0, 0, 504, 557, 502, 470, 509, 525, 509, 574, 6, 0, 0, 0, 0, 0, 0, 0, 542, 505, 513, 512, 475, 507, 491, 490, 28, 2, 0, 0, 0, 0, 0, 0, 521, 521, 540, 524, 507, 505, 465, 511, 0, 0, 0, 0, 0, 0, 0, 0, 521, 511, 508, 496, 538, 501, 535, 484, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4029, 4137, 4070, 4104, 4156, 4065, 4094, 4113] avg=4096.0000 max=4156 max_rank=4 min=4029 min_rank=0 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16428 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[535, 520, 496, 534, 513, 553, 543, 504, 0, 0, 0, 0, 0, 0, 0, 0, 482, 522, 511, 547, 534, 492, 501, 524, 4, 0, 0, 0, 0, 0, 0, 0, 534, 478, 512, 516, 494, 461, 523, 489, 0, 0, 0, 0, 0, 0, 0, 0, 472, 493, 469, 540, 504, 535, 521, 538, 7, 0, 0, 0, 0, 0, 0, 0, 462, 487, 549, 542, 536, 510, 501, 514, 5, 0, 0, 0, 0, 0, 0, 0, 457, 509, 500, 524, 528, 556, 475, 520, 30, 3, 0, 0, 0, 0, 0, 0, 524, 523, 547, 522, 501, 507, 495, 510, 0, 0, 0, 0, 0, 0, 0, 0, 498, 530, 483, 489, 468, 526, 488, 528, 20, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4198, 4117, 4007, 4079, 4106, 4102, 4129, 4030] avg=4096.0000 max=4198 max_rank=0 min=4007 min_rank=2 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[519, 488, 528, 477, 536, 519, 532, 508, 0, 0, 0, 0, 0, 0, 0, 0, 516, 502, 520, 490, 508, 500, 503, 511, 9, 0, 0, 0, 0, 0, 0, 0, 514, 522, 530, 523, 500, 508, 516, 503, 0, 0, 0, 0, 0, 0, 0, 0, 512, 503, 502, 545, 517, 550, 479, 520, 12, 7, 0, 0, 0, 0, 0, 0, 480, 522, 486, 526, 501, 524, 519, 505, 10, 0, 0, 0, 0, 0, 0, 0, 561, 519, 482, 493, 500, 503, 498, 512, 35, 7, 0, 0, 0, 0, 0, 0, 497, 515, 537, 498, 511, 538, 511, 514, 0, 0, 0, 0, 0, 0, 0, 0, 503, 484, 517, 513, 518, 501, 493, 483, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4059, 4116, 4147, 4073, 4110, 4121, 4035] avg=4096.0000 max=4147 max_rank=3 min=4035 min_rank=7 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16339 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3968, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[523, 544, 506, 513, 545, 497, 505, 519, 495, 478, 543, 504, 509, 520, 475, 525, 503, 509, 524, 534, 491, 518, 460, 526, 505, 516, 509, 511, 509, 525, 501, 514, 494, 507, 506, 512, 530, 538, 499, 510, 504, 490, 498, 514, 524, 534, 528, 528, 519, 509, 507, 504, 518, 495, 479, 498, 552, 542, 541, 492, 512, 498, 520, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[504, 522, 515, 497, 537, 482, 470, 499, 541, 504, 462, 520, 500, 496, 512, 478, 498, 549, 490, 474, 514, 514, 523, 530, 489, 520, 497, 526, 509, 552, 519, 507, 532, 474, 493, 541, 525, 523, 480, 500, 529, 514, 522, 513, 516, 525, 534, 565, 530, 516, 520, 527, 540, 493, 534, 518, 524, 515, 469, 504, 495, 529, 523, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[521, 521, 530, 518, 508, 534, 495, 463, 497, 540, 510, 505, 524, 522, 510, 529, 534, 536, 467, 541, 480, 523, 485, 514, 523, 501, 510, 465, 506, 504, 541, 513, 548, 515, 513, 548, 524, 492, 490, 493, 468, 483, 512, 492, 493, 526, 495, 517, 563, 506, 522, 523, 524, 469, 523, 531, 525, 473, 497, 525, 501, 510, 576, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[504, 510, 507, 505, 562, 538, 513, 512, 522, 487, 528, 527, 532, 530, 519, 505, 518, 477, 523, 516, 548, 482, 478, 527, 517, 541, 508, 493, 488, 538, 523, 516, 487, 521, 479, 480, 499, 493, 492, 525, 510, 495, 515, 552, 500, 523, 521, 531, 562, 527, 504, 520, 507, 478, 495, 514, 514, 504, 497, 497, 538, 529, 470, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[491, 498, 515, 521, 476, 545, 481, 481, 544, 539, 519, 516, 512, 517, 537, 511, 530, 520, 516, 459, 509, 507, 508, 495, 530, 514, 494, 521, 493, 516, 581, 505, 514, 504, 504, 529, 528, 538, 500, 510, 496, 513, 511, 496, 489, 486, 534, 524, 508, 504, 549, 507, 521, 499, 513, 515, 477, 531, 490, 533, 519, 507, 514, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[491, 506, 533, 490, 534, 512, 513, 534, 520, 508, 500, 542, 556, 506, 467, 519, 499, 530, 495, 485, 521, 485, 512, 518, 508, 485, 539, 526, 547, 512, 539, 497, 502, 502, 519, 540, 537, 514, 522, 518, 526, 513, 506, 488, 523, 507, 498, 524, 545, 506, 493, 462, 536, 489, 522, 523, 522, 518, 484, 498, 484, 487, 551, 480] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[529, 487, 527, 507, 520, 517, 506, 520, 542, 510, 489, 522, 482, 521, 483, 547, 520, 474, 512, 488, 492, 492, 492, 511, 518, 501, 467, 540, 547, 491, 493, 521, 511, 509, 510, 500, 515, 490, 550, 546, 518, 532, 513, 531, 490, 496, 542, 524, 519, 505, 529, 545, 552, 501, 538, 523, 510, 490, 508, 487, 510, 487, 518, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[528, 505, 497, 544, 530, 505, 525, 539, 526, 493, 498, 510, 536, 496, 539, 528, 518, 504, 484, 510, 537, 506, 507, 492, 496, 489, 516, 503, 540, 517, 519, 485, 546, 526, 479, 472, 491, 518, 541, 453, 521, 545, 532, 509, 551, 486, 525, 505, 493, 490, 557, 519, 469, 497, 526, 492, 508, 502, 532, 486, 523, 489, 520, 543] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[504, 522, 515, 497, 530, 482, 470, 499, 0, 0, 0, 0, 0, 0, 0, 0, 527, 504, 462, 520, 500, 496, 512, 478, 0, 0, 0, 0, 0, 0, 0, 0, 498, 549, 490, 474, 514, 514, 523, 530, 21, 19, 2, 0, 0, 0, 0, 0, 489, 520, 497, 526, 509, 552, 510, 507, 0, 0, 0, 0, 0, 0, 0, 0, 532, 474, 493, 541, 525, 523, 480, 500, 7, 1, 0, 0, 0, 0, 0, 0, 529, 514, 522, 513, 516, 525, 534, 544, 0, 0, 0, 0, 0, 0, 0, 0, 511, 516, 520, 527, 540, 493, 534, 518, 0, 0, 0, 0, 0, 0, 0, 0, 524, 515, 469, 504, 495, 529, 523, 495, 12, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4019, 3999, 4134, 4110, 4076, 4197, 4159, 4074] avg=4096.0000 max=4197 max_rank=5 min=3999 min_rank=1 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16506 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[523, 544, 506, 513, 538, 497, 505, 519, 0, 0, 0, 0, 0, 0, 0, 0, 483, 478, 543, 504, 509, 520, 475, 525, 0, 0, 0, 0, 0, 0, 0, 0, 503, 509, 524, 534, 491, 518, 460, 526, 19, 18, 2, 0, 0, 0, 0, 0, 505, 516, 509, 511, 509, 525, 492, 514, 0, 0, 0, 0, 0, 0, 0, 0, 494, 507, 506, 512, 530, 538, 499, 510, 7, 1, 0, 0, 0, 0, 0, 0, 504, 490, 498, 514, 524, 534, 528, 509, 0, 0, 0, 0, 0, 0, 0, 0, 501, 509, 507, 504, 518, 495, 479, 498, 0, 0, 0, 0, 0, 0, 0, 0, 552, 542, 541, 492, 512, 498, 520, 510, 10, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4145, 4037, 4104, 4081, 4104, 4101, 4011, 4185] avg=4096.0000 max=4185 max_rank=7 min=4011 min_rank=6 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16401 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[521, 521, 530, 518, 502, 534, 495, 463, 0, 0, 0, 0, 0, 0, 0, 0, 484, 540, 510, 505, 524, 522, 510, 529, 0, 0, 0, 0, 0, 0, 0, 0, 534, 536, 467, 541, 480, 523, 485, 514, 19, 20, 2, 0, 0, 0, 0, 0, 523, 501, 510, 465, 506, 504, 532, 513, 0, 0, 0, 0, 0, 0, 0, 0, 548, 515, 513, 548, 524, 492, 490, 493, 6, 1, 0, 0, 0, 0, 0, 0, 468, 483, 512, 492, 493, 526, 495, 498, 0, 0, 0, 0, 0, 0, 0, 0, 543, 506, 522, 523, 524, 469, 523, 531, 0, 0, 0, 0, 0, 0, 0, 0, 525, 473, 497, 525, 501, 510, 576, 521, 11, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4124, 4121, 4054, 4130, 3967, 4141, 4147] avg=4096.0000 max=4147 max_rank=7 min=3967 min_rank=5 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[491, 498, 515, 521, 470, 545, 481, 481, 0, 0, 0, 0, 0, 0, 0, 0, 530, 539, 519, 516, 512, 517, 537, 511, 0, 0, 0, 0, 0, 0, 0, 0, 530, 520, 516, 459, 509, 507, 508, 495, 19, 18, 2, 0, 0, 0, 0, 0, 530, 514, 494, 521, 493, 516, 571, 505, 0, 0, 0, 0, 0, 0, 0, 0, 514, 504, 504, 529, 528, 538, 500, 510, 6, 1, 0, 0, 0, 0, 0, 0, 496, 513, 511, 496, 489, 486, 534, 505, 0, 0, 0, 0, 0, 0, 0, 0, 490, 504, 549, 507, 521, 499, 513, 515, 0, 0, 0, 0, 0, 0, 0, 0, 477, 531, 490, 533, 519, 507, 514, 504, 12, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4002, 4181, 4083, 4144, 4134, 4030, 4098, 4096] avg=4096.0000 max=4181 max_rank=1 min=4002 min_rank=0 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16358 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[528, 505, 497, 544, 519, 505, 525, 539, 0, 0, 0, 0, 0, 0, 0, 0, 503, 493, 498, 510, 536, 496, 539, 528, 0, 0, 0, 0, 0, 0, 0, 0, 518, 504, 484, 510, 537, 506, 507, 492, 22, 22, 9, 0, 0, 0, 0, 0, 496, 489, 516, 503, 540, 517, 505, 485, 0, 0, 0, 0, 0, 0, 0, 0, 546, 526, 479, 472, 491, 518, 541, 453, 11, 2, 0, 0, 0, 0, 0, 0, 521, 545, 532, 509, 551, 486, 525, 483, 0, 0, 0, 0, 0, 0, 0, 0, 471, 490, 557, 519, 469, 497, 526, 492, 0, 0, 0, 0, 0, 0, 0, 0, 508, 502, 532, 486, 523, 489, 520, 543, 14, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4162, 4103, 4111, 4051, 4039, 4152, 4021, 4129] avg=4096.0000 max=4162 max_rank=0 min=4021 min_rank=6 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16341 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[504, 510, 507, 505, 555, 538, 513, 512, 0, 0, 0, 0, 0, 0, 0, 0, 509, 487, 528, 527, 532, 530, 519, 505, 0, 0, 0, 0, 0, 0, 0, 0, 518, 477, 523, 516, 548, 482, 478, 527, 19, 20, 2, 0, 0, 0, 0, 0, 517, 541, 508, 493, 488, 538, 514, 516, 0, 0, 0, 0, 0, 0, 0, 0, 487, 521, 479, 480, 499, 493, 492, 525, 7, 1, 0, 0, 0, 0, 0, 0, 510, 495, 515, 552, 500, 523, 521, 512, 0, 0, 0, 0, 0, 0, 0, 0, 542, 527, 504, 520, 507, 478, 495, 514, 0, 0, 0, 0, 0, 0, 0, 0, 514, 504, 497, 497, 538, 529, 470, 495, 11, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4144, 4137, 4110, 4115, 3984, 4128, 4087, 4063] avg=4096.0000 max=4144 max_rank=0 min=3984 min_rank=4 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16262 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[491, 506, 533, 490, 527, 512, 513, 534, 0, 0, 0, 0, 0, 0, 0, 0, 507, 508, 500, 542, 556, 506, 467, 519, 0, 0, 0, 0, 0, 0, 0, 0, 499, 530, 495, 485, 521, 485, 512, 518, 19, 19, 2, 0, 0, 0, 0, 0, 508, 485, 539, 526, 547, 512, 530, 497, 0, 0, 0, 0, 0, 0, 0, 0, 502, 502, 519, 540, 537, 514, 522, 518, 7, 1, 0, 0, 0, 0, 0, 0, 526, 513, 506, 488, 523, 507, 498, 505, 0, 0, 0, 0, 0, 0, 0, 0, 526, 506, 493, 462, 536, 489, 522, 523, 0, 0, 0, 0, 0, 0, 0, 0, 522, 518, 484, 498, 484, 487, 551, 480, 11, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4105, 4085, 4144, 4162, 4066, 4057, 4043] avg=4096.0000 max=4162 max_rank=4 min=4043 min_rank=7 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16328 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[529, 487, 527, 507, 513, 517, 506, 520, 0, 0, 0, 0, 0, 0, 0, 0, 528, 510, 489, 522, 482, 521, 483, 547, 0, 0, 0, 0, 0, 0, 0, 0, 520, 474, 512, 488, 492, 492, 492, 511, 19, 18, 2, 0, 0, 0, 0, 0, 518, 501, 467, 540, 547, 491, 484, 521, 0, 0, 0, 0, 0, 0, 0, 0, 511, 509, 510, 500, 515, 490, 550, 546, 7, 1, 0, 0, 0, 0, 0, 0, 518, 532, 513, 531, 490, 496, 542, 505, 0, 0, 0, 0, 0, 0, 0, 0, 501, 505, 529, 545, 552, 501, 538, 523, 0, 0, 0, 0, 0, 0, 0, 0, 510, 490, 508, 487, 510, 487, 518, 501, 12, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4082, 4020, 4069, 4139, 4127, 4194, 4031] avg=4096.0000 max=4194 max_rank=6 min=4020 min_rank=2 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16491 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 3968, 4224, 256, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 3968, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[499, 520, 496, 504, 535, 497, 519, 521, 531, 516, 518, 498, 518, 522, 498, 483, 522, 497, 513, 502, 490, 503, 530, 522, 546, 509, 505, 496, 526, 487, 500, 532, 507, 515, 528, 541, 477, 491, 532, 529, 519, 507, 520, 509, 481, 501, 511, 522, 494, 503, 519, 486, 548, 543, 512, 496, 512, 550, 518, 525, 534, 493, 492, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[491, 496, 482, 498, 516, 505, 536, 482, 489, 523, 504, 485, 497, 466, 499, 518, 532, 521, 475, 560, 463, 538, 522, 524, 503, 492, 503, 555, 520, 497, 506, 524, 492, 536, 538, 543, 488, 499, 538, 485, 512, 500, 519, 516, 530, 534, 510, 511, 497, 534, 526, 511, 507, 555, 481, 527, 519, 525, 519, 504, 532, 530, 524, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[535, 526, 529, 524, 511, 497, 530, 535, 489, 544, 515, 511, 521, 495, 515, 522, 517, 533, 524, 543, 496, 508, 503, 490, 517, 518, 480, 513, 485, 496, 515, 516, 533, 524, 520, 494, 490, 529, 514, 500, 520, 538, 487, 537, 465, 493, 501, 514, 521, 519, 513, 537, 505, 511, 505, 526, 532, 484, 513, 483, 489, 511, 508, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[491, 491, 563, 505, 505, 522, 497, 500, 519, 491, 524, 534, 478, 534, 504, 523, 520, 517, 504, 521, 541, 543, 499, 487, 526, 504, 509, 487, 485, 502, 493, 503, 492, 527, 513, 533, 492, 539, 485, 470, 543, 528, 491, 497, 537, 544, 492, 494, 487, 500, 507, 556, 533, 491, 510, 530, 533, 500, 520, 534, 516, 520, 486, 546] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[530, 518, 507, 545, 524, 494, 491, 508, 492, 517, 523, 524, 477, 548, 497, 508, 506, 488, 519, 493, 522, 529, 511, 505, 533, 515, 497, 519, 539, 484, 541, 536, 500, 515, 510, 516, 481, 489, 532, 505, 519, 508, 543, 475, 529, 463, 514, 506, 500, 506, 550, 511, 473, 498, 535, 523, 501, 487, 508, 522, 515, 522, 525, 547] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[498, 537, 503, 502, 555, 533, 510, 475, 492, 484, 503, 534, 547, 523, 520, 553, 496, 465, 500, 531, 504, 544, 517, 511, 487, 518, 522, 512, 528, 507, 507, 525, 523, 478, 574, 506, 463, 557, 514, 496, 501, 530, 508, 556, 489, 514, 472, 540, 490, 477, 516, 506, 525, 522, 534, 470, 508, 489, 492, 537, 504, 531, 524, 479] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[500, 509, 526, 532, 509, 508, 521, 490, 498, 482, 513, 526, 466, 501, 492, 562, 534, 488, 516, 525, 533, 539, 550, 532, 503, 519, 482, 471, 455, 528, 516, 512, 486, 538, 474, 482, 546, 483, 530, 507, 515, 519, 556, 543, 536, 527, 477, 498, 526, 501, 499, 517, 504, 519, 498, 541, 483, 509, 513, 499, 539, 506, 526, 533] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[522, 492, 528, 521, 495, 521, 520, 501, 505, 477, 537, 489, 515, 498, 530, 494, 516, 525, 523, 531, 523, 543, 509, 505, 494, 561, 485, 504, 481, 542, 536, 506, 508, 497, 545, 518, 509, 509, 541, 520, 470, 485, 522, 522, 509, 515, 535, 515, 566, 525, 506, 488, 524, 512, 485, 533, 508, 467, 523, 484, 474, 557, 471, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[499, 520, 496, 504, 534, 497, 519, 521, 0, 0, 0, 0, 0, 0, 0, 0, 531, 516, 518, 498, 518, 522, 498, 483, 18, 0, 0, 0, 0, 0, 0, 0, 522, 497, 513, 502, 490, 477, 530, 522, 0, 0, 0, 0, 0, 0, 0, 0, 546, 509, 505, 496, 526, 487, 500, 532, 11, 8, 0, 0, 0, 0, 0, 0, 507, 515, 528, 541, 477, 491, 532, 529, 1, 0, 0, 0, 0, 0, 0, 0, 519, 507, 520, 507, 481, 501, 511, 522, 0, 0, 0, 0, 0, 0, 0, 0, 494, 503, 519, 486, 548, 530, 512, 496, 0, 0, 0, 0, 0, 0, 0, 0, 512, 550, 518, 525, 534, 493, 492, 498, 2, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4102, 4053, 4120, 4121, 4068, 4088, 4126] avg=4096.0000 max=4126 max_rank=7 min=4053 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=28642 max_load_reduction_pct=0.874084 moved_to_echo=16403 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[535, 526, 529, 524, 510, 497, 530, 535, 0, 0, 0, 0, 0, 0, 0, 0, 489, 544, 515, 511, 521, 495, 515, 522, 18, 0, 0, 0, 0, 0, 0, 0, 517, 533, 524, 543, 496, 482, 503, 490, 0, 0, 0, 0, 0, 0, 0, 0, 517, 518, 480, 513, 485, 496, 515, 516, 10, 8, 0, 0, 0, 0, 0, 0, 533, 524, 520, 494, 490, 529, 514, 500, 1, 0, 0, 0, 0, 0, 0, 0, 520, 538, 487, 535, 465, 493, 501, 514, 0, 0, 0, 0, 0, 0, 0, 0, 521, 519, 513, 537, 505, 499, 505, 526, 0, 0, 0, 0, 0, 0, 0, 0, 532, 484, 513, 483, 489, 511, 508, 499, 2, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4186, 4130, 4088, 4058, 4105, 4053, 4125, 4023] avg=4096.0000 max=4186 max_rank=0 min=4023 min_rank=7 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16306 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[491, 496, 482, 498, 509, 505, 536, 482, 0, 0, 0, 0, 0, 0, 0, 0, 489, 523, 504, 485, 497, 466, 499, 518, 23, 0, 0, 0, 0, 0, 0, 0, 532, 521, 475, 560, 463, 503, 522, 524, 0, 0, 0, 0, 0, 0, 0, 0, 503, 492, 503, 555, 520, 497, 506, 524, 14, 12, 0, 0, 0, 0, 0, 0, 492, 536, 538, 543, 488, 499, 538, 485, 7, 7, 0, 0, 0, 0, 0, 0, 512, 500, 519, 504, 530, 534, 510, 511, 0, 0, 0, 0, 0, 0, 0, 0, 497, 534, 526, 511, 507, 538, 481, 527, 0, 0, 0, 0, 0, 0, 0, 0, 519, 525, 519, 504, 532, 530, 524, 504, 5, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3999, 4004, 4100, 4126, 4133, 4120, 4121, 4165] avg=4096.0000 max=4165 max_rank=7 min=3999 min_rank=0 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16539 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[491, 491, 563, 505, 504, 522, 497, 500, 0, 0, 0, 0, 0, 0, 0, 0, 519, 491, 524, 534, 478, 534, 504, 523, 20, 0, 0, 0, 0, 0, 0, 0, 520, 517, 504, 521, 541, 515, 499, 487, 0, 0, 0, 0, 0, 0, 0, 0, 526, 504, 509, 487, 485, 502, 493, 503, 10, 8, 0, 0, 0, 0, 0, 0, 492, 527, 513, 533, 492, 539, 485, 470, 1, 0, 0, 0, 0, 0, 0, 0, 543, 528, 491, 495, 537, 544, 492, 494, 0, 0, 0, 0, 0, 0, 0, 0, 487, 500, 507, 556, 533, 479, 510, 530, 0, 0, 0, 0, 0, 0, 0, 0, 533, 500, 520, 534, 516, 520, 486, 546, 2, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4127, 4104, 4027, 4052, 4124, 4102, 4159] avg=4096.0000 max=4159 max_rank=7 min=4027 min_rank=3 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16437 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[500, 509, 526, 532, 508, 508, 521, 490, 0, 0, 0, 0, 0, 0, 0, 0, 498, 482, 513, 526, 466, 501, 492, 562, 19, 0, 0, 0, 0, 0, 0, 0, 534, 488, 516, 525, 533, 512, 550, 532, 0, 0, 0, 0, 0, 0, 0, 0, 503, 519, 482, 471, 455, 528, 516, 512, 10, 8, 0, 0, 0, 0, 0, 0, 486, 538, 474, 482, 546, 483, 530, 507, 1, 0, 0, 0, 0, 0, 0, 0, 515, 519, 556, 541, 536, 527, 477, 498, 0, 0, 0, 0, 0, 0, 0, 0, 526, 501, 499, 517, 504, 507, 498, 541, 0, 0, 0, 0, 0, 0, 0, 0, 483, 509, 513, 499, 539, 506, 526, 533, 2, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4059, 4190, 4004, 4047, 4169, 4093, 4112] avg=4096.0000 max=4190 max_rank=2 min=4004 min_rank=3 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16421 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[522, 492, 528, 521, 494, 521, 520, 501, 0, 0, 0, 0, 0, 0, 0, 0, 505, 477, 537, 489, 515, 498, 530, 494, 20, 0, 0, 0, 0, 0, 0, 0, 516, 525, 523, 531, 523, 515, 509, 505, 0, 0, 0, 0, 0, 0, 0, 0, 494, 561, 485, 504, 481, 542, 536, 506, 10, 8, 0, 0, 0, 0, 0, 0, 508, 497, 545, 518, 509, 509, 541, 520, 1, 0, 0, 0, 0, 0, 0, 0, 470, 485, 522, 520, 509, 515, 535, 515, 0, 0, 0, 0, 0, 0, 0, 0, 566, 525, 506, 488, 524, 500, 485, 533, 0, 0, 0, 0, 0, 0, 0, 0, 508, 467, 523, 484, 474, 557, 471, 496, 2, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4065, 4147, 4127, 4148, 4071, 4127, 3984] avg=4096.0000 max=4148 max_rank=4 min=3984 min_rank=7 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16330 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[530, 518, 507, 545, 523, 494, 491, 508, 0, 0, 0, 0, 0, 0, 0, 0, 492, 517, 523, 524, 477, 548, 497, 508, 19, 0, 0, 0, 0, 0, 0, 0, 506, 488, 519, 493, 522, 502, 511, 505, 0, 0, 0, 0, 0, 0, 0, 0, 533, 515, 497, 519, 539, 484, 541, 536, 10, 8, 0, 0, 0, 0, 0, 0, 500, 515, 510, 516, 481, 489, 532, 505, 1, 0, 0, 0, 0, 0, 0, 0, 519, 508, 543, 473, 529, 463, 514, 506, 0, 0, 0, 0, 0, 0, 0, 0, 500, 506, 550, 511, 473, 486, 535, 523, 0, 0, 0, 0, 0, 0, 0, 0, 501, 487, 508, 522, 515, 522, 525, 547, 2, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4105, 4046, 4182, 4049, 4055, 4084, 4131] avg=4096.0000 max=4182 max_rank=3 min=4046 min_rank=2 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16319 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[498, 537, 503, 502, 553, 533, 510, 475, 0, 0, 0, 0, 0, 0, 0, 0, 492, 484, 503, 534, 547, 523, 520, 553, 20, 0, 0, 0, 0, 0, 0, 0, 496, 465, 500, 531, 504, 516, 517, 511, 0, 0, 0, 0, 0, 0, 0, 0, 487, 518, 522, 512, 528, 507, 507, 525, 10, 8, 0, 0, 0, 0, 0, 0, 523, 478, 574, 506, 463, 557, 514, 496, 2, 0, 0, 0, 0, 0, 0, 0, 501, 530, 508, 554, 489, 514, 472, 540, 0, 0, 0, 0, 0, 0, 0, 0, 490, 477, 516, 506, 525, 510, 534, 470, 0, 0, 0, 0, 0, 0, 0, 0, 508, 489, 492, 537, 504, 531, 524, 479, 2, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4176, 4040, 4124, 4113, 4108, 4028, 4068] avg=4096.0000 max=4176 max_rank=1 min=4028 min_rank=6 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16317 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 3968, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[522, 542, 513, 484, 543, 521, 500, 534, 520, 522, 513, 485, 477, 514, 495, 488, 501, 509, 542, 505, 538, 515, 515, 540, 490, 492, 538, 490, 536, 545, 509, 537, 518, 502, 505, 490, 513, 465, 488, 503, 537, 491, 490, 491, 527, 501, 525, 541, 523, 508, 531, 494, 492, 537, 524, 517, 553, 505, 494, 546, 516, 484, 495, 487] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[541, 512, 547, 489, 505, 503, 528, 483, 538, 487, 471, 525, 499, 496, 519, 537, 507, 520, 511, 511, 488, 471, 498, 531, 553, 520, 531, 485, 507, 555, 550, 507, 531, 509, 518, 519, 506, 536, 517, 526, 532, 551, 511, 505, 472, 466, 529, 509, 492, 499, 488, 489, 495, 501, 512, 513, 519, 518, 545, 500, 509, 537, 505, 484] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[507, 563, 501, 522, 538, 537, 547, 488, 497, 489, 474, 532, 512, 548, 503, 507, 498, 483, 524, 501, 522, 483, 527, 493, 505, 493, 513, 511, 523, 517, 501, 532, 503, 470, 483, 501, 533, 542, 512, 513, 491, 498, 515, 531, 496, 544, 491, 508, 506, 482, 540, 537, 523, 486, 507, 498, 502, 536, 543, 492, 545, 521, 523, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[477, 526, 489, 471, 503, 479, 525, 474, 513, 520, 521, 492, 533, 510, 487, 508, 551, 549, 495, 515, 523, 501, 541, 470, 511, 511, 503, 520, 524, 537, 488, 504, 527, 519, 522, 506, 502, 533, 541, 532, 498, 574, 526, 511, 486, 507, 534, 498, 508, 525, 551, 472, 512, 487, 491, 514, 558, 513, 517, 508, 492, 518, 531, 484] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[466, 531, 503, 516, 514, 541, 498, 520, 498, 520, 517, 507, 484, 526, 512, 514, 545, 497, 543, 503, 501, 493, 497, 466, 523, 520, 500, 538, 499, 477, 486, 465, 534, 485, 530, 550, 516, 504, 495, 521, 543, 475, 541, 509, 532, 503, 481, 491, 566, 510, 516, 506, 480, 501, 535, 539, 530, 539, 521, 532, 510, 528, 517, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[553, 508, 482, 522, 504, 489, 481, 501, 529, 517, 525, 524, 563, 478, 510, 481, 505, 507, 495, 490, 517, 525, 540, 518, 473, 518, 499, 516, 510, 497, 535, 508, 524, 517, 525, 517, 513, 521, 501, 550, 496, 501, 528, 506, 508, 523, 497, 524, 506, 524, 522, 536, 509, 543, 471, 526, 547, 510, 489, 515, 514, 507, 495, 483] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[528, 498, 523, 508, 513, 464, 526, 513, 500, 504, 510, 517, 523, 500, 500, 484, 488, 504, 539, 540, 539, 530, 509, 501, 512, 494, 492, 496, 524, 499, 517, 482, 500, 528, 515, 527, 534, 506, 542, 525, 540, 466, 492, 502, 526, 516, 493, 541, 540, 510, 570, 508, 509, 497, 517, 505, 531, 503, 526, 506, 532, 480, 494, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[521, 501, 496, 500, 514, 543, 498, 523, 504, 491, 538, 529, 494, 495, 499, 547, 521, 524, 511, 511, 500, 506, 533, 501, 519, 493, 475, 517, 482, 523, 560, 544, 495, 506, 522, 457, 492, 501, 538, 504, 535, 500, 482, 522, 518, 530, 469, 512, 521, 537, 479, 555, 506, 502, 513, 532, 519, 539, 526, 518, 511, 504, 504, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[522, 542, 513, 484, 543, 521, 500, 534, 6, 0, 0, 0, 0, 0, 0, 0, 520, 522, 513, 485, 477, 514, 495, 488, 22, 3, 0, 0, 0, 0, 0, 0, 501, 509, 542, 505, 538, 515, 514, 540, 0, 0, 0, 0, 0, 0, 0, 0, 490, 492, 538, 490, 536, 545, 509, 537, 3, 1, 0, 0, 0, 0, 0, 0, 518, 502, 505, 490, 513, 465, 488, 491, 0, 0, 0, 0, 0, 0, 0, 0, 537, 491, 490, 491, 527, 501, 525, 541, 9, 0, 0, 0, 0, 0, 0, 0, 523, 508, 522, 494, 492, 537, 524, 517, 0, 0, 0, 0, 0, 0, 0, 0, 531, 505, 494, 546, 516, 484, 495, 487, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4165, 4039, 4164, 4141, 3972, 4112, 4117, 4058] avg=4096.0000 max=4165 max_rank=0 min=3972 min_rank=4 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16259 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[541, 512, 547, 489, 505, 503, 528, 483, 5, 0, 0, 0, 0, 0, 0, 0, 538, 487, 471, 525, 499, 496, 519, 537, 20, 3, 0, 0, 0, 0, 0, 0, 507, 520, 511, 511, 488, 471, 497, 531, 0, 0, 0, 0, 0, 0, 0, 0, 553, 520, 531, 485, 507, 555, 550, 507, 3, 1, 0, 0, 0, 0, 0, 0, 531, 509, 518, 519, 506, 536, 517, 513, 0, 0, 0, 0, 0, 0, 0, 0, 532, 551, 511, 505, 472, 466, 529, 509, 10, 0, 0, 0, 0, 0, 0, 0, 492, 499, 480, 489, 495, 501, 512, 513, 0, 0, 0, 0, 0, 0, 0, 0, 499, 518, 545, 500, 509, 537, 505, 484, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4095, 4036, 4212, 4149, 4085, 3981, 4097] avg=4096.0000 max=4212 max_rank=3 min=3981 min_rank=6 imbalance_ratio=1.028320 max_load_reduction=28556 max_load_reduction_pct=0.871460 moved_to_echo=16312 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[507, 563, 501, 522, 538, 537, 547, 488, 9, 7, 0, 0, 0, 0, 0, 0, 497, 489, 474, 532, 512, 548, 503, 507, 24, 4, 0, 0, 0, 0, 0, 0, 498, 483, 524, 501, 522, 483, 525, 493, 0, 0, 0, 0, 0, 0, 0, 0, 505, 493, 513, 511, 523, 517, 501, 532, 7, 2, 0, 0, 0, 0, 0, 0, 503, 470, 483, 501, 533, 542, 512, 489, 0, 0, 0, 0, 0, 0, 0, 0, 491, 498, 515, 531, 496, 544, 491, 508, 13, 0, 0, 0, 0, 0, 0, 0, 506, 482, 524, 537, 523, 486, 507, 498, 0, 0, 0, 0, 0, 0, 0, 0, 478, 536, 543, 492, 545, 521, 523, 505, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4219, 4090, 4029, 4104, 4033, 4087, 4063, 4143] avg=4096.0000 max=4219 max_rank=0 min=4029 min_rank=2 imbalance_ratio=1.030029 max_load_reduction=28549 max_load_reduction_pct=0.871246 moved_to_echo=16326 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[553, 508, 482, 522, 504, 489, 481, 501, 6, 0, 0, 0, 0, 0, 0, 0, 529, 517, 525, 524, 563, 478, 510, 481, 21, 3, 0, 0, 0, 0, 0, 0, 505, 507, 495, 490, 517, 525, 539, 518, 0, 0, 0, 0, 0, 0, 0, 0, 473, 518, 499, 516, 510, 497, 535, 508, 3, 1, 0, 0, 0, 0, 0, 0, 524, 517, 525, 517, 513, 521, 501, 537, 0, 0, 0, 0, 0, 0, 0, 0, 496, 501, 528, 506, 508, 523, 497, 524, 10, 0, 0, 0, 0, 0, 0, 0, 506, 524, 513, 536, 509, 543, 471, 526, 0, 0, 0, 0, 0, 0, 0, 0, 526, 510, 489, 515, 514, 507, 495, 483, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4046, 4151, 4096, 4060, 4155, 4093, 4128, 4039] avg=4096.0000 max=4155 max_rank=4 min=4039 min_rank=7 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16415 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[477, 526, 489, 471, 503, 479, 525, 474, 6, 0, 0, 0, 0, 0, 0, 0, 513, 520, 521, 492, 533, 510, 487, 508, 22, 3, 0, 0, 0, 0, 0, 0, 551, 549, 495, 515, 523, 501, 540, 470, 0, 0, 0, 0, 0, 0, 0, 0, 511, 511, 503, 520, 524, 537, 488, 504, 3, 1, 0, 0, 0, 0, 0, 0, 527, 519, 522, 506, 502, 533, 541, 519, 0, 0, 0, 0, 0, 0, 0, 0, 498, 574, 526, 511, 486, 507, 534, 498, 10, 0, 0, 0, 0, 0, 0, 0, 508, 525, 542, 472, 512, 487, 491, 514, 0, 0, 0, 0, 0, 0, 0, 0, 536, 513, 517, 508, 492, 518, 531, 484, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3950, 4109, 4144, 4102, 4169, 4144, 4051, 4099] avg=4096.0000 max=4169 max_rank=4 min=3950 min_rank=0 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16463 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[466, 531, 503, 516, 514, 541, 498, 520, 6, 0, 0, 0, 0, 0, 0, 0, 498, 520, 517, 507, 484, 526, 512, 514, 21, 3, 0, 0, 0, 0, 0, 0, 545, 497, 543, 503, 501, 493, 496, 466, 0, 0, 0, 0, 0, 0, 0, 0, 523, 520, 500, 538, 499, 477, 486, 465, 3, 1, 0, 0, 0, 0, 0, 0, 534, 485, 530, 550, 516, 504, 495, 509, 0, 0, 0, 0, 0, 0, 0, 0, 543, 475, 541, 509, 532, 503, 481, 491, 9, 0, 0, 0, 0, 0, 0, 0, 566, 510, 507, 506, 480, 501, 535, 539, 0, 0, 0, 0, 0, 0, 0, 0, 509, 539, 521, 532, 510, 528, 517, 508, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4102, 4044, 4012, 4123, 4084, 4144, 4164] avg=4096.0000 max=4164 max_rank=7 min=4012 min_rank=3 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16515 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[528, 498, 523, 508, 513, 464, 526, 513, 6, 0, 0, 0, 0, 0, 0, 0, 500, 504, 510, 517, 523, 500, 500, 484, 21, 3, 0, 0, 0, 0, 0, 0, 488, 504, 539, 540, 539, 530, 508, 501, 0, 0, 0, 0, 0, 0, 0, 0, 512, 494, 492, 496, 524, 499, 517, 482, 3, 1, 0, 0, 0, 0, 0, 0, 500, 528, 515, 527, 534, 506, 542, 512, 0, 0, 0, 0, 0, 0, 0, 0, 540, 466, 492, 502, 526, 516, 493, 541, 10, 0, 0, 0, 0, 0, 0, 0, 540, 510, 561, 508, 509, 497, 517, 505, 0, 0, 0, 0, 0, 0, 0, 0, 510, 503, 526, 506, 532, 480, 494, 510, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4062, 4149, 4020, 4164, 4086, 4147, 4061] avg=4096.0000 max=4164 max_rank=4 min=4020 min_rank=3 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16458 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[521, 501, 496, 500, 514, 543, 498, 523, 5, 0, 0, 0, 0, 0, 0, 0, 504, 491, 538, 529, 494, 495, 499, 547, 20, 3, 0, 0, 0, 0, 0, 0, 521, 524, 511, 511, 500, 506, 532, 501, 0, 0, 0, 0, 0, 0, 0, 0, 519, 493, 475, 517, 482, 523, 560, 544, 3, 1, 0, 0, 0, 0, 0, 0, 495, 506, 522, 457, 492, 501, 538, 492, 0, 0, 0, 0, 0, 0, 0, 0, 535, 500, 482, 522, 518, 530, 469, 512, 9, 0, 0, 0, 0, 0, 0, 0, 521, 537, 471, 555, 506, 502, 513, 532, 0, 0, 0, 0, 0, 0, 0, 0, 499, 539, 526, 518, 511, 504, 504, 506, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4120, 4106, 4117, 4003, 4077, 4137, 4107] avg=4096.0000 max=4137 max_rank=6 min=4003 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16324 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4096, 3968, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[491, 479, 496, 517, 482, 519, 493, 519, 494, 498, 503, 541, 476, 525, 517, 536, 476, 494, 496, 538, 508, 504, 477, 506, 483, 536, 514, 540, 503, 504, 529, 521, 493, 499, 499, 533, 536, 530, 505, 523, 500, 518, 491, 534, 505, 495, 535, 482, 499, 551, 543, 526, 523, 510, 539, 523, 547, 518, 494, 531, 509, 522, 551, 489] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[461, 489, 539, 479, 519, 517, 522, 503, 529, 513, 473, 551, 488, 504, 525, 531, 506, 528, 494, 535, 480, 519, 534, 518, 471, 535, 544, 510, 519, 529, 506, 516, 490, 563, 471, 529, 484, 519, 512, 514, 515, 522, 537, 510, 529, 525, 521, 503, 532, 516, 510, 516, 495, 488, 486, 478, 487, 535, 465, 533, 541, 542, 491, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[519, 521, 565, 519, 519, 497, 510, 492, 506, 506, 513, 525, 504, 491, 484, 499, 494, 508, 540, 505, 520, 520, 529, 489, 467, 528, 534, 491, 502, 511, 501, 562, 487, 547, 494, 539, 535, 502, 523, 523, 519, 491, 524, 473, 530, 513, 499, 493, 496, 502, 485, 537, 530, 497, 480, 482, 541, 563, 522, 543, 499, 495, 517, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[548, 542, 489, 504, 505, 523, 521, 541, 507, 514, 490, 525, 479, 506, 518, 528, 476, 527, 542, 505, 533, 473, 530, 493, 500, 501, 476, 512, 528, 505, 494, 480, 536, 502, 498, 506, 539, 533, 523, 533, 506, 535, 471, 547, 538, 523, 495, 490, 501, 482, 518, 519, 472, 499, 495, 531, 515, 521, 547, 518, 526, 501, 491, 542] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[492, 498, 456, 532, 504, 558, 509, 527, 528, 485, 498, 526, 515, 529, 458, 508, 458, 531, 521, 492, 536, 490, 560, 537, 505, 537, 496, 486, 500, 481, 541, 515, 525, 519, 501, 489, 510, 498, 497, 514, 481, 522, 503, 522, 514, 515, 516, 504, 505, 492, 498, 535, 488, 529, 515, 504, 554, 520, 575, 516, 532, 533, 483, 550] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[514, 557, 483, 491, 479, 530, 543, 522, 479, 472, 486, 503, 541, 476, 521, 536, 514, 485, 509, 509, 512, 511, 485, 482, 538, 499, 540, 519, 491, 513, 540, 505, 528, 487, 525, 511, 539, 546, 505, 545, 560, 558, 498, 495, 507, 498, 500, 479, 508, 503, 529, 519, 506, 518, 510, 502, 548, 494, 516, 480, 541, 508, 499, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[524, 465, 519, 521, 509, 492, 509, 537, 492, 514, 500, 489, 517, 497, 511, 533, 506, 509, 502, 487, 488, 571, 524, 511, 560, 520, 538, 494, 491, 521, 495, 532, 482, 506, 501, 550, 525, 551, 490, 544, 504, 550, 507, 554, 491, 488, 508, 517, 534, 498, 527, 475, 511, 497, 489, 509, 511, 511, 542, 530, 513, 501, 490, 484] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[517, 504, 472, 489, 497, 513, 480, 496, 541, 522, 491, 557, 550, 506, 519, 536, 519, 505, 552, 484, 499, 491, 511, 523, 528, 526, 500, 475, 500, 499, 504, 524, 514, 504, 516, 541, 501, 509, 483, 517, 527, 525, 504, 492, 534, 488, 540, 504, 509, 506, 506, 530, 534, 536, 506, 535, 521, 517, 475, 494, 528, 514, 507, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[491, 479, 496, 517, 482, 519, 493, 519, 24, 0, 0, 0, 0, 0, 0, 0, 494, 498, 503, 541, 476, 525, 517, 536, 16, 0, 0, 0, 0, 0, 0, 0, 476, 494, 496, 538, 508, 504, 477, 506, 20, 0, 0, 0, 0, 0, 0, 0, 483, 536, 514, 540, 503, 504, 529, 521, 0, 0, 0, 0, 0, 0, 0, 0, 493, 499, 499, 533, 536, 530, 505, 496, 0, 0, 0, 0, 0, 0, 0, 0, 500, 518, 491, 534, 505, 495, 535, 482, 0, 0, 0, 0, 0, 0, 0, 0, 499, 551, 543, 526, 523, 510, 539, 523, 11, 6, 0, 0, 0, 0, 0, 0, 497, 518, 494, 531, 509, 522, 551, 489, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4020, 4106, 4019, 4130, 4091, 4060, 4231, 4111] avg=4096.0000 max=4231 max_rank=6 min=4019 min_rank=2 imbalance_ratio=1.032959 max_load_reduction=28537 max_load_reduction_pct=0.870880 moved_to_echo=16493 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[519, 521, 565, 519, 519, 497, 510, 492, 24, 0, 0, 0, 0, 0, 0, 0, 506, 506, 513, 525, 504, 491, 484, 499, 16, 0, 0, 0, 0, 0, 0, 0, 494, 508, 540, 505, 520, 520, 529, 489, 20, 0, 0, 0, 0, 0, 0, 0, 467, 528, 534, 491, 502, 511, 501, 562, 0, 0, 0, 0, 0, 0, 0, 0, 487, 547, 494, 539, 535, 502, 523, 496, 0, 0, 0, 0, 0, 0, 0, 0, 519, 491, 524, 473, 530, 513, 499, 493, 0, 0, 0, 0, 0, 0, 0, 0, 496, 502, 485, 537, 530, 497, 480, 482, 11, 6, 0, 0, 0, 0, 0, 0, 491, 563, 522, 543, 499, 495, 517, 516, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4166, 4044, 4125, 4096, 4123, 4042, 4026, 4146] avg=4096.0000 max=4166 max_rank=0 min=4026 min_rank=6 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16337 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[548, 542, 489, 504, 505, 523, 521, 541, 23, 0, 0, 0, 0, 0, 0, 0, 507, 514, 490, 525, 479, 506, 518, 528, 16, 0, 0, 0, 0, 0, 0, 0, 476, 527, 542, 505, 533, 473, 530, 493, 19, 0, 0, 0, 0, 0, 0, 0, 500, 501, 476, 512, 528, 505, 494, 480, 0, 0, 0, 0, 0, 0, 0, 0, 536, 502, 498, 506, 539, 533, 523, 505, 0, 0, 0, 0, 0, 0, 0, 0, 506, 535, 471, 547, 538, 523, 495, 490, 0, 0, 0, 0, 0, 0, 0, 0, 501, 482, 518, 519, 472, 499, 495, 531, 12, 5, 0, 0, 0, 0, 0, 0, 468, 521, 547, 518, 526, 501, 491, 542, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4196, 4083, 4098, 3996, 4142, 4105, 4034, 4114] avg=4096.0000 max=4196 max_rank=0 min=3996 min_rank=3 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16395 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[461, 489, 539, 479, 519, 517, 522, 503, 21, 0, 0, 0, 0, 0, 0, 0, 529, 513, 473, 551, 488, 504, 525, 531, 15, 0, 0, 0, 0, 0, 0, 0, 506, 528, 494, 535, 480, 519, 534, 518, 18, 0, 0, 0, 0, 0, 0, 0, 471, 535, 544, 510, 519, 529, 506, 516, 0, 0, 0, 0, 0, 0, 0, 0, 490, 563, 471, 529, 484, 519, 512, 488, 0, 0, 0, 0, 0, 0, 0, 0, 515, 522, 537, 510, 529, 525, 521, 503, 0, 0, 0, 0, 0, 0, 0, 0, 532, 516, 510, 516, 495, 488, 486, 478, 11, 5, 0, 0, 0, 0, 0, 0, 443, 535, 465, 533, 541, 542, 491, 522, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4050, 4129, 4132, 4130, 4056, 4162, 4037, 4072] avg=4096.0000 max=4162 max_rank=5 min=4037 min_rank=6 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16327 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[492, 498, 456, 532, 504, 558, 509, 527, 24, 0, 0, 0, 0, 0, 0, 0, 528, 485, 498, 526, 515, 529, 458, 508, 15, 0, 0, 0, 0, 0, 0, 0, 458, 531, 521, 492, 536, 490, 560, 537, 20, 0, 0, 0, 0, 0, 0, 0, 505, 537, 496, 486, 500, 481, 541, 515, 0, 0, 0, 0, 0, 0, 0, 0, 525, 519, 501, 489, 510, 498, 497, 488, 0, 0, 0, 0, 0, 0, 0, 0, 481, 522, 503, 522, 514, 515, 516, 504, 0, 0, 0, 0, 0, 0, 0, 0, 505, 492, 498, 535, 488, 529, 515, 504, 11, 6, 0, 0, 0, 0, 0, 0, 504, 520, 575, 516, 532, 533, 483, 550, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4062, 4145, 4061, 4027, 4077, 4083, 4213] avg=4096.0000 max=4213 max_rank=7 min=4027 min_rank=4 imbalance_ratio=1.028564 max_load_reduction=28555 max_load_reduction_pct=0.871429 moved_to_echo=16400 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[514, 557, 483, 491, 479, 530, 543, 522, 28, 0, 0, 0, 0, 0, 0, 0, 479, 472, 486, 503, 541, 476, 521, 536, 21, 3, 0, 0, 0, 0, 0, 0, 514, 485, 509, 509, 512, 511, 485, 482, 23, 0, 0, 0, 0, 0, 0, 0, 538, 499, 540, 519, 491, 513, 540, 505, 3, 0, 0, 0, 0, 0, 0, 0, 528, 487, 525, 511, 539, 546, 505, 508, 0, 0, 0, 0, 0, 0, 0, 0, 560, 552, 498, 495, 507, 498, 500, 479, 0, 0, 0, 0, 0, 0, 0, 0, 508, 503, 529, 519, 506, 518, 510, 502, 16, 10, 0, 0, 0, 0, 0, 0, 487, 494, 516, 480, 541, 508, 499, 521, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4147, 4038, 4030, 4148, 4149, 4089, 4121, 4046] avg=4096.0000 max=4149 max_rank=4 min=4030 min_rank=2 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16405 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[524, 465, 519, 521, 509, 492, 509, 537, 22, 0, 0, 0, 0, 0, 0, 0, 492, 514, 500, 489, 517, 497, 511, 533, 16, 0, 0, 0, 0, 0, 0, 0, 506, 509, 502, 487, 488, 571, 524, 511, 18, 0, 0, 0, 0, 0, 0, 0, 560, 520, 538, 494, 491, 521, 495, 532, 0, 0, 0, 0, 0, 0, 0, 0, 482, 506, 501, 550, 525, 551, 490, 516, 0, 0, 0, 0, 0, 0, 0, 0, 504, 550, 507, 554, 491, 488, 508, 517, 0, 0, 0, 0, 0, 0, 0, 0, 534, 498, 527, 475, 511, 497, 489, 509, 12, 5, 0, 0, 0, 0, 0, 0, 466, 511, 542, 530, 513, 501, 490, 484, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4069, 4116, 4151, 4121, 4119, 4057, 4037] avg=4096.0000 max=4151 max_rank=3 min=4037 min_rank=7 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16334 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[517, 504, 472, 489, 497, 513, 480, 496, 23, 0, 0, 0, 0, 0, 0, 0, 541, 522, 491, 557, 550, 506, 519, 536, 15, 0, 0, 0, 0, 0, 0, 0, 519, 505, 552, 484, 499, 491, 511, 523, 19, 0, 0, 0, 0, 0, 0, 0, 528, 526, 500, 475, 500, 499, 504, 524, 0, 0, 0, 0, 0, 0, 0, 0, 514, 504, 516, 541, 501, 509, 483, 491, 0, 0, 0, 0, 0, 0, 0, 0, 527, 525, 504, 492, 534, 488, 540, 504, 0, 0, 0, 0, 0, 0, 0, 0, 509, 506, 506, 530, 534, 536, 506, 535, 11, 6, 0, 0, 0, 0, 0, 0, 473, 517, 475, 494, 528, 514, 507, 521, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3991, 4237, 4103, 4056, 4059, 4114, 4179, 4029] avg=4096.0000 max=4237 max_rank=1 min=3991 min_rank=0 imbalance_ratio=1.034424 max_load_reduction=28531 max_load_reduction_pct=0.870697 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 3968, 4224, 4096, 4096, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3840, 4224, 4224, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3968, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[496, 514, 516, 532, 517, 499, 522, 546, 552, 519, 523, 521, 505, 521, 503, 529, 517, 521, 535, 499, 504, 530, 514, 515, 493, 530, 506, 491, 507, 507, 516, 493, 523, 524, 492, 538, 499, 519, 514, 547, 509, 468, 515, 515, 519, 512, 522, 515, 494, 443, 503, 511, 480, 536, 513, 441, 526, 542, 471, 553, 539, 488, 499, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[513, 499, 564, 529, 497, 560, 498, 521, 513, 482, 522, 498, 531, 502, 491, 503, 474, 508, 509, 533, 496, 537, 508, 492, 489, 510, 535, 477, 507, 523, 544, 547, 526, 516, 472, 520, 493, 521, 503, 491, 518, 529, 500, 479, 521, 516, 512, 517, 496, 501, 520, 490, 534, 564, 528, 515, 514, 517, 504, 535, 497, 479, 495, 533] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[517, 552, 525, 517, 525, 494, 511, 506, 503, 501, 520, 559, 478, 550, 552, 515, 514, 522, 515, 528, 541, 509, 516, 467, 507, 507, 516, 518, 522, 485, 526, 504, 487, 531, 533, 523, 499, 493, 527, 528, 522, 491, 476, 511, 500, 482, 491, 470, 551, 475, 474, 516, 489, 519, 514, 547, 503, 484, 491, 521, 519, 538, 508, 533] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[526, 485, 518, 526, 520, 495, 508, 497, 467, 502, 507, 501, 488, 489, 495, 512, 518, 556, 485, 502, 521, 513, 531, 514, 537, 545, 486, 545, 518, 530, 484, 523, 526, 509, 531, 523, 491, 522, 513, 509, 521, 475, 486, 540, 527, 486, 476, 513, 502, 527, 508, 521, 509, 537, 550, 543, 477, 493, 503, 512, 516, 524, 521, 533] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[520, 475, 465, 494, 505, 521, 527, 514, 516, 510, 488, 511, 545, 526, 509, 509, 450, 490, 509, 480, 528, 505, 509, 524, 488, 507, 532, 531, 527, 472, 559, 501, 515, 516, 565, 538, 514, 529, 537, 521, 540, 486, 510, 524, 543, 499, 496, 527, 497, 511, 474, 525, 515, 520, 521, 490, 535, 504, 502, 532, 532, 500, 520, 483] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[480, 481, 520, 490, 514, 529, 520, 514, 505, 519, 550, 482, 487, 517, 520, 488, 500, 522, 572, 496, 509, 525, 505, 518, 493, 503, 525, 471, 525, 541, 463, 512, 496, 480, 510, 503, 519, 523, 487, 526, 553, 533, 535, 484, 528, 532, 498, 528, 500, 502, 519, 518, 530, 526, 503, 523, 513, 517, 492, 520, 520, 520, 537, 497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[519, 482, 502, 482, 498, 475, 512, 485, 511, 482, 535, 501, 474, 498, 477, 510, 530, 509, 520, 537, 551, 523, 536, 533, 558, 536, 490, 506, 539, 485, 522, 505, 530, 513, 513, 499, 544, 480, 528, 518, 481, 525, 516, 508, 526, 541, 549, 516, 542, 499, 493, 506, 541, 523, 529, 494, 482, 508, 501, 537, 516, 494, 480, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[479, 549, 541, 513, 482, 542, 497, 513, 519, 511, 488, 536, 523, 488, 541, 496, 538, 530, 507, 481, 518, 517, 540, 551, 490, 483, 483, 528, 503, 514, 526, 498, 482, 482, 505, 482, 530, 522, 514, 518, 511, 510, 494, 531, 566, 523, 469, 534, 506, 501, 501, 531, 486, 534, 499, 501, 517, 517, 495, 508, 507, 500, 533, 534] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[496, 514, 516, 532, 517, 499, 522, 546, 10, 0, 0, 0, 0, 0, 0, 0, 552, 519, 523, 521, 505, 521, 503, 529, 17, 0, 0, 0, 0, 0, 0, 0, 517, 521, 535, 499, 476, 530, 514, 515, 0, 0, 0, 0, 0, 0, 0, 0, 493, 530, 506, 491, 507, 507, 516, 493, 3, 0, 0, 0, 0, 0, 0, 0, 523, 524, 492, 538, 499, 519, 514, 534, 0, 0, 0, 0, 0, 0, 0, 0, 509, 468, 515, 515, 519, 512, 522, 515, 2, 0, 0, 0, 0, 0, 0, 0, 494, 443, 503, 511, 480, 536, 513, 441, 5, 1, 0, 0, 0, 0, 0, 0, 526, 542, 471, 553, 539, 488, 499, 505, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4152, 4190, 4107, 4046, 4143, 4077, 3927, 4126] avg=4096.0000 max=4190 max_rank=1 min=3927 min_rank=6 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16273 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[526, 485, 518, 526, 520, 495, 508, 497, 10, 0, 0, 0, 0, 0, 0, 0, 467, 502, 507, 501, 488, 489, 495, 512, 17, 0, 0, 0, 0, 0, 0, 0, 518, 556, 485, 502, 493, 513, 531, 514, 0, 0, 0, 0, 0, 0, 0, 0, 537, 545, 486, 545, 518, 530, 484, 523, 2, 0, 0, 0, 0, 0, 0, 0, 526, 509, 531, 523, 491, 522, 513, 497, 0, 0, 0, 0, 0, 0, 0, 0, 521, 475, 486, 540, 527, 486, 476, 513, 2, 0, 0, 0, 0, 0, 0, 0, 502, 527, 508, 521, 509, 537, 550, 543, 5, 1, 0, 0, 0, 0, 0, 0, 477, 493, 503, 512, 516, 524, 521, 533, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 3978, 4112, 4170, 4112, 4026, 4203, 4082] avg=4096.0000 max=4203 max_rank=6 min=3978 min_rank=1 imbalance_ratio=1.026123 max_load_reduction=28565 max_load_reduction_pct=0.871735 moved_to_echo=16423 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[513, 499, 564, 529, 497, 560, 498, 521, 9, 0, 0, 0, 0, 0, 0, 0, 513, 482, 522, 498, 531, 502, 491, 503, 16, 0, 0, 0, 0, 0, 0, 0, 474, 508, 509, 533, 470, 537, 508, 492, 0, 0, 0, 0, 0, 0, 0, 0, 489, 510, 535, 477, 507, 523, 544, 547, 2, 0, 0, 0, 0, 0, 0, 0, 526, 516, 472, 520, 493, 521, 503, 479, 0, 0, 0, 0, 0, 0, 0, 0, 518, 529, 500, 479, 521, 516, 512, 517, 2, 0, 0, 0, 0, 0, 0, 0, 496, 501, 520, 490, 534, 564, 528, 515, 5, 1, 0, 0, 0, 0, 0, 0, 514, 517, 504, 535, 497, 479, 495, 533, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4190, 4058, 4031, 4134, 4030, 4094, 4154, 4077] avg=4096.0000 max=4190 max_rank=0 min=4030 min_rank=4 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16355 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[517, 552, 525, 517, 525, 494, 511, 506, 10, 0, 0, 0, 0, 0, 0, 0, 503, 501, 520, 559, 478, 550, 552, 515, 18, 0, 0, 0, 0, 0, 0, 0, 514, 522, 515, 528, 512, 509, 516, 467, 0, 0, 0, 0, 0, 0, 0, 0, 507, 507, 516, 518, 522, 485, 526, 504, 3, 0, 0, 0, 0, 0, 0, 0, 487, 531, 533, 523, 499, 493, 527, 515, 0, 0, 0, 0, 0, 0, 0, 0, 522, 491, 476, 511, 500, 482, 491, 470, 2, 0, 0, 0, 0, 0, 0, 0, 551, 475, 474, 516, 489, 519, 514, 547, 5, 1, 0, 0, 0, 0, 0, 0, 503, 484, 491, 521, 519, 538, 508, 533, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4157, 4196, 4083, 4088, 4108, 3945, 4091, 4100] avg=4096.0000 max=4196 max_rank=1 min=3945 min_rank=5 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16244 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[480, 481, 520, 490, 514, 529, 520, 514, 10, 0, 0, 0, 0, 0, 0, 0, 505, 519, 550, 482, 487, 517, 520, 488, 17, 0, 0, 0, 0, 0, 0, 0, 500, 522, 572, 496, 481, 525, 505, 518, 0, 0, 0, 0, 0, 0, 0, 0, 493, 503, 525, 471, 525, 541, 463, 512, 3, 0, 0, 0, 0, 0, 0, 0, 496, 480, 510, 503, 519, 523, 487, 513, 0, 0, 0, 0, 0, 0, 0, 0, 553, 533, 535, 484, 528, 532, 498, 528, 2, 0, 0, 0, 0, 0, 0, 0, 500, 502, 519, 518, 530, 526, 503, 523, 5, 1, 0, 0, 0, 0, 0, 0, 513, 517, 492, 520, 520, 520, 537, 497, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4058, 4085, 4119, 4036, 4031, 4193, 4127, 4119] avg=4096.0000 max=4193 max_rank=5 min=4031 min_rank=4 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16470 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[520, 475, 465, 494, 505, 521, 527, 514, 10, 0, 0, 0, 0, 0, 0, 0, 516, 510, 488, 511, 545, 526, 509, 509, 17, 0, 0, 0, 0, 0, 0, 0, 450, 490, 509, 480, 500, 505, 509, 524, 0, 0, 0, 0, 0, 0, 0, 0, 488, 507, 532, 531, 527, 472, 559, 501, 3, 0, 0, 0, 0, 0, 0, 0, 515, 516, 565, 538, 514, 529, 537, 508, 0, 0, 0, 0, 0, 0, 0, 0, 540, 486, 510, 524, 543, 499, 496, 527, 2, 0, 0, 0, 0, 0, 0, 0, 497, 511, 474, 525, 515, 520, 521, 490, 5, 1, 0, 0, 0, 0, 0, 0, 535, 504, 502, 532, 532, 500, 520, 483, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4031, 4131, 3967, 4120, 4222, 4127, 4059, 4111] avg=4096.0000 max=4222 max_rank=4 min=3967 min_rank=2 imbalance_ratio=1.030762 max_load_reduction=28546 max_load_reduction_pct=0.871155 moved_to_echo=16519 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[519, 482, 502, 482, 498, 475, 512, 485, 14, 0, 0, 0, 0, 0, 0, 0, 511, 482, 535, 501, 474, 498, 477, 510, 23, 0, 0, 0, 0, 0, 0, 0, 530, 509, 520, 537, 507, 523, 536, 533, 0, 0, 0, 0, 0, 0, 0, 0, 558, 536, 490, 506, 539, 485, 522, 505, 6, 0, 0, 0, 0, 0, 0, 0, 530, 513, 513, 499, 544, 480, 528, 492, 0, 0, 0, 0, 0, 0, 0, 0, 481, 525, 516, 508, 526, 541, 549, 516, 4, 0, 0, 0, 0, 0, 0, 0, 542, 499, 493, 506, 541, 523, 529, 494, 8, 7, 0, 0, 0, 0, 0, 0, 482, 508, 501, 537, 516, 494, 480, 513, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3969, 4011, 4195, 4147, 4099, 4166, 4142, 4039] avg=4096.0000 max=4195 max_rank=2 min=3969 min_rank=0 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16446 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[479, 549, 541, 513, 482, 542, 497, 513, 10, 0, 0, 0, 0, 0, 0, 0, 519, 511, 488, 536, 523, 488, 541, 496, 17, 0, 0, 0, 0, 0, 0, 0, 538, 530, 507, 481, 490, 517, 540, 551, 0, 0, 0, 0, 0, 0, 0, 0, 490, 483, 483, 528, 503, 514, 526, 498, 2, 0, 0, 0, 0, 0, 0, 0, 482, 482, 505, 482, 530, 522, 514, 506, 0, 0, 0, 0, 0, 0, 0, 0, 511, 510, 494, 531, 566, 523, 469, 534, 2, 0, 0, 0, 0, 0, 0, 0, 506, 501, 501, 531, 486, 534, 499, 501, 5, 1, 0, 0, 0, 0, 0, 0, 517, 517, 495, 508, 507, 500, 533, 534, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4119, 4154, 4027, 4023, 4140, 4065, 4114] avg=4096.0000 max=4154 max_rank=2 min=4023 min_rank=4 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16342 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 3968, 4224, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 3968, 4096, 4224, 4096, 4352, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 3968, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4352, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[507, 521, 507, 528, 526, 525, 504, 530, 481, 483, 521, 497, 526, 480, 489, 505, 503, 508, 514, 544, 520, 520, 505, 529, 541, 449, 529, 516, 514, 517, 505, 516, 533, 449, 539, 520, 534, 484, 485, 512, 515, 539, 543, 530, 557, 503, 565, 537, 473, 484, 488, 534, 490, 481, 492, 499, 543, 535, 521, 515, 498, 507, 512, 491] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[501, 513, 523, 509, 526, 490, 519, 515, 478, 501, 500, 497, 520, 515, 510, 467, 518, 499, 524, 515, 515, 521, 501, 492, 528, 497, 534, 511, 539, 539, 520, 471, 552, 533, 500, 511, 527, 528, 518, 495, 530, 523, 503, 500, 532, 554, 504, 526, 482, 527, 483, 489, 489, 549, 511, 504, 524, 515, 497, 524, 508, 513, 509, 500] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[523, 466, 532, 491, 510, 522, 532, 507, 504, 479, 490, 530, 502, 505, 515, 501, 511, 515, 488, 532, 468, 489, 506, 513, 526, 517, 520, 480, 537, 506, 552, 509, 494, 487, 512, 512, 522, 486, 485, 520, 555, 487, 561, 527, 504, 527, 486, 507, 471, 526, 530, 537, 511, 530, 481, 565, 525, 490, 507, 519, 550, 550, 518, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[545, 510, 483, 477, 531, 559, 530, 492, 521, 528, 530, 498, 493, 529, 540, 498, 524, 481, 481, 515, 539, 504, 545, 536, 499, 498, 503, 506, 478, 502, 533, 479, 517, 497, 475, 504, 542, 539, 535, 470, 503, 519, 504, 504, 482, 527, 509, 491, 526, 501, 509, 501, 485, 549, 492, 490, 550, 498, 527, 544, 536, 492, 533, 530] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[512, 493, 547, 472, 521, 502, 491, 498, 514, 518, 512, 516, 503, 519, 526, 475, 577, 533, 509, 503, 534, 482, 504, 498, 501, 520, 498, 508, 515, 514, 528, 509, 483, 479, 513, 526, 534, 499, 553, 505, 538, 547, 529, 491, 494, 532, 484, 519, 478, 480, 515, 533, 498, 508, 508, 552, 516, 540, 529, 495, 516, 525, 494, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[491, 520, 546, 504, 539, 473, 464, 503, 515, 513, 506, 541, 545, 493, 491, 538, 555, 510, 471, 518, 480, 505, 519, 495, 494, 533, 501, 529, 482, 497, 509, 527, 539, 482, 513, 501, 503, 502, 538, 527, 526, 524, 471, 531, 480, 512, 518, 505, 500, 515, 516, 525, 469, 543, 533, 524, 548, 550, 491, 507, 503, 532, 512, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[491, 505, 489, 534, 505, 557, 504, 524, 521, 539, 487, 494, 525, 482, 492, 476, 507, 504, 511, 504, 520, 506, 493, 557, 505, 501, 494, 527, 531, 563, 511, 539, 503, 526, 528, 530, 567, 535, 471, 560, 512, 513, 527, 534, 488, 484, 536, 519, 496, 505, 511, 495, 496, 501, 530, 472, 528, 506, 503, 527, 502, 479, 521, 465] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[503, 490, 544, 502, 507, 507, 519, 484, 522, 479, 505, 503, 485, 489, 521, 501, 555, 530, 555, 514, 517, 530, 502, 506, 521, 490, 522, 503, 505, 535, 535, 524, 527, 502, 515, 481, 531, 497, 523, 536, 504, 511, 546, 508, 526, 509, 488, 508, 520, 528, 500, 518, 532, 484, 492, 492, 524, 523, 519, 540, 496, 513, 498, 472] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[507, 521, 507, 528, 526, 525, 504, 530, 8, 0, 0, 0, 0, 0, 0, 0, 481, 483, 521, 497, 526, 480, 489, 505, 38, 11, 0, 0, 0, 0, 0, 0, 490, 508, 514, 544, 520, 520, 505, 529, 0, 0, 0, 0, 0, 0, 0, 0, 541, 449, 529, 516, 514, 517, 497, 516, 0, 0, 0, 0, 0, 0, 0, 0, 533, 449, 539, 520, 525, 484, 485, 512, 0, 0, 0, 0, 0, 0, 0, 0, 515, 539, 505, 530, 557, 503, 565, 537, 0, 0, 0, 0, 0, 0, 0, 0, 473, 484, 488, 534, 490, 481, 492, 499, 16, 13, 9, 0, 0, 0, 0, 0, 516, 535, 521, 515, 498, 507, 512, 491, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4156, 4031, 4130, 4079, 4047, 4251, 3979, 4095] avg=4096.0000 max=4251 max_rank=5 min=3979 min_rank=6 imbalance_ratio=1.037842 max_load_reduction=28517 max_load_reduction_pct=0.870270 moved_to_echo=16372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[501, 513, 523, 509, 526, 490, 519, 515, 9, 0, 0, 0, 0, 0, 0, 0, 478, 501, 500, 497, 520, 515, 510, 467, 36, 10, 0, 0, 0, 0, 0, 0, 505, 499, 524, 515, 515, 521, 501, 492, 0, 0, 0, 0, 0, 0, 0, 0, 528, 497, 534, 511, 539, 539, 511, 471, 0, 0, 0, 0, 0, 0, 0, 0, 552, 533, 500, 511, 518, 528, 518, 495, 0, 0, 0, 0, 0, 0, 0, 0, 530, 523, 467, 500, 532, 554, 504, 526, 0, 0, 0, 0, 0, 0, 0, 0, 482, 527, 483, 489, 489, 549, 511, 504, 15, 13, 9, 0, 0, 0, 0, 0, 499, 515, 497, 524, 508, 513, 509, 500, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4034, 4072, 4130, 4155, 4136, 4071, 4065] avg=4096.0000 max=4155 max_rank=4 min=4034 min_rank=1 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16427 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[523, 466, 532, 491, 510, 522, 532, 507, 9, 0, 0, 0, 0, 0, 0, 0, 504, 479, 490, 530, 502, 505, 515, 501, 40, 10, 0, 0, 0, 0, 0, 0, 498, 515, 488, 532, 468, 489, 506, 513, 0, 0, 0, 0, 0, 0, 0, 0, 526, 517, 520, 480, 537, 506, 543, 509, 0, 0, 0, 0, 0, 0, 0, 0, 494, 487, 512, 512, 513, 486, 485, 520, 0, 0, 0, 0, 0, 0, 0, 0, 555, 487, 521, 527, 504, 527, 486, 507, 0, 0, 0, 0, 0, 0, 0, 0, 471, 526, 530, 537, 511, 530, 481, 565, 16, 13, 9, 0, 0, 0, 0, 0, 499, 490, 507, 519, 550, 550, 518, 508, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4076, 4009, 4138, 4009, 4114, 4189, 4141] avg=4096.0000 max=4189 max_rank=6 min=4009 min_rank=2 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16453 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[545, 510, 483, 477, 531, 559, 530, 492, 9, 0, 0, 0, 0, 0, 0, 0, 521, 528, 530, 498, 493, 529, 540, 498, 36, 11, 0, 0, 0, 0, 0, 0, 511, 481, 481, 515, 539, 504, 545, 536, 0, 0, 0, 0, 0, 0, 0, 0, 499, 498, 503, 506, 478, 502, 524, 479, 0, 0, 0, 0, 0, 0, 0, 0, 517, 497, 475, 504, 533, 539, 535, 470, 0, 0, 0, 0, 0, 0, 0, 0, 503, 519, 468, 504, 482, 527, 509, 491, 0, 0, 0, 0, 0, 0, 0, 0, 526, 501, 509, 501, 485, 549, 492, 490, 16, 13, 9, 0, 0, 0, 0, 0, 523, 498, 527, 544, 536, 492, 533, 530, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4136, 4184, 4112, 3989, 4070, 4003, 4091, 4183] avg=4096.0000 max=4184 max_rank=1 min=3989 min_rank=3 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[512, 493, 547, 472, 521, 502, 491, 498, 9, 0, 0, 0, 0, 0, 0, 0, 514, 518, 512, 516, 503, 519, 526, 475, 37, 10, 0, 0, 0, 0, 0, 0, 562, 533, 509, 503, 534, 482, 504, 498, 0, 0, 0, 0, 0, 0, 0, 0, 501, 520, 498, 508, 515, 514, 519, 509, 0, 0, 0, 0, 0, 0, 0, 0, 483, 479, 513, 526, 525, 499, 553, 505, 0, 0, 0, 0, 0, 0, 0, 0, 538, 547, 492, 491, 494, 532, 484, 519, 0, 0, 0, 0, 0, 0, 0, 0, 478, 480, 515, 533, 498, 508, 508, 552, 15, 15, 9, 0, 0, 0, 0, 0, 491, 540, 529, 495, 516, 525, 494, 503, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4045, 4130, 4125, 4084, 4083, 4097, 4111, 4093] avg=4096.0000 max=4130 max_rank=1 min=4045 min_rank=0 imbalance_ratio=1.008301 max_load_reduction=28638 max_load_reduction_pct=0.873962 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[491, 505, 489, 534, 505, 557, 504, 524, 12, 0, 0, 0, 0, 0, 0, 0, 521, 539, 487, 494, 525, 482, 492, 476, 41, 16, 0, 0, 0, 0, 0, 0, 491, 504, 511, 504, 520, 506, 493, 557, 0, 0, 0, 0, 0, 0, 0, 0, 505, 501, 494, 527, 531, 563, 498, 539, 0, 0, 0, 0, 0, 0, 0, 0, 503, 526, 528, 530, 552, 535, 471, 560, 0, 0, 0, 0, 0, 0, 0, 0, 512, 513, 486, 534, 488, 484, 536, 519, 0, 0, 0, 0, 0, 0, 0, 0, 496, 505, 511, 495, 496, 501, 530, 472, 21, 16, 15, 1, 0, 0, 0, 0, 491, 506, 503, 527, 502, 479, 521, 465, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4073, 4086, 4158, 4205, 4072, 4059, 3994] avg=4096.0000 max=4205 max_rank=4 min=3994 min_rank=7 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16330 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[491, 520, 546, 504, 539, 473, 464, 503, 8, 0, 0, 0, 0, 0, 0, 0, 515, 513, 506, 541, 545, 493, 491, 538, 33, 11, 0, 0, 0, 0, 0, 0, 541, 510, 471, 518, 480, 505, 519, 495, 0, 0, 0, 0, 0, 0, 0, 0, 494, 533, 501, 529, 482, 497, 501, 527, 0, 0, 0, 0, 0, 0, 0, 0, 539, 482, 513, 501, 494, 502, 538, 527, 0, 0, 0, 0, 0, 0, 0, 0, 526, 524, 438, 531, 480, 512, 518, 505, 0, 0, 0, 0, 0, 0, 0, 0, 500, 515, 516, 525, 469, 543, 533, 524, 16, 14, 9, 0, 0, 0, 0, 0, 521, 550, 491, 507, 503, 532, 512, 521, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4048, 4186, 4039, 4064, 4096, 4034, 4164, 4137] avg=4096.0000 max=4186 max_rank=1 min=4034 min_rank=5 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[503, 490, 544, 502, 507, 507, 519, 484, 9, 0, 0, 0, 0, 0, 0, 0, 522, 479, 505, 503, 485, 489, 521, 501, 39, 10, 0, 0, 0, 0, 0, 0, 541, 530, 555, 514, 517, 530, 502, 506, 0, 0, 0, 0, 0, 0, 0, 0, 521, 490, 522, 503, 505, 535, 526, 524, 0, 0, 0, 0, 0, 0, 0, 0, 527, 502, 515, 481, 522, 497, 523, 536, 0, 0, 0, 0, 0, 0, 0, 0, 504, 511, 507, 508, 526, 509, 488, 508, 0, 0, 0, 0, 0, 0, 0, 0, 520, 528, 500, 518, 532, 484, 492, 492, 15, 14, 9, 0, 0, 0, 0, 0, 499, 523, 519, 540, 496, 513, 498, 472, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4065, 4054, 4195, 4126, 4103, 4061, 4104, 4060] avg=4096.0000 max=4195 max_rank=2 min=4054 min_rank=1 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16328 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 3968, 4096, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[3968, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 256, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 3968, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4096, 3968, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[504, 520, 548, 491, 516, 518, 496, 491, 485, 506, 491, 499, 521, 502, 516, 504, 500, 536, 530, 576, 492, 519, 492, 525, 520, 506, 494, 483, 551, 525, 513, 534, 506, 502, 492, 486, 521, 540, 520, 517, 540, 536, 524, 526, 522, 475, 518, 490, 519, 496, 526, 493, 517, 524, 503, 506, 534, 501, 498, 504, 519, 522, 494, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[479, 509, 530, 487, 547, 496, 503, 499, 482, 543, 541, 476, 515, 506, 541, 523, 543, 533, 461, 496, 531, 489, 520, 505, 530, 534, 502, 498, 506, 520, 543, 500, 564, 477, 520, 497, 493, 509, 510, 525, 557, 508, 533, 547, 507, 482, 493, 537, 486, 520, 516, 511, 525, 508, 490, 514, 485, 505, 513, 541, 506, 485, 506, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[496, 524, 503, 525, 521, 491, 511, 506, 537, 499, 474, 536, 534, 507, 544, 498, 483, 495, 489, 508, 501, 492, 515, 460, 496, 485, 497, 542, 508, 540, 529, 493, 519, 523, 516, 491, 512, 519, 537, 537, 532, 538, 541, 467, 487, 565, 513, 535, 487, 520, 514, 526, 490, 503, 476, 533, 512, 508, 522, 518, 517, 505, 559, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[571, 506, 519, 514, 499, 492, 543, 518, 528, 489, 482, 529, 561, 514, 512, 487, 527, 525, 496, 516, 518, 505, 506, 517, 508, 489, 506, 532, 504, 484, 484, 517, 489, 529, 496, 535, 507, 547, 510, 492, 501, 518, 494, 533, 518, 501, 531, 525, 502, 499, 520, 537, 507, 495, 543, 474, 514, 483, 492, 500, 535, 486, 514, 543] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[538, 510, 514, 544, 510, 567, 501, 515, 502, 485, 517, 495, 503, 494, 495, 508, 530, 473, 513, 505, 489, 527, 485, 559, 503, 518, 551, 551, 523, 536, 494, 528, 510, 473, 544, 493, 511, 492, 500, 494, 519, 539, 510, 483, 520, 508, 497, 518, 477, 495, 517, 524, 518, 483, 548, 511, 550, 510, 517, 494, 507, 514, 515, 494] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[534, 484, 551, 513, 516, 513, 538, 545, 523, 548, 569, 516, 493, 505, 506, 493, 502, 475, 497, 555, 483, 530, 497, 505, 496, 488, 508, 504, 523, 507, 485, 533, 570, 520, 483, 515, 503, 486, 504, 532, 481, 506, 461, 517, 474, 544, 519, 468, 488, 515, 527, 499, 540, 493, 505, 526, 513, 563, 501, 553, 506, 531, 478, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[500, 514, 511, 523, 517, 495, 484, 519, 514, 520, 555, 531, 502, 512, 520, 493, 530, 478, 520, 493, 533, 508, 522, 497, 541, 535, 504, 498, 501, 509, 507, 500, 531, 538, 513, 521, 499, 468, 493, 504, 462, 511, 490, 508, 536, 505, 515, 511, 554, 548, 509, 544, 527, 547, 515, 518, 511, 481, 512, 504, 486, 508, 500, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[488, 488, 543, 524, 500, 489, 523, 521, 500, 503, 489, 526, 521, 503, 502, 526, 486, 520, 513, 481, 520, 469, 532, 489, 499, 529, 538, 544, 510, 545, 494, 518, 513, 517, 538, 520, 487, 514, 510, 523, 523, 511, 507, 537, 524, 530, 488, 515, 511, 517, 511, 484, 541, 513, 511, 552, 475, 487, 505, 522, 506, 533, 493, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[479, 509, 513, 487, 547, 496, 503, 499, 0, 0, 0, 0, 0, 0, 0, 0, 482, 543, 541, 476, 515, 506, 541, 523, 2, 0, 0, 0, 0, 0, 0, 0, 543, 533, 461, 496, 531, 489, 520, 505, 17, 14, 0, 0, 0, 0, 0, 0, 530, 534, 502, 498, 506, 505, 543, 500, 0, 0, 0, 0, 0, 0, 0, 0, 564, 477, 520, 497, 493, 509, 510, 525, 1, 0, 0, 0, 0, 0, 0, 0, 557, 501, 533, 547, 507, 482, 493, 537, 0, 0, 0, 0, 0, 0, 0, 0, 486, 520, 516, 511, 515, 508, 490, 514, 0, 0, 0, 0, 0, 0, 0, 0, 485, 505, 513, 541, 506, 485, 506, 510, 10, 4, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4033, 4129, 4109, 4118, 4096, 4157, 4060, 4066] avg=4096.0000 max=4157 max_rank=5 min=4033 min_rank=0 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[504, 520, 531, 491, 516, 518, 496, 491, 0, 0, 0, 0, 0, 0, 0, 0, 485, 506, 491, 499, 521, 502, 516, 504, 2, 0, 0, 0, 0, 0, 0, 0, 500, 536, 530, 576, 492, 519, 492, 525, 17, 14, 0, 0, 0, 0, 0, 0, 520, 506, 494, 483, 551, 510, 513, 534, 0, 0, 0, 0, 0, 0, 0, 0, 506, 502, 492, 486, 521, 540, 520, 517, 1, 0, 0, 0, 0, 0, 0, 0, 540, 529, 524, 526, 522, 475, 518, 490, 0, 0, 0, 0, 0, 0, 0, 0, 519, 496, 526, 493, 508, 524, 503, 506, 0, 0, 0, 0, 0, 0, 0, 0, 534, 501, 498, 504, 519, 522, 494, 493, 9, 4, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4026, 4201, 4111, 4085, 4124, 4075, 4079] avg=4096.0000 max=4201 max_rank=2 min=4026 min_rank=1 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[496, 524, 487, 525, 521, 491, 511, 506, 0, 0, 0, 0, 0, 0, 0, 0, 537, 499, 474, 536, 534, 507, 544, 498, 2, 0, 0, 0, 0, 0, 0, 0, 483, 495, 489, 508, 501, 492, 515, 460, 16, 14, 0, 0, 0, 0, 0, 0, 496, 485, 497, 542, 508, 525, 529, 493, 0, 0, 0, 0, 0, 0, 0, 0, 519, 523, 516, 491, 512, 519, 537, 537, 1, 0, 0, 0, 0, 0, 0, 0, 532, 531, 541, 467, 487, 565, 513, 535, 0, 0, 0, 0, 0, 0, 0, 0, 487, 520, 514, 526, 481, 503, 476, 533, 0, 0, 0, 0, 0, 0, 0, 0, 512, 508, 522, 518, 517, 505, 559, 507, 9, 4, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4131, 3973, 4075, 4155, 4171, 4040, 4162] avg=4096.0000 max=4171 max_rank=5 min=3973 min_rank=2 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16528 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[538, 510, 493, 544, 510, 567, 501, 515, 0, 0, 0, 0, 0, 0, 0, 0, 502, 485, 517, 495, 503, 494, 495, 508, 3, 0, 0, 0, 0, 0, 0, 0, 530, 473, 513, 505, 489, 527, 485, 559, 21, 19, 0, 0, 0, 0, 0, 0, 503, 518, 551, 551, 523, 513, 494, 528, 0, 0, 0, 0, 0, 0, 0, 0, 510, 473, 544, 493, 511, 492, 500, 494, 4, 0, 0, 0, 0, 0, 0, 0, 519, 525, 510, 483, 520, 508, 497, 518, 0, 0, 0, 0, 0, 0, 0, 0, 477, 495, 517, 524, 505, 483, 548, 511, 0, 0, 0, 0, 0, 0, 0, 0, 550, 510, 517, 494, 507, 514, 515, 494, 13, 7, 4, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4178, 4002, 4121, 4181, 4021, 4080, 4060, 4125] avg=4096.0000 max=4181 max_rank=3 min=4002 min_rank=1 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16286 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[571, 506, 503, 514, 499, 492, 543, 518, 0, 0, 0, 0, 0, 0, 0, 0, 528, 489, 482, 529, 561, 514, 512, 487, 2, 0, 0, 0, 0, 0, 0, 0, 527, 525, 496, 516, 518, 505, 506, 517, 16, 13, 0, 0, 0, 0, 0, 0, 508, 489, 506, 532, 504, 470, 484, 517, 0, 0, 0, 0, 0, 0, 0, 0, 489, 529, 496, 535, 507, 547, 510, 492, 1, 0, 0, 0, 0, 0, 0, 0, 501, 511, 494, 533, 518, 501, 531, 525, 0, 0, 0, 0, 0, 0, 0, 0, 502, 499, 520, 537, 498, 495, 543, 474, 0, 0, 0, 0, 0, 0, 0, 0, 514, 483, 492, 500, 535, 486, 514, 543, 9, 4, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4146, 4104, 4139, 4010, 4106, 4114, 4068, 4081] avg=4096.0000 max=4146 max_rank=0 min=4010 min_rank=3 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16369 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[500, 514, 495, 523, 517, 495, 484, 519, 0, 0, 0, 0, 0, 0, 0, 0, 514, 520, 555, 531, 502, 512, 520, 493, 2, 0, 0, 0, 0, 0, 0, 0, 530, 478, 520, 493, 533, 508, 522, 497, 16, 13, 0, 0, 0, 0, 0, 0, 541, 535, 504, 498, 501, 495, 507, 500, 0, 0, 0, 0, 0, 0, 0, 0, 531, 538, 513, 521, 499, 468, 493, 504, 1, 0, 0, 0, 0, 0, 0, 0, 462, 504, 490, 508, 536, 505, 515, 511, 0, 0, 0, 0, 0, 0, 0, 0, 554, 548, 509, 544, 517, 547, 515, 518, 0, 0, 0, 0, 0, 0, 0, 0, 511, 481, 512, 504, 486, 508, 500, 513, 10, 4, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4047, 4149, 4110, 4081, 4068, 4031, 4252, 4030] avg=4096.0000 max=4252 max_rank=6 min=4030 min_rank=7 imbalance_ratio=1.038086 max_load_reduction=28516 max_load_reduction_pct=0.870239 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[534, 484, 534, 513, 516, 513, 538, 545, 0, 0, 0, 0, 0, 0, 0, 0, 523, 548, 569, 516, 493, 505, 506, 493, 2, 0, 0, 0, 0, 0, 0, 0, 502, 475, 497, 555, 483, 530, 497, 505, 17, 13, 0, 0, 0, 0, 0, 0, 496, 488, 508, 504, 523, 493, 485, 533, 0, 0, 0, 0, 0, 0, 0, 0, 570, 520, 483, 515, 503, 486, 504, 532, 1, 0, 0, 0, 0, 0, 0, 0, 481, 499, 461, 517, 474, 544, 519, 468, 0, 0, 0, 0, 0, 0, 0, 0, 488, 515, 527, 499, 530, 493, 505, 526, 0, 0, 0, 0, 0, 0, 0, 0, 513, 563, 501, 553, 506, 531, 478, 512, 10, 4, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4177, 4155, 4074, 4030, 4114, 3963, 4083, 4172] avg=4096.0000 max=4177 max_rank=0 min=3963 min_rank=5 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16332 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[488, 488, 526, 524, 500, 489, 523, 521, 0, 0, 0, 0, 0, 0, 0, 0, 500, 503, 489, 526, 521, 503, 502, 526, 2, 0, 0, 0, 0, 0, 0, 0, 486, 520, 513, 481, 520, 469, 532, 489, 17, 14, 0, 0, 0, 0, 0, 0, 499, 529, 538, 544, 510, 530, 494, 518, 0, 0, 0, 0, 0, 0, 0, 0, 513, 517, 538, 520, 487, 514, 510, 523, 1, 0, 0, 0, 0, 0, 0, 0, 523, 504, 507, 537, 524, 530, 488, 515, 0, 0, 0, 0, 0, 0, 0, 0, 511, 517, 511, 484, 531, 513, 511, 552, 0, 0, 0, 0, 0, 0, 0, 0, 475, 487, 505, 522, 506, 533, 493, 517, 10, 4, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4059, 4072, 4041, 4162, 4123, 4128, 4130, 4053] avg=4096.0000 max=4162 max_rank=3 min=4041 min_rank=2 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16434 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[541, 572, 509, 487, 531, 502, 508, 497, 492, 528, 482, 500, 509, 535, 518, 509, 490, 506, 511, 509, 507, 483, 519, 521, 532, 506, 522, 491, 558, 519, 517, 532, 502, 508, 536, 549, 487, 532, 501, 518, 538, 567, 487, 531, 482, 531, 506, 487, 516, 509, 481, 525, 529, 504, 519, 473, 509, 494, 498, 472, 481, 502, 548, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[490, 539, 545, 510, 518, 487, 497, 516, 525, 510, 506, 504, 445, 520, 534, 487, 482, 505, 497, 497, 565, 510, 537, 538, 485, 514, 551, 517, 497, 519, 494, 527, 480, 527, 492, 496, 503, 504, 500, 537, 567, 537, 485, 517, 553, 549, 491, 517, 466, 501, 530, 570, 482, 535, 505, 524, 525, 460, 516, 511, 497, 505, 526, 492] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[498, 510, 498, 534, 565, 499, 524, 507, 545, 541, 491, 504, 532, 526, 468, 500, 508, 509, 556, 510, 514, 479, 506, 529, 475, 509, 511, 508, 511, 497, 508, 522, 492, 499, 529, 492, 508, 480, 475, 529, 520, 492, 556, 481, 536, 508, 504, 521, 529, 503, 520, 496, 552, 489, 519, 500, 503, 469, 492, 530, 548, 529, 524, 549] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[520, 575, 514, 557, 518, 533, 477, 495, 510, 519, 507, 504, 527, 474, 541, 490, 535, 515, 498, 480, 511, 510, 491, 505, 503, 516, 501, 475, 521, 525, 539, 507, 481, 494, 481, 528, 514, 477, 506, 517, 488, 567, 541, 495, 541, 502, 502, 497, 500, 521, 495, 476, 512, 513, 534, 512, 509, 520, 517, 498, 579, 536, 506, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[500, 528, 535, 504, 499, 503, 517, 499, 517, 525, 535, 511, 532, 557, 532, 490, 499, 497, 513, 479, 484, 508, 532, 485, 504, 474, 503, 529, 482, 502, 526, 511, 493, 509, 508, 530, 515, 518, 541, 527, 518, 503, 528, 512, 502, 507, 508, 501, 524, 495, 539, 496, 552, 486, 519, 503, 508, 500, 527, 473, 554, 545, 516, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[510, 506, 485, 503, 487, 517, 532, 513, 533, 526, 525, 512, 499, 535, 536, 522, 517, 475, 477, 511, 484, 471, 488, 509, 516, 509, 512, 545, 495, 490, 445, 551, 534, 546, 509, 531, 495, 496, 491, 519, 513, 485, 518, 545, 543, 529, 512, 508, 559, 506, 502, 508, 485, 517, 486, 498, 479, 527, 556, 551, 529, 519, 506, 530] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[515, 559, 520, 540, 518, 508, 497, 537, 542, 538, 535, 497, 510, 510, 498, 502, 522, 455, 550, 507, 508, 474, 498, 534, 508, 523, 510, 497, 506, 513, 530, 513, 504, 527, 526, 499, 515, 510, 496, 470, 522, 524, 501, 516, 476, 504, 539, 545, 561, 520, 523, 501, 507, 467, 506, 515, 489, 450, 493, 509, 511, 543, 518, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[515, 497, 512, 477, 496, 528, 495, 493, 529, 516, 488, 515, 523, 493, 513, 487, 502, 529, 503, 483, 519, 493, 532, 511, 519, 508, 516, 502, 499, 517, 533, 543, 512, 529, 493, 508, 537, 494, 530, 525, 524, 511, 486, 537, 504, 504, 509, 476, 478, 529, 513, 479, 534, 520, 549, 548, 498, 514, 541, 507, 512, 522, 509, 550] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[490, 521, 545, 510, 518, 487, 497, 516, 0, 0, 0, 0, 0, 0, 0, 0, 525, 499, 506, 504, 445, 520, 534, 487, 0, 0, 0, 0, 0, 0, 0, 0, 482, 505, 497, 497, 565, 510, 537, 538, 32, 18, 0, 0, 0, 0, 0, 0, 485, 514, 551, 517, 497, 519, 494, 527, 3, 0, 0, 0, 0, 0, 0, 0, 480, 527, 492, 496, 503, 504, 500, 537, 11, 6, 0, 0, 0, 0, 0, 0, 535, 537, 485, 517, 553, 549, 491, 517, 0, 0, 0, 0, 0, 0, 0, 0, 466, 501, 530, 570, 482, 535, 505, 524, 0, 0, 0, 0, 0, 0, 0, 0, 525, 460, 516, 511, 488, 505, 526, 492, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4020, 4181, 4107, 4056, 4184, 4113, 4023] avg=4096.0000 max=4184 max_rank=5 min=4020 min_rank=1 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16376 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[541, 553, 509, 487, 531, 502, 508, 497, 0, 0, 0, 0, 0, 0, 0, 0, 492, 516, 482, 500, 509, 535, 518, 509, 0, 0, 0, 0, 0, 0, 0, 0, 490, 506, 511, 509, 507, 483, 519, 521, 30, 19, 0, 0, 0, 0, 0, 0, 532, 506, 522, 491, 558, 519, 517, 532, 3, 0, 0, 0, 0, 0, 0, 0, 502, 508, 536, 549, 487, 532, 501, 518, 12, 6, 0, 0, 0, 0, 0, 0, 508, 567, 487, 531, 482, 531, 506, 487, 0, 0, 0, 0, 0, 0, 0, 0, 516, 509, 481, 525, 529, 504, 519, 473, 0, 0, 0, 0, 0, 0, 0, 0, 509, 494, 498, 472, 472, 502, 548, 503, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4061, 4095, 4180, 4151, 4099, 4056, 3998] avg=4096.0000 max=4180 max_rank=3 min=3998 min_rank=7 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16304 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[500, 510, 535, 504, 499, 503, 517, 499, 0, 0, 0, 0, 0, 0, 0, 0, 517, 513, 535, 511, 532, 557, 532, 490, 0, 0, 0, 0, 0, 0, 0, 0, 499, 497, 513, 479, 484, 508, 532, 485, 29, 18, 0, 0, 0, 0, 0, 0, 504, 474, 503, 529, 482, 502, 526, 511, 3, 0, 0, 0, 0, 0, 0, 0, 493, 509, 508, 530, 515, 518, 541, 527, 12, 7, 0, 0, 0, 0, 0, 0, 489, 503, 528, 512, 502, 507, 508, 501, 0, 0, 0, 0, 0, 0, 0, 0, 524, 495, 539, 496, 552, 486, 519, 503, 0, 0, 0, 0, 0, 0, 0, 0, 508, 500, 527, 473, 544, 545, 516, 499, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4187, 4044, 4034, 4160, 4050, 4114, 4112] avg=4096.0000 max=4187 max_rank=1 min=4034 min_rank=3 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16436 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[498, 493, 498, 534, 565, 499, 524, 507, 0, 0, 0, 0, 0, 0, 0, 0, 545, 529, 491, 504, 532, 526, 468, 500, 0, 0, 0, 0, 0, 0, 0, 0, 508, 509, 556, 510, 514, 479, 506, 529, 29, 17, 0, 0, 0, 0, 0, 0, 475, 509, 511, 508, 511, 497, 508, 522, 3, 0, 0, 0, 0, 0, 0, 0, 492, 499, 529, 492, 508, 480, 475, 529, 12, 7, 0, 0, 0, 0, 0, 0, 491, 492, 556, 481, 536, 508, 504, 521, 0, 0, 0, 0, 0, 0, 0, 0, 529, 503, 520, 496, 552, 489, 519, 500, 0, 0, 0, 0, 0, 0, 0, 0, 503, 469, 492, 530, 538, 529, 524, 549, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4095, 4157, 4044, 4023, 4089, 4108, 4134] avg=4096.0000 max=4157 max_rank=2 min=4023 min_rank=4 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16354 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[510, 483, 485, 503, 487, 517, 532, 513, 0, 0, 0, 0, 0, 0, 0, 0, 533, 511, 525, 512, 499, 535, 536, 522, 0, 0, 0, 0, 0, 0, 0, 0, 517, 475, 477, 511, 484, 471, 488, 509, 33, 21, 0, 0, 0, 0, 0, 0, 516, 509, 512, 545, 495, 490, 445, 551, 7, 0, 0, 0, 0, 0, 0, 0, 534, 546, 509, 531, 495, 496, 491, 519, 15, 12, 2, 0, 0, 0, 0, 0, 480, 485, 518, 545, 543, 529, 512, 508, 0, 0, 0, 0, 0, 0, 0, 0, 559, 506, 502, 508, 485, 517, 486, 498, 3, 0, 0, 0, 0, 0, 0, 0, 479, 527, 556, 551, 507, 519, 506, 530, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4030, 4173, 3986, 4070, 4150, 4120, 4064, 4175] avg=4096.0000 max=4175 max_rank=7 min=3986 min_rank=2 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16509 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[520, 556, 514, 557, 518, 533, 477, 495, 0, 0, 0, 0, 0, 0, 0, 0, 510, 507, 507, 504, 527, 474, 541, 490, 0, 0, 0, 0, 0, 0, 0, 0, 535, 515, 498, 480, 511, 510, 491, 505, 27, 19, 0, 0, 0, 0, 0, 0, 503, 516, 501, 475, 521, 525, 539, 507, 3, 0, 0, 0, 0, 0, 0, 0, 481, 494, 481, 528, 514, 477, 506, 517, 12, 7, 0, 0, 0, 0, 0, 0, 461, 567, 541, 495, 541, 502, 502, 497, 0, 0, 0, 0, 0, 0, 0, 0, 500, 521, 495, 476, 512, 513, 534, 512, 0, 0, 0, 0, 0, 0, 0, 0, 509, 520, 517, 498, 569, 536, 506, 516, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4170, 4060, 4091, 4090, 4017, 4106, 4063, 4171] avg=4096.0000 max=4171 max_rank=7 min=4017 min_rank=4 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16357 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[515, 540, 520, 540, 518, 508, 497, 537, 0, 0, 0, 0, 0, 0, 0, 0, 542, 526, 535, 497, 510, 510, 498, 502, 0, 0, 0, 0, 0, 0, 0, 0, 522, 455, 550, 507, 508, 474, 498, 534, 29, 19, 0, 0, 0, 0, 0, 0, 508, 523, 510, 497, 506, 513, 530, 513, 3, 0, 0, 0, 0, 0, 0, 0, 504, 527, 526, 499, 515, 510, 496, 470, 12, 6, 0, 0, 0, 0, 0, 0, 493, 524, 501, 516, 476, 504, 539, 545, 0, 0, 0, 0, 0, 0, 0, 0, 561, 520, 523, 501, 507, 467, 506, 515, 0, 0, 0, 0, 0, 0, 0, 0, 489, 450, 493, 509, 502, 543, 518, 507, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4175, 4120, 4096, 4103, 4065, 4098, 4100, 4011] avg=4096.0000 max=4175 max_rank=0 min=4011 min_rank=7 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16274 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[515, 480, 512, 477, 496, 528, 495, 493, 0, 0, 0, 0, 0, 0, 0, 0, 529, 504, 488, 515, 523, 493, 513, 487, 0, 0, 0, 0, 0, 0, 0, 0, 502, 529, 503, 483, 519, 493, 532, 511, 29, 17, 0, 0, 0, 0, 0, 0, 519, 508, 516, 502, 499, 517, 533, 543, 3, 0, 0, 0, 0, 0, 0, 0, 512, 529, 493, 508, 537, 494, 530, 525, 12, 6, 0, 0, 0, 0, 0, 0, 495, 511, 486, 537, 504, 504, 509, 476, 0, 0, 0, 0, 0, 0, 0, 0, 478, 529, 513, 479, 534, 520, 549, 548, 0, 0, 0, 0, 0, 0, 0, 0, 498, 514, 541, 507, 503, 522, 509, 550, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3996, 4052, 4118, 4140, 4146, 4022, 4150, 4144] avg=4096.0000 max=4150 max_rank=6 min=3996 min_rank=0 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16462 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3968, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 3968, 4224, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 3968, 4224, 4224, 256, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[511, 506, 478, 510, 485, 530, 524, 498, 511, 541, 558, 507, 552, 493, 515, 520, 524, 505, 525, 474, 561, 515, 480, 527, 523, 489, 564, 517, 518, 524, 478, 527, 491, 498, 516, 519, 525, 528, 512, 525, 513, 499, 509, 467, 514, 502, 519, 511, 547, 477, 519, 512, 513, 505, 498, 502, 502, 529, 505, 512, 511, 515, 527, 456] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[541, 504, 516, 500, 519, 524, 483, 480, 497, 503, 525, 519, 507, 471, 512, 561, 489, 512, 525, 462, 511, 506, 563, 491, 508, 531, 521, 532, 506, 513, 495, 489, 444, 498, 502, 517, 515, 551, 534, 536, 528, 516, 527, 478, 521, 507, 502, 493, 543, 515, 512, 528, 496, 543, 521, 506, 526, 486, 515, 552, 498, 492, 530, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[510, 521, 496, 559, 510, 492, 534, 531, 513, 541, 513, 511, 528, 505, 512, 459, 530, 475, 521, 490, 505, 581, 525, 534, 509, 506, 544, 534, 518, 516, 533, 461, 501, 473, 506, 521, 544, 496, 491, 490, 502, 489, 553, 534, 501, 496, 535, 495, 546, 492, 554, 500, 509, 532, 527, 519, 510, 503, 487, 507, 486, 469, 525, 458] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[525, 541, 513, 496, 503, 521, 542, 468, 505, 527, 491, 501, 505, 483, 509, 499, 564, 467, 507, 524, 516, 543, 516, 497, 499, 514, 499, 458, 525, 521, 515, 508, 526, 460, 526, 536, 507, 493, 540, 511, 513, 532, 514, 525, 506, 529, 490, 501, 500, 503, 532, 529, 502, 519, 477, 542, 510, 489, 531, 544, 525, 552, 499, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[494, 503, 526, 523, 479, 488, 518, 488, 503, 490, 492, 497, 537, 493, 528, 539, 528, 526, 478, 506, 471, 499, 524, 496, 533, 498, 496, 521, 497, 517, 527, 518, 498, 520, 496, 523, 542, 501, 524, 528, 524, 532, 501, 541, 533, 512, 530, 511, 496, 501, 474, 507, 536, 514, 529, 508, 537, 512, 532, 541, 512, 521, 483, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[498, 513, 535, 480, 510, 506, 504, 518, 509, 529, 485, 483, 493, 493, 527, 514, 528, 475, 495, 544, 494, 514, 530, 515, 485, 518, 509, 498, 496, 521, 508, 487, 535, 530, 493, 472, 522, 527, 525, 520, 507, 524, 540, 495, 492, 509, 528, 515, 484, 540, 525, 504, 518, 541, 544, 519, 527, 488, 524, 492, 513, 558, 541, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[490, 503, 529, 473, 535, 493, 509, 526, 495, 513, 527, 500, 504, 496, 498, 540, 519, 465, 513, 478, 484, 492, 546, 532, 528, 493, 507, 512, 544, 550, 530, 470, 511, 528, 512, 513, 531, 514, 519, 521, 525, 518, 546, 505, 540, 508, 521, 504, 492, 534, 500, 488, 494, 542, 538, 493, 525, 468, 459, 555, 519, 530, 510, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[546, 503, 503, 514, 492, 517, 512, 544, 551, 488, 531, 507, 492, 507, 552, 528, 565, 510, 499, 480, 492, 485, 561, 516, 486, 500, 496, 491, 498, 517, 532, 508, 488, 531, 495, 504, 507, 527, 488, 532, 507, 502, 507, 513, 463, 510, 523, 529, 523, 511, 504, 528, 544, 533, 523, 497, 516, 495, 519, 501, 493, 488, 514, 530] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[511, 506, 478, 510, 485, 530, 524, 498, 14, 0, 0, 0, 0, 0, 0, 0, 511, 541, 558, 507, 552, 493, 515, 520, 4, 0, 0, 0, 0, 0, 0, 0, 524, 505, 525, 474, 561, 515, 480, 527, 4, 1, 0, 0, 0, 0, 0, 0, 523, 489, 564, 517, 518, 524, 478, 527, 9, 0, 0, 0, 0, 0, 0, 0, 491, 498, 516, 519, 520, 528, 512, 525, 0, 0, 0, 0, 0, 0, 0, 0, 513, 499, 501, 467, 514, 502, 519, 511, 0, 0, 0, 0, 0, 0, 0, 0, 547, 477, 519, 512, 513, 479, 498, 502, 0, 0, 0, 0, 0, 0, 0, 0, 502, 529, 505, 512, 511, 515, 527, 456, 4, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4056, 4201, 4116, 4149, 4109, 4026, 4047, 4064] avg=4096.0000 max=4201 max_rank=1 min=4026 min_rank=5 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16246 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[510, 521, 496, 559, 510, 492, 534, 531, 15, 0, 0, 0, 0, 0, 0, 0, 513, 541, 513, 511, 528, 505, 512, 459, 4, 0, 0, 0, 0, 0, 0, 0, 530, 475, 521, 490, 505, 581, 525, 534, 4, 1, 0, 0, 0, 0, 0, 0, 509, 506, 544, 534, 518, 516, 533, 461, 10, 0, 0, 0, 0, 0, 0, 0, 501, 473, 506, 521, 539, 496, 491, 490, 0, 0, 0, 0, 0, 0, 0, 0, 502, 489, 545, 534, 501, 496, 535, 495, 0, 0, 0, 0, 0, 0, 0, 0, 546, 492, 554, 500, 509, 504, 527, 519, 0, 0, 0, 0, 0, 0, 0, 0, 510, 503, 487, 507, 486, 469, 525, 458, 4, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4168, 4086, 4166, 4131, 4017, 4097, 4151, 3952] avg=4096.0000 max=4168 max_rank=0 min=3952 min_rank=7 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16217 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[494, 503, 526, 523, 479, 488, 518, 488, 15, 0, 0, 0, 0, 0, 0, 0, 503, 490, 492, 497, 537, 493, 528, 539, 4, 0, 0, 0, 0, 0, 0, 0, 528, 526, 478, 506, 471, 499, 524, 496, 4, 1, 0, 0, 0, 0, 0, 0, 533, 498, 496, 521, 497, 517, 527, 518, 9, 0, 0, 0, 0, 0, 0, 0, 498, 520, 496, 523, 537, 501, 524, 528, 0, 0, 0, 0, 0, 0, 0, 0, 524, 532, 494, 541, 533, 512, 530, 511, 0, 0, 0, 0, 0, 0, 0, 0, 496, 501, 474, 507, 536, 487, 529, 508, 0, 0, 0, 0, 0, 0, 0, 0, 537, 512, 532, 541, 512, 521, 483, 516, 3, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4034, 4083, 4033, 4116, 4127, 4177, 4038, 4160] avg=4096.0000 max=4177 max_rank=5 min=4033 min_rank=2 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16502 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[525, 541, 513, 496, 503, 521, 542, 468, 15, 0, 0, 0, 0, 0, 0, 0, 505, 527, 491, 501, 505, 483, 509, 499, 3, 0, 0, 0, 0, 0, 0, 0, 564, 467, 507, 524, 516, 543, 516, 497, 4, 0, 0, 0, 0, 0, 0, 0, 499, 514, 499, 458, 525, 521, 515, 508, 10, 0, 0, 0, 0, 0, 0, 0, 526, 460, 526, 536, 504, 493, 540, 511, 0, 0, 0, 0, 0, 0, 0, 0, 513, 532, 506, 525, 506, 529, 490, 501, 0, 0, 0, 0, 0, 0, 0, 0, 500, 503, 532, 529, 502, 491, 477, 542, 0, 0, 0, 0, 0, 0, 0, 0, 510, 489, 531, 544, 525, 552, 499, 503, 4, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4124, 4023, 4138, 4049, 4096, 4102, 4076, 4160] avg=4096.0000 max=4160 max_rank=7 min=4023 min_rank=1 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16434 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[541, 504, 516, 500, 519, 524, 483, 480, 16, 0, 0, 0, 0, 0, 0, 0, 497, 503, 525, 519, 507, 471, 512, 561, 4, 0, 0, 0, 0, 0, 0, 0, 489, 512, 525, 462, 511, 506, 563, 491, 4, 0, 0, 0, 0, 0, 0, 0, 508, 531, 521, 532, 506, 513, 495, 489, 10, 0, 0, 0, 0, 0, 0, 0, 444, 498, 502, 517, 511, 551, 534, 536, 0, 0, 0, 0, 0, 0, 0, 0, 528, 516, 519, 478, 521, 507, 502, 493, 0, 0, 0, 0, 0, 0, 0, 0, 543, 515, 512, 528, 496, 514, 521, 506, 0, 0, 0, 0, 0, 0, 0, 0, 526, 486, 515, 552, 498, 492, 530, 520, 4, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4099, 4063, 4105, 4093, 4064, 4135, 4126] avg=4096.0000 max=4135 max_rank=6 min=4063 min_rank=2 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16418 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[498, 513, 535, 480, 510, 506, 504, 518, 19, 0, 0, 0, 0, 0, 0, 0, 509, 529, 485, 483, 493, 493, 527, 514, 7, 0, 0, 0, 0, 0, 0, 0, 528, 475, 495, 544, 494, 514, 530, 515, 7, 4, 0, 0, 0, 0, 0, 0, 485, 518, 509, 498, 496, 521, 508, 487, 14, 0, 0, 0, 0, 0, 0, 0, 535, 530, 493, 472, 511, 527, 525, 520, 0, 0, 0, 0, 0, 0, 0, 0, 507, 524, 526, 495, 492, 509, 528, 515, 0, 0, 0, 0, 0, 0, 0, 0, 484, 540, 525, 504, 518, 500, 544, 519, 0, 0, 0, 0, 0, 0, 0, 0, 527, 488, 524, 492, 513, 558, 541, 502, 7, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4040, 4106, 4036, 4113, 4096, 4134, 4160] avg=4096.0000 max=4160 max_rank=7 min=4036 min_rank=3 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16503 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[490, 503, 529, 473, 535, 493, 509, 526, 16, 0, 0, 0, 0, 0, 0, 0, 495, 513, 527, 500, 504, 496, 498, 540, 4, 0, 0, 0, 0, 0, 0, 0, 519, 465, 513, 478, 484, 492, 546, 532, 4, 1, 0, 0, 0, 0, 0, 0, 528, 493, 507, 512, 544, 550, 530, 470, 10, 0, 0, 0, 0, 0, 0, 0, 511, 528, 512, 513, 526, 514, 519, 521, 0, 0, 0, 0, 0, 0, 0, 0, 525, 518, 538, 505, 540, 508, 521, 504, 0, 0, 0, 0, 0, 0, 0, 0, 492, 534, 500, 488, 494, 513, 538, 493, 0, 0, 0, 0, 0, 0, 0, 0, 525, 468, 459, 555, 519, 530, 510, 511, 4, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4077, 4034, 4144, 4144, 4159, 4052, 4084] avg=4096.0000 max=4159 max_rank=5 min=4034 min_rank=2 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16439 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[546, 503, 503, 514, 492, 517, 512, 544, 15, 0, 0, 0, 0, 0, 0, 0, 551, 488, 531, 507, 492, 507, 552, 528, 3, 0, 0, 0, 0, 0, 0, 0, 565, 510, 499, 480, 492, 485, 561, 516, 4, 0, 0, 0, 0, 0, 0, 0, 486, 500, 496, 491, 498, 517, 532, 508, 10, 0, 0, 0, 0, 0, 0, 0, 488, 531, 495, 504, 504, 527, 488, 532, 0, 0, 0, 0, 0, 0, 0, 0, 507, 502, 500, 513, 463, 510, 523, 529, 0, 0, 0, 0, 0, 0, 0, 0, 523, 511, 504, 528, 544, 505, 523, 497, 0, 0, 0, 0, 0, 0, 0, 0, 516, 495, 519, 501, 493, 488, 514, 530, 3, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4146, 4159, 4112, 4038, 4069, 4047, 4135, 4062] avg=4096.0000 max=4159 max_rank=1 min=4038 min_rank=3 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16313 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 3968, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4352, 3968, 4096, 3968, 4096, 4224, 4352, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4224, 3968, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[539, 537, 538, 536, 480, 526, 531, 493, 516, 477, 509, 505, 518, 541, 508, 468, 517, 539, 498, 510, 534, 498, 512, 487, 514, 499, 460, 509, 527, 492, 544, 521, 547, 493, 505, 495, 537, 532, 536, 505, 498, 505, 535, 486, 538, 498, 506, 511, 476, 497, 537, 522, 513, 509, 511, 518, 515, 517, 475, 525, 484, 526, 507, 526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[521, 516, 500, 501, 504, 526, 526, 513, 514, 504, 507, 497, 511, 519, 526, 540, 478, 500, 522, 495, 472, 540, 525, 496, 508, 501, 483, 507, 527, 472, 521, 511, 532, 485, 479, 536, 515, 551, 516, 515, 493, 532, 499, 516, 537, 528, 520, 519, 508, 506, 543, 545, 495, 479, 519, 494, 516, 520, 535, 509, 508, 524, 502, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[532, 487, 534, 543, 509, 501, 488, 471, 511, 499, 533, 527, 524, 499, 526, 483, 476, 507, 488, 515, 488, 550, 512, 519, 506, 515, 514, 509, 518, 500, 524, 533, 497, 481, 506, 523, 531, 467, 481, 497, 529, 515, 536, 523, 511, 512, 481, 508, 531, 461, 516, 499, 541, 525, 511, 516, 553, 523, 570, 534, 500, 501, 508, 540] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[540, 534, 463, 531, 482, 527, 519, 530, 521, 521, 513, 519, 502, 511, 489, 509, 489, 512, 504, 522, 523, 491, 498, 526, 519, 522, 470, 508, 512, 485, 488, 509, 543, 542, 474, 500, 522, 531, 510, 518, 521, 499, 522, 505, 521, 518, 529, 500, 494, 513, 510, 482, 487, 507, 522, 508, 499, 528, 565, 554, 552, 537, 511, 455] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[508, 498, 520, 495, 497, 526, 498, 502, 521, 545, 532, 506, 513, 523, 482, 511, 517, 511, 479, 494, 565, 500, 504, 531, 536, 490, 480, 520, 524, 544, 538, 494, 501, 531, 503, 510, 538, 529, 503, 467, 510, 514, 498, 498, 531, 539, 511, 490, 520, 518, 532, 481, 521, 505, 497, 523, 534, 511, 524, 498, 525, 522, 487, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[495, 490, 469, 522, 537, 518, 515, 505, 493, 499, 520, 474, 516, 506, 516, 574, 496, 515, 482, 492, 511, 498, 521, 536, 544, 527, 498, 531, 497, 515, 525, 470, 502, 508, 519, 504, 492, 500, 500, 505, 500, 520, 532, 531, 537, 471, 539, 507, 501, 504, 497, 499, 529, 537, 555, 500, 509, 494, 532, 531, 544, 512, 508, 542] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[517, 482, 529, 515, 501, 541, 498, 546, 515, 505, 477, 489, 534, 503, 535, 550, 500, 515, 507, 529, 469, 527, 491, 494, 527, 488, 504, 526, 533, 532, 521, 518, 498, 538, 511, 459, 510, 480, 549, 468, 495, 500, 523, 533, 516, 528, 495, 523, 498, 564, 501, 488, 543, 511, 535, 504, 505, 498, 539, 524, 521, 498, 505, 490] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[526, 548, 529, 521, 505, 479, 516, 509, 530, 485, 526, 513, 467, 530, 510, 563, 515, 506, 493, 491, 495, 516, 541, 508, 505, 522, 498, 511, 505, 474, 508, 505, 504, 513, 492, 564, 487, 533, 532, 522, 500, 490, 543, 524, 497, 502, 493, 527, 460, 510, 515, 511, 512, 530, 472, 537, 537, 480, 551, 517, 541, 515, 507, 500] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[514, 516, 500, 501, 504, 526, 526, 513, 0, 0, 0, 0, 0, 0, 0, 0, 514, 504, 507, 497, 511, 519, 526, 535, 0, 0, 0, 0, 0, 0, 0, 0, 478, 500, 522, 495, 472, 540, 525, 496, 34, 0, 0, 0, 0, 0, 0, 0, 508, 501, 483, 507, 527, 472, 521, 511, 9, 6, 0, 0, 0, 0, 0, 0, 532, 485, 479, 536, 515, 551, 516, 515, 6, 5, 0, 0, 0, 0, 0, 0, 493, 532, 499, 516, 525, 528, 520, 519, 0, 0, 0, 0, 0, 0, 0, 0, 508, 506, 543, 545, 495, 479, 519, 494, 5, 2, 0, 0, 0, 0, 0, 0, 516, 520, 492, 509, 508, 524, 502, 509, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4113, 4062, 4045, 4140, 4132, 4096, 4080] avg=4096.0000 max=4140 max_rank=4 min=4045 min_rank=3 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16448 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[533, 534, 463, 531, 482, 527, 519, 530, 0, 0, 0, 0, 0, 0, 0, 0, 521, 521, 513, 519, 502, 511, 489, 504, 0, 0, 0, 0, 0, 0, 0, 0, 489, 512, 504, 522, 523, 491, 498, 526, 36, 0, 0, 0, 0, 0, 0, 0, 519, 522, 470, 508, 512, 485, 488, 509, 10, 6, 0, 0, 0, 0, 0, 0, 543, 542, 474, 500, 522, 531, 510, 518, 5, 5, 0, 0, 0, 0, 0, 0, 521, 499, 522, 505, 510, 518, 529, 500, 0, 0, 0, 0, 0, 0, 0, 0, 494, 513, 510, 482, 487, 507, 522, 508, 5, 2, 0, 0, 0, 0, 0, 0, 499, 528, 519, 554, 552, 537, 511, 455, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4080, 4101, 4029, 4150, 4104, 4030, 4155] avg=4096.0000 max=4155 max_rank=7 min=4029 min_rank=3 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16439 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[501, 498, 520, 495, 497, 526, 498, 502, 0, 0, 0, 0, 0, 0, 0, 0, 521, 545, 532, 506, 513, 523, 482, 506, 0, 0, 0, 0, 0, 0, 0, 0, 517, 511, 479, 494, 565, 500, 504, 531, 33, 0, 0, 0, 0, 0, 0, 0, 536, 490, 480, 520, 524, 544, 538, 494, 9, 6, 0, 0, 0, 0, 0, 0, 501, 531, 503, 510, 538, 529, 503, 467, 6, 5, 0, 0, 0, 0, 0, 0, 510, 514, 498, 498, 519, 539, 511, 490, 0, 0, 0, 0, 0, 0, 0, 0, 520, 518, 532, 481, 521, 505, 497, 523, 5, 2, 0, 0, 0, 0, 0, 0, 534, 511, 482, 498, 525, 522, 487, 493, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4037, 4128, 4134, 4141, 4093, 4079, 4104, 4052] avg=4096.0000 max=4141 max_rank=3 min=4037 min_rank=0 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16328 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[532, 537, 538, 536, 480, 526, 531, 493, 0, 0, 0, 0, 0, 0, 0, 0, 516, 477, 509, 505, 518, 541, 508, 464, 0, 0, 0, 0, 0, 0, 0, 0, 517, 539, 498, 510, 534, 498, 512, 487, 30, 0, 0, 0, 0, 0, 0, 0, 514, 499, 460, 509, 527, 492, 544, 521, 8, 6, 0, 0, 0, 0, 0, 0, 547, 493, 505, 495, 537, 532, 536, 505, 6, 5, 0, 0, 0, 0, 0, 0, 498, 505, 535, 486, 526, 498, 506, 511, 0, 0, 0, 0, 0, 0, 0, 0, 476, 497, 537, 522, 513, 509, 511, 518, 4, 2, 0, 0, 0, 0, 0, 0, 515, 517, 437, 525, 484, 526, 507, 526, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4173, 4038, 4125, 4080, 4161, 4065, 4089, 4037] avg=4096.0000 max=4173 max_rank=0 min=4037 min_rank=7 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16352 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[525, 487, 534, 543, 509, 501, 488, 471, 0, 0, 0, 0, 0, 0, 0, 0, 511, 499, 533, 527, 524, 499, 526, 479, 0, 0, 0, 0, 0, 0, 0, 0, 476, 507, 488, 515, 488, 550, 512, 519, 36, 0, 0, 0, 0, 0, 0, 0, 506, 515, 514, 509, 518, 500, 524, 533, 10, 6, 0, 0, 0, 0, 0, 0, 497, 481, 506, 523, 531, 467, 481, 497, 5, 5, 0, 0, 0, 0, 0, 0, 529, 515, 536, 523, 500, 512, 481, 508, 0, 0, 0, 0, 0, 0, 0, 0, 531, 461, 516, 499, 541, 525, 511, 516, 4, 2, 0, 0, 0, 0, 0, 0, 553, 523, 524, 534, 500, 501, 508, 540, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4058, 4098, 4091, 4135, 3993, 4104, 4106, 4183] avg=4096.0000 max=4183 max_rank=7 min=3993 min_rank=4 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[477, 490, 469, 522, 537, 518, 515, 505, 0, 0, 0, 0, 0, 0, 0, 0, 493, 499, 520, 474, 516, 506, 516, 565, 0, 0, 0, 0, 0, 0, 0, 0, 496, 515, 482, 492, 511, 498, 521, 536, 38, 0, 0, 0, 0, 0, 0, 0, 544, 527, 498, 531, 497, 515, 525, 470, 13, 10, 0, 0, 0, 0, 0, 0, 502, 508, 519, 504, 492, 500, 500, 505, 10, 11, 0, 0, 0, 0, 0, 0, 500, 520, 532, 531, 517, 471, 539, 507, 0, 0, 0, 0, 0, 0, 0, 0, 501, 504, 497, 499, 529, 537, 555, 500, 9, 7, 0, 0, 0, 0, 0, 0, 509, 494, 481, 531, 544, 512, 508, 542, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4033, 4089, 4089, 4130, 4051, 4117, 4138, 4121] avg=4096.0000 max=4138 max_rank=6 min=4033 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16427 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[510, 482, 529, 515, 501, 541, 498, 546, 0, 0, 0, 0, 0, 0, 0, 0, 515, 505, 477, 489, 534, 503, 535, 545, 0, 0, 0, 0, 0, 0, 0, 0, 500, 515, 507, 529, 469, 527, 491, 494, 34, 0, 0, 0, 0, 0, 0, 0, 527, 488, 504, 526, 533, 532, 521, 518, 9, 6, 0, 0, 0, 0, 0, 0, 498, 538, 511, 459, 510, 480, 549, 468, 5, 5, 0, 0, 0, 0, 0, 0, 495, 500, 523, 533, 505, 528, 495, 523, 0, 0, 0, 0, 0, 0, 0, 0, 498, 564, 501, 488, 543, 511, 535, 504, 5, 2, 0, 0, 0, 0, 0, 0, 505, 498, 496, 524, 521, 498, 505, 490, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4122, 4103, 4066, 4164, 4023, 4102, 4151, 4037] avg=4096.0000 max=4164 max_rank=3 min=4023 min_rank=4 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16313 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[519, 548, 529, 521, 505, 479, 516, 509, 0, 0, 0, 0, 0, 0, 0, 0, 530, 485, 526, 513, 467, 530, 510, 558, 0, 0, 0, 0, 0, 0, 0, 0, 515, 506, 493, 491, 495, 516, 541, 508, 35, 0, 0, 0, 0, 0, 0, 0, 505, 522, 498, 511, 505, 474, 508, 505, 10, 6, 0, 0, 0, 0, 0, 0, 504, 513, 492, 564, 487, 533, 532, 522, 5, 5, 0, 0, 0, 0, 0, 0, 500, 490, 543, 524, 486, 502, 493, 527, 0, 0, 0, 0, 0, 0, 0, 0, 460, 510, 515, 511, 512, 530, 472, 537, 5, 2, 0, 0, 0, 0, 0, 0, 537, 480, 506, 517, 541, 515, 507, 500, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4119, 4100, 4044, 4157, 4065, 4054, 4103] avg=4096.0000 max=4157 max_rank=4 min=4044 min_rank=3 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 3968, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 3968, 4224, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[537, 485, 482, 498, 526, 526, 490, 536, 508, 579, 484, 539, 543, 521, 514, 516, 503, 504, 498, 488, 509, 516, 510, 534, 500, 525, 541, 524, 505, 549, 488, 488, 504, 511, 513, 510, 518, 497, 526, 538, 503, 469, 513, 516, 483, 546, 522, 513, 478, 491, 486, 487, 519, 500, 526, 513, 522, 530, 500, 527, 530, 471, 520, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[528, 498, 538, 548, 524, 513, 540, 531, 549, 516, 513, 514, 501, 480, 481, 544, 496, 497, 520, 513, 511, 513, 521, 485, 518, 506, 491, 527, 480, 499, 475, 504, 542, 459, 519, 531, 481, 507, 559, 511, 569, 497, 523, 518, 490, 506, 480, 498, 519, 512, 501, 500, 509, 505, 538, 499, 479, 500, 518, 535, 472, 524, 542, 551] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[524, 538, 535, 503, 476, 555, 534, 509, 504, 514, 499, 521, 537, 512, 503, 493, 536, 516, 498, 483, 498, 485, 533, 484, 503, 498, 499, 530, 526, 506, 523, 517, 513, 529, 533, 486, 512, 518, 489, 567, 497, 505, 508, 501, 539, 548, 482, 518, 500, 497, 533, 492, 511, 496, 521, 535, 518, 504, 506, 536, 452, 518, 492, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[493, 451, 510, 490, 518, 499, 508, 530, 519, 557, 541, 481, 496, 509, 522, 490, 534, 484, 535, 500, 509, 526, 558, 535, 543, 527, 478, 508, 513, 500, 491, 532, 525, 538, 507, 526, 533, 484, 497, 505, 526, 525, 526, 471, 498, 527, 492, 518, 517, 496, 505, 529, 536, 502, 509, 526, 489, 492, 493, 525, 500, 514, 523, 527] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[543, 493, 511, 490, 520, 484, 545, 484, 510, 498, 519, 491, 521, 474, 506, 511, 495, 500, 527, 469, 476, 522, 524, 518, 504, 526, 514, 484, 532, 546, 533, 506, 524, 486, 527, 511, 474, 550, 511, 516, 525, 504, 528, 527, 504, 493, 498, 503, 523, 506, 542, 558, 521, 505, 526, 513, 519, 542, 502, 490, 529, 511, 505, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[523, 518, 538, 489, 534, 515, 521, 532, 516, 505, 485, 540, 522, 558, 494, 524, 527, 549, 527, 481, 522, 486, 509, 522, 488, 502, 530, 506, 513, 469, 521, 526, 499, 500, 509, 545, 510, 492, 559, 537, 500, 482, 547, 512, 499, 508, 457, 497, 481, 524, 485, 512, 495, 514, 474, 515, 528, 500, 506, 491, 495, 525, 493, 585] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[501, 513, 526, 500, 507, 484, 535, 520, 509, 523, 508, 528, 503, 523, 550, 539, 484, 474, 515, 500, 488, 485, 503, 526, 560, 525, 506, 515, 527, 522, 525, 481, 502, 462, 507, 493, 513, 485, 522, 549, 506, 513, 502, 480, 508, 509, 508, 489, 483, 510, 541, 554, 506, 523, 546, 526, 506, 528, 526, 545, 516, 506, 487, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[546, 488, 488, 509, 514, 509, 532, 512, 488, 510, 531, 505, 518, 522, 522, 474, 460, 540, 506, 577, 548, 534, 490, 546, 511, 539, 517, 517, 498, 496, 499, 521, 484, 520, 513, 512, 500, 495, 486, 531, 516, 492, 505, 511, 507, 492, 521, 510, 532, 515, 512, 539, 523, 504, 517, 541, 497, 520, 473, 495, 557, 514, 495, 472] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[537, 485, 482, 498, 526, 526, 475, 536, 0, 0, 0, 0, 0, 0, 0, 0, 508, 558, 484, 539, 543, 521, 514, 516, 0, 0, 0, 0, 0, 0, 0, 0, 503, 504, 498, 488, 509, 516, 510, 534, 10, 2, 0, 0, 0, 0, 0, 0, 500, 525, 541, 524, 505, 549, 488, 488, 0, 0, 0, 0, 0, 0, 0, 0, 504, 511, 513, 510, 518, 497, 526, 533, 0, 0, 0, 0, 0, 0, 0, 0, 503, 469, 513, 516, 483, 546, 522, 513, 21, 15, 0, 0, 0, 0, 0, 0, 478, 491, 486, 477, 519, 500, 526, 513, 0, 0, 0, 0, 0, 0, 0, 0, 522, 530, 500, 527, 530, 471, 520, 520, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4065, 4183, 4074, 4120, 4112, 4101, 3990, 4123] avg=4096.0000 max=4183 max_rank=1 min=3990 min_rank=6 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16326 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[524, 538, 535, 503, 476, 555, 518, 509, 0, 0, 0, 0, 0, 0, 0, 0, 504, 495, 499, 521, 537, 512, 503, 493, 0, 0, 0, 0, 0, 0, 0, 0, 536, 516, 498, 483, 498, 485, 533, 484, 10, 2, 0, 0, 0, 0, 0, 0, 503, 498, 499, 530, 526, 506, 523, 517, 0, 0, 0, 0, 0, 0, 0, 0, 513, 529, 533, 486, 512, 518, 489, 562, 0, 0, 0, 0, 0, 0, 0, 0, 497, 505, 508, 501, 539, 548, 482, 518, 19, 16, 0, 0, 0, 0, 0, 0, 500, 497, 533, 482, 511, 496, 521, 535, 0, 0, 0, 0, 0, 0, 0, 0, 518, 504, 506, 536, 452, 518, 492, 520, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4158, 4064, 4045, 4102, 4142, 4133, 4075, 4049] avg=4096.0000 max=4158 max_rank=0 min=4045 min_rank=2 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16399 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[493, 451, 510, 490, 518, 499, 489, 530, 0, 0, 0, 0, 0, 0, 0, 0, 519, 532, 541, 481, 496, 509, 522, 490, 0, 0, 0, 0, 0, 0, 0, 0, 534, 484, 535, 500, 509, 526, 558, 535, 13, 6, 0, 0, 0, 0, 0, 0, 543, 522, 478, 508, 513, 500, 491, 532, 0, 0, 0, 0, 0, 0, 0, 0, 525, 538, 507, 526, 533, 484, 497, 493, 0, 0, 0, 0, 0, 0, 0, 0, 526, 525, 526, 471, 498, 527, 492, 518, 25, 19, 0, 0, 0, 0, 0, 0, 517, 496, 505, 516, 536, 502, 509, 526, 0, 0, 0, 0, 0, 0, 0, 0, 489, 492, 493, 525, 500, 514, 523, 527, 6, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3980, 4090, 4200, 4087, 4103, 4127, 4107, 4074] avg=4096.0000 max=4200 max_rank=2 min=3980 min_rank=0 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16411 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[543, 493, 511, 490, 520, 484, 529, 484, 0, 0, 0, 0, 0, 0, 0, 0, 510, 480, 519, 491, 521, 474, 506, 511, 0, 0, 0, 0, 0, 0, 0, 0, 495, 500, 527, 469, 476, 522, 524, 518, 11, 2, 0, 0, 0, 0, 0, 0, 504, 526, 514, 484, 532, 546, 533, 506, 0, 0, 0, 0, 0, 0, 0, 0, 524, 486, 527, 511, 474, 550, 511, 512, 0, 0, 0, 0, 0, 0, 0, 0, 525, 504, 528, 527, 504, 493, 498, 503, 18, 16, 0, 0, 0, 0, 0, 0, 523, 506, 542, 547, 521, 505, 526, 513, 0, 0, 0, 0, 0, 0, 0, 0, 519, 542, 502, 490, 529, 511, 505, 519, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4054, 4012, 4044, 4145, 4095, 4116, 4183, 4119] avg=4096.0000 max=4183 max_rank=6 min=4012 min_rank=1 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16513 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[523, 518, 538, 489, 534, 515, 506, 532, 0, 0, 0, 0, 0, 0, 0, 0, 516, 486, 485, 540, 522, 558, 494, 524, 0, 0, 0, 0, 0, 0, 0, 0, 527, 549, 527, 481, 522, 486, 509, 522, 10, 2, 0, 0, 0, 0, 0, 0, 488, 502, 530, 506, 513, 469, 521, 526, 0, 0, 0, 0, 0, 0, 0, 0, 499, 500, 509, 545, 510, 492, 559, 532, 0, 0, 0, 0, 0, 0, 0, 0, 500, 482, 547, 512, 499, 508, 457, 497, 19, 15, 0, 0, 0, 0, 0, 0, 481, 524, 485, 502, 495, 514, 474, 515, 0, 0, 0, 0, 0, 0, 0, 0, 528, 500, 506, 491, 495, 525, 493, 585, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4155, 4125, 4135, 4055, 4146, 4036, 3990, 4126] avg=4096.0000 max=4155 max_rank=0 min=3990 min_rank=6 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16298 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[501, 513, 526, 500, 507, 484, 519, 520, 0, 0, 0, 0, 0, 0, 0, 0, 509, 504, 508, 528, 503, 523, 550, 539, 0, 0, 0, 0, 0, 0, 0, 0, 484, 474, 515, 500, 488, 485, 503, 526, 11, 2, 0, 0, 0, 0, 0, 0, 560, 525, 506, 515, 527, 522, 525, 481, 0, 0, 0, 0, 0, 0, 0, 0, 502, 462, 507, 493, 513, 485, 522, 544, 0, 0, 0, 0, 0, 0, 0, 0, 506, 513, 502, 480, 508, 509, 508, 489, 19, 16, 0, 0, 0, 0, 0, 0, 483, 510, 541, 543, 506, 523, 546, 526, 0, 0, 0, 0, 0, 0, 0, 0, 506, 528, 526, 545, 516, 506, 487, 512, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4070, 4164, 3988, 4161, 4028, 4050, 4178, 4129] avg=4096.0000 max=4178 max_rank=6 min=3988 min_rank=2 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16385 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[546, 488, 488, 509, 514, 509, 516, 512, 0, 0, 0, 0, 0, 0, 0, 0, 488, 491, 531, 505, 518, 522, 522, 474, 0, 0, 0, 0, 0, 0, 0, 0, 460, 540, 506, 577, 548, 534, 490, 546, 11, 2, 0, 0, 0, 0, 0, 0, 511, 539, 517, 517, 498, 496, 499, 521, 0, 0, 0, 0, 0, 0, 0, 0, 484, 520, 513, 512, 500, 495, 486, 527, 0, 0, 0, 0, 0, 0, 0, 0, 516, 492, 505, 511, 507, 492, 521, 510, 19, 16, 0, 0, 0, 0, 0, 0, 532, 515, 512, 528, 523, 504, 517, 541, 0, 0, 0, 0, 0, 0, 0, 0, 497, 520, 473, 495, 557, 514, 495, 472, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4051, 4214, 4098, 4037, 4089, 4172, 4025] avg=4096.0000 max=4214 max_rank=2 min=4025 min_rank=7 imbalance_ratio=1.028809 max_load_reduction=28554 max_load_reduction_pct=0.871399 moved_to_echo=16323 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[528, 498, 538, 548, 524, 513, 524, 531, 0, 0, 0, 0, 0, 0, 0, 0, 549, 497, 513, 514, 501, 480, 481, 544, 0, 0, 0, 0, 0, 0, 0, 0, 496, 497, 520, 513, 511, 513, 521, 485, 10, 2, 0, 0, 0, 0, 0, 0, 518, 506, 491, 527, 480, 499, 475, 504, 0, 0, 0, 0, 0, 0, 0, 0, 542, 459, 519, 531, 481, 507, 559, 507, 0, 0, 0, 0, 0, 0, 0, 0, 569, 497, 523, 518, 490, 506, 480, 498, 19, 16, 0, 0, 0, 0, 0, 0, 519, 512, 501, 490, 509, 505, 538, 499, 0, 0, 0, 0, 0, 0, 0, 0, 479, 500, 518, 535, 472, 524, 542, 551, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4204, 4079, 4068, 4000, 4105, 4116, 4073, 4123] avg=4096.0000 max=4204 max_rank=0 min=4000 min_rank=3 imbalance_ratio=1.026367 max_load_reduction=28564 max_load_reduction_pct=0.871704 moved_to_echo=16417 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 3968, 4096, 256, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[502, 495, 507, 495, 507, 511, 507, 522, 538, 521, 498, 486, 531, 535, 516, 511, 518, 493, 540, 520, 499, 486, 515, 547, 492, 513, 504, 498, 474, 506, 517, 453, 514, 512, 506, 534, 502, 513, 518, 506, 506, 550, 509, 501, 527, 531, 526, 523, 534, 532, 486, 509, 497, 534, 514, 506, 511, 508, 512, 540, 512, 508, 547, 483] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[494, 565, 565, 516, 521, 465, 471, 470, 495, 474, 478, 520, 535, 532, 504, 546, 508, 460, 498, 527, 511, 500, 491, 526, 518, 461, 499, 535, 512, 501, 499, 489, 513, 536, 477, 549, 532, 541, 508, 536, 529, 513, 504, 516, 507, 541, 515, 502, 505, 517, 485, 534, 526, 508, 511, 501, 463, 544, 543, 538, 543, 531, 485, 529] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[479, 524, 467, 511, 548, 520, 535, 490, 489, 480, 535, 497, 502, 523, 547, 495, 543, 544, 514, 508, 526, 531, 511, 510, 512, 506, 532, 518, 543, 494, 526, 517, 524, 513, 514, 509, 503, 511, 515, 541, 468, 503, 539, 522, 500, 505, 503, 508, 477, 515, 511, 452, 479, 557, 507, 566, 524, 519, 500, 499, 475, 463, 544, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[486, 485, 533, 514, 500, 527, 482, 554, 478, 513, 542, 517, 532, 488, 487, 529, 549, 518, 508, 505, 540, 549, 504, 535, 478, 516, 498, 508, 512, 528, 486, 522, 491, 543, 526, 531, 510, 506, 509, 575, 512, 492, 528, 474, 553, 514, 471, 510, 525, 512, 488, 524, 502, 493, 506, 522, 514, 483, 461, 522, 504, 522, 491, 531] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[490, 528, 490, 515, 513, 519, 531, 518, 525, 539, 519, 496, 509, 494, 549, 513, 477, 503, 470, 522, 532, 527, 493, 488, 484, 532, 537, 501, 498, 526, 478, 519, 535, 519, 507, 497, 530, 523, 530, 506, 498, 488, 519, 525, 516, 502, 527, 547, 515, 501, 474, 520, 521, 489, 529, 544, 504, 515, 493, 477, 528, 526, 488, 540] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[486, 538, 484, 483, 509, 510, 535, 549, 528, 523, 514, 497, 485, 505, 493, 463, 485, 509, 531, 506, 491, 522, 525, 513, 486, 512, 490, 507, 501, 516, 522, 554, 498, 513, 509, 544, 503, 531, 514, 549, 520, 547, 542, 509, 503, 485, 499, 473, 515, 509, 493, 511, 529, 515, 505, 546, 532, 466, 508, 493, 545, 533, 502, 555] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[525, 510, 521, 499, 518, 495, 542, 481, 500, 516, 531, 529, 511, 481, 545, 556, 513, 509, 501, 486, 529, 560, 507, 520, 458, 497, 536, 522, 508, 487, 504, 514, 500, 515, 476, 471, 503, 503, 529, 530, 531, 522, 520, 519, 516, 511, 505, 472, 549, 507, 524, 448, 519, 522, 510, 514, 535, 518, 558, 511, 493, 514, 524, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[545, 530, 516, 536, 509, 531, 501, 517, 512, 516, 498, 468, 516, 493, 490, 523, 519, 499, 505, 517, 549, 513, 490, 472, 524, 482, 529, 518, 505, 542, 500, 488, 505, 516, 486, 510, 529, 508, 522, 514, 523, 453, 534, 526, 562, 464, 514, 487, 528, 485, 535, 531, 539, 511, 524, 537, 483, 479, 502, 510, 531, 522, 535, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[502, 495, 507, 495, 507, 511, 507, 522, 1, 1, 0, 0, 0, 0, 0, 0, 538, 521, 498, 486, 531, 535, 516, 511, 4, 2, 0, 0, 0, 0, 0, 0, 518, 493, 540, 520, 499, 481, 515, 547, 0, 0, 0, 0, 0, 0, 0, 0, 492, 513, 504, 498, 474, 506, 517, 453, 32, 5, 0, 0, 0, 0, 0, 0, 514, 512, 506, 534, 502, 513, 518, 474, 0, 0, 0, 0, 0, 0, 0, 0, 506, 550, 509, 501, 527, 531, 526, 523, 0, 0, 0, 0, 0, 0, 0, 0, 534, 532, 486, 509, 497, 534, 514, 499, 0, 0, 0, 0, 0, 0, 0, 0, 511, 508, 512, 540, 512, 508, 547, 482, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4048, 4142, 4113, 3994, 4073, 4173, 4105, 4120] avg=4096.0000 max=4173 max_rank=5 min=3994 min_rank=3 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16471 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[494, 565, 565, 516, 521, 465, 471, 470, 2, 1, 0, 0, 0, 0, 0, 0, 495, 474, 478, 520, 535, 532, 504, 546, 4, 2, 0, 0, 0, 0, 0, 0, 508, 460, 498, 527, 511, 495, 491, 526, 0, 0, 0, 0, 0, 0, 0, 0, 518, 461, 499, 535, 512, 501, 499, 489, 34, 5, 0, 0, 0, 0, 0, 0, 513, 536, 477, 549, 532, 541, 508, 502, 0, 0, 0, 0, 0, 0, 0, 0, 529, 513, 504, 516, 507, 541, 515, 502, 0, 0, 0, 0, 0, 0, 0, 0, 505, 517, 485, 534, 526, 508, 511, 494, 0, 0, 0, 0, 0, 0, 0, 0, 463, 544, 543, 538, 543, 531, 485, 527, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4070, 4090, 4016, 4053, 4158, 4127, 4080, 4174] avg=4096.0000 max=4174 max_rank=7 min=4016 min_rank=2 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16539 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[479, 524, 467, 511, 548, 520, 535, 490, 2, 1, 0, 0, 0, 0, 0, 0, 489, 480, 535, 497, 502, 523, 547, 495, 5, 2, 0, 0, 0, 0, 0, 0, 543, 544, 514, 508, 526, 525, 511, 510, 0, 0, 0, 0, 0, 0, 0, 0, 512, 506, 532, 518, 543, 494, 526, 517, 34, 6, 0, 0, 0, 0, 0, 0, 524, 513, 514, 509, 503, 511, 515, 507, 0, 0, 0, 0, 0, 0, 0, 0, 468, 503, 539, 522, 500, 505, 503, 508, 0, 0, 0, 0, 0, 0, 0, 0, 477, 515, 511, 452, 479, 557, 507, 558, 0, 0, 0, 0, 0, 0, 0, 0, 524, 519, 500, 499, 475, 463, 544, 523, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4075, 4181, 4188, 4096, 4048, 4056, 4047] avg=4096.0000 max=4188 max_rank=3 min=4047 min_rank=7 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16247 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[486, 538, 484, 483, 509, 510, 535, 549, 2, 1, 0, 0, 0, 0, 0, 0, 528, 523, 514, 497, 485, 505, 493, 463, 4, 2, 0, 0, 0, 0, 0, 0, 485, 509, 531, 506, 491, 517, 525, 513, 0, 0, 0, 0, 0, 0, 0, 0, 486, 512, 490, 507, 501, 516, 522, 554, 35, 6, 0, 0, 0, 0, 0, 0, 498, 513, 509, 544, 503, 531, 514, 514, 0, 0, 0, 0, 0, 0, 0, 0, 520, 547, 542, 509, 503, 485, 499, 473, 0, 0, 0, 0, 0, 0, 0, 0, 515, 509, 493, 511, 529, 515, 505, 538, 0, 0, 0, 0, 0, 0, 0, 0, 532, 466, 508, 493, 545, 533, 502, 553, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4014, 4077, 4129, 4126, 4078, 4115, 4132] avg=4096.0000 max=4132 max_rank=7 min=4014 min_rank=1 imbalance_ratio=1.008789 max_load_reduction=28636 max_load_reduction_pct=0.873901 moved_to_echo=16451 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[525, 510, 521, 499, 518, 495, 542, 481, 1, 1, 0, 0, 0, 0, 0, 0, 500, 516, 531, 529, 511, 481, 545, 556, 5, 2, 0, 0, 0, 0, 0, 0, 513, 509, 501, 486, 529, 554, 507, 520, 0, 0, 0, 0, 0, 0, 0, 0, 458, 497, 536, 522, 508, 487, 504, 514, 34, 5, 0, 0, 0, 0, 0, 0, 500, 515, 476, 471, 503, 503, 529, 496, 0, 0, 0, 0, 0, 0, 0, 0, 531, 522, 520, 519, 516, 511, 505, 472, 0, 0, 0, 0, 0, 0, 0, 0, 549, 507, 524, 448, 519, 522, 510, 507, 0, 0, 0, 0, 0, 0, 0, 0, 535, 518, 558, 511, 493, 514, 524, 487, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4176, 4119, 4065, 3993, 4096, 4086, 4140] avg=4096.0000 max=4176 max_rank=1 min=3993 min_rank=4 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16315 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[486, 485, 533, 514, 500, 527, 482, 554, 2, 1, 0, 0, 0, 0, 0, 0, 478, 513, 542, 517, 532, 488, 487, 529, 5, 2, 0, 0, 0, 0, 0, 0, 549, 518, 508, 505, 540, 543, 504, 535, 0, 0, 0, 0, 0, 0, 0, 0, 478, 516, 498, 508, 512, 528, 486, 522, 37, 6, 0, 0, 0, 0, 0, 0, 491, 543, 526, 531, 510, 506, 509, 538, 0, 0, 0, 0, 0, 0, 0, 0, 512, 492, 528, 474, 553, 514, 471, 510, 0, 0, 0, 0, 0, 0, 0, 0, 525, 512, 488, 524, 502, 493, 506, 514, 0, 0, 0, 0, 0, 0, 0, 0, 514, 483, 461, 522, 504, 522, 491, 529, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4093, 4202, 4091, 4154, 4054, 4064, 4026] avg=4096.0000 max=4202 max_rank=2 min=4026 min_rank=7 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16298 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[490, 528, 490, 515, 513, 519, 531, 518, 5, 2, 0, 0, 0, 0, 0, 0, 525, 539, 519, 496, 509, 494, 549, 513, 9, 3, 0, 0, 0, 0, 0, 0, 477, 503, 470, 522, 532, 516, 493, 488, 0, 0, 0, 0, 0, 0, 0, 0, 484, 532, 537, 501, 498, 526, 478, 519, 36, 10, 0, 0, 0, 0, 0, 0, 535, 519, 507, 497, 530, 523, 530, 470, 0, 0, 0, 0, 0, 0, 0, 0, 498, 488, 519, 525, 516, 502, 527, 547, 7, 0, 0, 0, 0, 0, 0, 0, 515, 501, 474, 520, 521, 489, 529, 531, 0, 0, 0, 0, 0, 0, 0, 0, 504, 515, 493, 477, 528, 526, 488, 528, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4156, 4001, 4121, 4111, 4129, 4080, 4059] avg=4096.0000 max=4156 max_rank=1 min=4001 min_rank=2 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[545, 530, 516, 536, 509, 531, 501, 517, 2, 1, 0, 0, 0, 0, 0, 0, 512, 516, 498, 468, 516, 493, 490, 523, 4, 2, 0, 0, 0, 0, 0, 0, 519, 499, 505, 517, 549, 508, 490, 472, 0, 0, 0, 0, 0, 0, 0, 0, 524, 482, 529, 518, 505, 542, 500, 488, 33, 6, 0, 0, 0, 0, 0, 0, 505, 516, 486, 510, 529, 508, 522, 481, 0, 0, 0, 0, 0, 0, 0, 0, 523, 453, 534, 526, 562, 464, 514, 487, 0, 0, 0, 0, 0, 0, 0, 0, 528, 485, 535, 531, 539, 511, 524, 529, 0, 0, 0, 0, 0, 0, 0, 0, 483, 479, 502, 510, 531, 522, 535, 508, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4188, 4022, 4059, 4127, 4057, 4063, 4182, 4070] avg=4096.0000 max=4188 max_rank=0 min=4022 min_rank=1 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[3968, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[551, 503, 560, 504, 508, 543, 506, 521, 505, 509, 537, 510, 503, 497, 515, 518, 544, 513, 507, 520, 504, 475, 540, 512, 518, 511, 504, 526, 497, 533, 522, 504, 508, 512, 480, 489, 508, 513, 516, 483, 514, 529, 525, 522, 504, 534, 526, 533, 484, 522, 526, 527, 501, 464, 533, 536, 494, 524, 496, 492, 473, 501, 484, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[508, 527, 483, 512, 535, 499, 533, 516, 529, 540, 513, 505, 503, 510, 481, 519, 497, 505, 502, 541, 497, 567, 489, 521, 534, 484, 548, 535, 492, 551, 510, 512, 524, 506, 532, 500, 494, 522, 492, 491, 513, 558, 526, 508, 483, 497, 496, 497, 540, 497, 517, 481, 517, 524, 521, 506, 527, 498, 471, 483, 519, 505, 518, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[556, 497, 515, 542, 505, 503, 493, 488, 496, 515, 505, 522, 497, 489, 490, 532, 505, 483, 496, 523, 522, 520, 518, 492, 515, 536, 545, 492, 514, 527, 494, 522, 511, 513, 507, 524, 527, 521, 499, 520, 512, 509, 486, 521, 519, 519, 543, 522, 505, 518, 498, 487, 513, 494, 520, 514, 507, 562, 494, 506, 488, 521, 518, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[540, 528, 526, 519, 537, 534, 552, 484, 527, 533, 484, 572, 533, 502, 500, 518, 515, 479, 517, 511, 505, 495, 510, 499, 503, 504, 519, 552, 548, 514, 482, 509, 511, 511, 509, 457, 508, 499, 487, 490, 505, 511, 557, 506, 551, 520, 507, 515, 516, 508, 474, 502, 512, 511, 498, 497, 508, 505, 496, 555, 484, 506, 497, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[503, 526, 488, 533, 499, 531, 516, 492, 510, 480, 513, 475, 538, 499, 491, 515, 521, 508, 527, 456, 475, 510, 504, 514, 535, 488, 492, 500, 517, 537, 502, 524, 527, 515, 509, 513, 504, 525, 474, 524, 544, 509, 507, 514, 555, 520, 518, 496, 498, 517, 538, 536, 515, 507, 516, 526, 503, 527, 520, 484, 515, 523, 507, 563] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[519, 478, 522, 516, 478, 517, 507, 492, 492, 544, 481, 511, 514, 466, 507, 529, 519, 525, 544, 505, 508, 514, 548, 510, 538, 530, 524, 517, 503, 513, 496, 524, 482, 507, 499, 537, 485, 485, 526, 535, 522, 479, 534, 555, 498, 496, 507, 515, 523, 509, 472, 524, 538, 543, 519, 526, 506, 519, 475, 570, 501, 491, 492, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[477, 489, 531, 478, 543, 542, 517, 536, 508, 474, 485, 508, 506, 555, 500, 515, 531, 540, 514, 523, 532, 526, 529, 516, 503, 529, 505, 508, 491, 507, 501, 516, 509, 517, 501, 556, 518, 510, 502, 522, 446, 523, 476, 525, 555, 534, 519, 494, 511, 498, 522, 518, 525, 508, 496, 495, 515, 512, 522, 504, 507, 478, 511, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[540, 488, 490, 504, 531, 517, 512, 493, 484, 539, 507, 496, 517, 468, 491, 502, 514, 534, 495, 503, 546, 505, 515, 520, 545, 533, 514, 509, 483, 499, 491, 520, 526, 505, 516, 539, 513, 484, 502, 470, 522, 488, 533, 490, 521, 493, 497, 524, 525, 512, 517, 553, 545, 497, 536, 536, 490, 508, 505, 547, 486, 511, 538, 534] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[531, 503, 560, 504, 508, 543, 506, 521, 0, 0, 0, 0, 0, 0, 0, 0, 505, 509, 537, 510, 503, 497, 515, 518, 15, 14, 0, 0, 0, 0, 0, 0, 544, 513, 507, 520, 504, 475, 529, 512, 0, 0, 0, 0, 0, 0, 0, 0, 496, 511, 504, 526, 497, 533, 522, 504, 0, 0, 0, 0, 0, 0, 0, 0, 508, 512, 480, 489, 508, 513, 516, 483, 25, 7, 0, 0, 0, 0, 0, 0, 514, 529, 525, 522, 479, 534, 526, 533, 0, 0, 0, 0, 0, 0, 0, 0, 484, 522, 526, 527, 490, 464, 533, 536, 0, 0, 0, 0, 0, 0, 0, 0, 494, 524, 496, 492, 473, 501, 484, 495, 11, 11, 6, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4176, 4123, 4104, 4093, 4041, 4162, 4082, 3987] avg=4096.0000 max=4176 max_rank=0 min=3987 min_rank=7 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16272 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[484, 526, 488, 533, 499, 531, 516, 492, 0, 0, 0, 0, 0, 0, 0, 0, 510, 480, 513, 475, 538, 499, 491, 515, 15, 13, 0, 0, 0, 0, 0, 0, 521, 508, 527, 456, 475, 510, 494, 514, 0, 0, 0, 0, 0, 0, 0, 0, 513, 488, 492, 500, 517, 537, 502, 524, 0, 0, 0, 0, 0, 0, 0, 0, 527, 515, 509, 513, 504, 525, 474, 524, 27, 7, 0, 0, 0, 0, 0, 0, 544, 509, 507, 514, 528, 520, 518, 496, 0, 0, 0, 0, 0, 0, 0, 0, 498, 517, 538, 536, 504, 507, 516, 526, 0, 0, 0, 0, 0, 0, 0, 0, 503, 527, 520, 484, 515, 523, 507, 563, 11, 10, 6, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4049, 4005, 4073, 4125, 4136, 4142, 4169] avg=4096.0000 max=4169 max_rank=7 min=4005 min_rank=2 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16572 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[526, 497, 515, 542, 505, 503, 493, 488, 0, 0, 0, 0, 0, 0, 0, 0, 496, 515, 505, 522, 497, 489, 490, 532, 18, 20, 0, 0, 0, 0, 0, 0, 505, 483, 496, 523, 522, 520, 504, 492, 0, 0, 0, 0, 0, 0, 0, 0, 486, 536, 545, 492, 514, 527, 494, 522, 0, 0, 0, 0, 0, 0, 0, 0, 511, 513, 507, 524, 527, 521, 499, 520, 29, 11, 0, 0, 0, 0, 0, 0, 512, 509, 486, 521, 490, 519, 543, 522, 0, 0, 0, 0, 0, 0, 0, 0, 505, 518, 498, 487, 497, 494, 520, 514, 0, 0, 0, 0, 0, 0, 0, 0, 507, 562, 494, 506, 488, 521, 518, 521, 16, 14, 10, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4084, 4045, 4116, 4162, 4102, 4033, 4157] avg=4096.0000 max=4162 max_rank=4 min=4033 min_rank=6 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16454 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[499, 478, 522, 516, 478, 517, 507, 492, 0, 0, 0, 0, 0, 0, 0, 0, 492, 544, 481, 511, 514, 466, 507, 529, 16, 14, 0, 0, 0, 0, 0, 0, 519, 525, 544, 505, 508, 514, 537, 510, 0, 0, 0, 0, 0, 0, 0, 0, 515, 530, 524, 517, 503, 513, 496, 524, 0, 0, 0, 0, 0, 0, 0, 0, 482, 507, 499, 537, 485, 485, 526, 535, 24, 7, 0, 0, 0, 0, 0, 0, 522, 479, 534, 555, 474, 496, 507, 515, 0, 0, 0, 0, 0, 0, 0, 0, 523, 509, 472, 524, 526, 543, 519, 526, 0, 0, 0, 0, 0, 0, 0, 0, 506, 519, 475, 570, 501, 491, 492, 507, 12, 11, 6, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4009, 4074, 4162, 4122, 4087, 4082, 4142, 4090] avg=4096.0000 max=4162 max_rank=2 min=4009 min_rank=0 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16401 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[489, 527, 483, 512, 535, 499, 533, 516, 0, 0, 0, 0, 0, 0, 0, 0, 529, 540, 513, 505, 503, 510, 481, 519, 15, 13, 0, 0, 0, 0, 0, 0, 497, 505, 502, 541, 497, 567, 479, 521, 0, 0, 0, 0, 0, 0, 0, 0, 512, 484, 548, 535, 492, 551, 510, 512, 0, 0, 0, 0, 0, 0, 0, 0, 524, 506, 532, 500, 494, 522, 492, 491, 24, 7, 0, 0, 0, 0, 0, 0, 513, 558, 526, 508, 459, 497, 496, 497, 0, 0, 0, 0, 0, 0, 0, 0, 540, 497, 517, 481, 506, 524, 521, 506, 0, 0, 0, 0, 0, 0, 0, 0, 527, 498, 471, 483, 519, 505, 518, 507, 11, 10, 6, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4128, 4109, 4144, 4092, 4054, 4092, 4055] avg=4096.0000 max=4144 max_rank=3 min=4054 min_rank=5 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16293 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[520, 528, 526, 519, 537, 534, 552, 484, 0, 0, 0, 0, 0, 0, 0, 0, 527, 533, 484, 572, 533, 502, 500, 518, 15, 14, 0, 0, 0, 0, 0, 0, 515, 479, 517, 511, 505, 495, 500, 499, 0, 0, 0, 0, 0, 0, 0, 0, 482, 504, 519, 552, 548, 514, 482, 509, 0, 0, 0, 0, 0, 0, 0, 0, 511, 511, 509, 457, 508, 499, 487, 490, 27, 6, 0, 0, 0, 0, 0, 0, 505, 511, 557, 506, 524, 520, 507, 515, 0, 0, 0, 0, 0, 0, 0, 0, 516, 508, 474, 502, 501, 511, 498, 497, 0, 0, 0, 0, 0, 0, 0, 0, 508, 505, 496, 555, 484, 506, 497, 504, 11, 10, 6, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4200, 4198, 4021, 4110, 4005, 4145, 4007, 4082] avg=4096.0000 max=4200 max_rank=0 min=4005 min_rank=4 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16239 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[460, 489, 531, 478, 543, 542, 517, 536, 0, 0, 0, 0, 0, 0, 0, 0, 508, 474, 485, 508, 506, 555, 500, 515, 15, 12, 0, 0, 0, 0, 0, 0, 531, 540, 514, 523, 532, 526, 518, 516, 0, 0, 0, 0, 0, 0, 0, 0, 482, 529, 505, 508, 491, 507, 501, 516, 0, 0, 0, 0, 0, 0, 0, 0, 509, 517, 501, 556, 518, 510, 502, 522, 27, 6, 0, 0, 0, 0, 0, 0, 446, 523, 476, 525, 528, 534, 519, 494, 0, 0, 0, 0, 0, 0, 0, 0, 511, 498, 522, 518, 513, 508, 496, 495, 0, 0, 0, 0, 0, 0, 0, 0, 515, 512, 522, 504, 507, 478, 511, 504, 12, 11, 5, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4078, 4200, 4039, 4168, 4045, 4061, 4081] avg=4096.0000 max=4200 max_rank=2 min=4039 min_rank=3 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16355 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[520, 488, 490, 504, 531, 517, 512, 493, 0, 0, 0, 0, 0, 0, 0, 0, 484, 539, 507, 496, 517, 468, 491, 502, 16, 14, 0, 0, 0, 0, 0, 0, 514, 534, 495, 503, 546, 505, 505, 520, 0, 0, 0, 0, 0, 0, 0, 0, 522, 533, 514, 509, 483, 499, 491, 520, 0, 0, 0, 0, 0, 0, 0, 0, 526, 505, 516, 539, 513, 484, 502, 470, 26, 7, 0, 0, 0, 0, 0, 0, 522, 488, 533, 490, 495, 493, 497, 524, 0, 0, 0, 0, 0, 0, 0, 0, 525, 512, 517, 553, 533, 497, 536, 536, 0, 0, 0, 0, 0, 0, 0, 0, 490, 508, 505, 547, 486, 511, 538, 534, 12, 10, 6, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4055, 4034, 4122, 4071, 4088, 4042, 4209, 4147] avg=4096.0000 max=4209 max_rank=6 min=4034 min_rank=1 imbalance_ratio=1.027588 max_load_reduction=28559 max_load_reduction_pct=0.871552 moved_to_echo=16486 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[508, 493, 497, 537, 509, 524, 496, 481, 537, 521, 507, 490, 478, 480, 478, 491, 495, 534, 505, 502, 546, 561, 521, 547, 539, 465, 501, 532, 536, 506, 533, 528, 501, 501, 495, 515, 481, 562, 490, 493, 490, 505, 517, 502, 496, 522, 523, 499, 483, 538, 536, 523, 513, 532, 498, 526, 539, 523, 507, 521, 517, 518, 524, 500] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[510, 513, 510, 516, 524, 484, 513, 502, 494, 518, 511, 523, 529, 534, 555, 503, 503, 498, 487, 512, 559, 491, 491, 511, 526, 549, 499, 512, 502, 533, 480, 537, 481, 517, 500, 503, 516, 548, 521, 501, 487, 544, 509, 459, 522, 531, 562, 515, 544, 538, 493, 521, 505, 492, 501, 517, 505, 484, 495, 486, 531, 491, 514, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[474, 512, 510, 553, 493, 487, 508, 524, 526, 481, 526, 489, 520, 501, 531, 537, 494, 552, 504, 493, 515, 489, 511, 516, 534, 536, 513, 529, 538, 520, 478, 496, 550, 507, 542, 521, 494, 491, 492, 494, 521, 502, 495, 501, 513, 499, 521, 489, 531, 495, 524, 494, 547, 510, 525, 503, 486, 505, 497, 516, 506, 563, 530, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[531, 528, 494, 527, 543, 509, 530, 519, 481, 501, 535, 545, 465, 482, 533, 518, 502, 518, 567, 473, 489, 526, 511, 534, 543, 479, 505, 522, 510, 496, 494, 474, 466, 502, 469, 545, 515, 512, 505, 514, 517, 509, 533, 515, 524, 494, 509, 496, 522, 550, 532, 518, 511, 489, 560, 522, 560, 491, 499, 491, 503, 495, 500, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[520, 523, 528, 465, 520, 516, 523, 496, 537, 520, 515, 491, 495, 516, 530, 539, 514, 524, 541, 509, 510, 518, 500, 496, 536, 500, 518, 522, 509, 479, 509, 501, 514, 481, 501, 523, 510, 504, 511, 522, 493, 563, 507, 517, 508, 505, 485, 528, 509, 514, 516, 520, 526, 464, 546, 532, 514, 501, 531, 497, 454, 514, 538, 500] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[490, 516, 495, 529, 505, 549, 491, 467, 506, 486, 477, 517, 533, 562, 526, 554, 536, 529, 539, 508, 501, 505, 476, 515, 521, 497, 558, 528, 536, 519, 469, 551, 492, 508, 479, 552, 502, 510, 511, 479, 495, 535, 526, 521, 534, 530, 495, 499, 538, 519, 497, 523, 490, 468, 471, 521, 520, 489, 473, 499, 562, 521, 498, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[514, 533, 516, 472, 538, 489, 492, 520, 535, 523, 517, 510, 498, 490, 494, 527, 506, 491, 500, 514, 503, 499, 526, 518, 499, 546, 497, 523, 488, 494, 528, 512, 540, 490, 521, 546, 526, 493, 482, 480, 476, 501, 534, 547, 516, 536, 493, 516, 498, 536, 535, 509, 531, 530, 506, 563, 488, 486, 510, 499, 515, 515, 491, 547] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[477, 512, 526, 542, 517, 533, 488, 487, 485, 546, 499, 512, 503, 500, 500, 513, 494, 527, 501, 569, 519, 476, 483, 547, 464, 514, 519, 484, 493, 540, 511, 519, 541, 524, 526, 522, 509, 506, 521, 533, 551, 506, 512, 510, 490, 510, 532, 505, 495, 529, 532, 533, 518, 515, 505, 504, 515, 490, 513, 490, 531, 476, 498, 526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[508, 493, 497, 537, 509, 524, 496, 481, 7, 3, 2, 0, 0, 0, 0, 0, 537, 521, 507, 490, 478, 480, 478, 491, 0, 0, 0, 0, 0, 0, 0, 0, 495, 534, 505, 502, 546, 561, 521, 538, 0, 0, 0, 0, 0, 0, 0, 0, 532, 465, 501, 532, 536, 506, 533, 528, 0, 0, 0, 0, 0, 0, 0, 0, 501, 501, 495, 515, 481, 562, 490, 493, 13, 7, 0, 0, 0, 0, 0, 0, 490, 502, 517, 502, 496, 522, 523, 499, 0, 0, 0, 0, 0, 0, 0, 0, 483, 498, 536, 523, 513, 532, 498, 526, 0, 0, 0, 0, 0, 0, 0, 0, 539, 523, 507, 521, 517, 518, 524, 500, 27, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4057, 3982, 4202, 4133, 4058, 4051, 4109, 4176] avg=4096.0000 max=4202 max_rank=2 min=3982 min_rank=1 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16394 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[510, 513, 510, 516, 524, 484, 513, 502, 7, 3, 2, 0, 0, 0, 0, 0, 494, 518, 511, 523, 529, 534, 555, 503, 0, 0, 0, 0, 0, 0, 0, 0, 503, 498, 487, 512, 559, 491, 491, 502, 0, 0, 0, 0, 0, 0, 0, 0, 519, 549, 499, 512, 502, 533, 480, 537, 0, 0, 0, 0, 0, 0, 0, 0, 481, 517, 500, 503, 516, 548, 521, 501, 13, 7, 0, 0, 0, 0, 0, 0, 487, 541, 509, 459, 522, 531, 562, 515, 0, 0, 0, 0, 0, 0, 0, 0, 544, 498, 493, 521, 505, 492, 501, 517, 0, 0, 0, 0, 0, 0, 0, 0, 505, 484, 495, 486, 531, 491, 514, 506, 27, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4167, 4043, 4131, 4107, 4126, 4071, 4039] avg=4096.0000 max=4167 max_rank=1 min=4039 min_rank=7 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16343 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[490, 516, 495, 529, 505, 549, 491, 467, 7, 3, 2, 1, 0, 0, 0, 0, 506, 486, 477, 517, 533, 562, 526, 553, 0, 0, 0, 0, 0, 0, 0, 0, 536, 529, 539, 508, 501, 505, 476, 506, 0, 0, 0, 0, 0, 0, 0, 0, 514, 497, 558, 528, 536, 519, 469, 551, 0, 0, 0, 0, 0, 0, 0, 0, 492, 508, 479, 552, 502, 510, 511, 479, 12, 7, 0, 0, 0, 0, 0, 0, 495, 532, 526, 521, 534, 530, 495, 499, 0, 0, 0, 0, 0, 0, 0, 0, 538, 481, 497, 523, 490, 468, 471, 521, 0, 0, 0, 0, 0, 0, 0, 0, 520, 489, 473, 499, 562, 521, 498, 520, 26, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4055, 4160, 4100, 4172, 4052, 4132, 3989, 4108] avg=4096.0000 max=4172 max_rank=3 min=3989 min_rank=6 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16281 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[520, 523, 528, 465, 520, 516, 523, 496, 7, 3, 2, 1, 0, 0, 0, 0, 537, 520, 515, 491, 495, 516, 530, 538, 0, 0, 0, 0, 0, 0, 0, 0, 514, 524, 541, 509, 510, 518, 500, 487, 0, 0, 0, 0, 0, 0, 0, 0, 529, 500, 518, 522, 509, 479, 509, 501, 0, 0, 0, 0, 0, 0, 0, 0, 514, 481, 501, 523, 510, 504, 511, 522, 12, 7, 0, 0, 0, 0, 0, 0, 493, 560, 507, 517, 508, 505, 485, 528, 0, 0, 0, 0, 0, 0, 0, 0, 509, 476, 516, 520, 526, 464, 546, 532, 0, 0, 0, 0, 0, 0, 0, 0, 514, 501, 531, 497, 454, 514, 538, 500, 26, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4142, 4103, 4067, 4085, 4103, 4089, 4075] avg=4096.0000 max=4142 max_rank=1 min=4067 min_rank=3 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16352 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[474, 512, 510, 553, 493, 487, 508, 524, 7, 3, 2, 1, 0, 0, 0, 0, 526, 481, 526, 489, 520, 501, 531, 536, 0, 0, 0, 0, 0, 0, 0, 0, 494, 552, 504, 493, 515, 489, 511, 507, 0, 0, 0, 0, 0, 0, 0, 0, 527, 536, 513, 529, 538, 520, 478, 496, 0, 0, 0, 0, 0, 0, 0, 0, 550, 507, 542, 521, 494, 491, 492, 494, 12, 7, 0, 0, 0, 0, 0, 0, 521, 499, 495, 501, 513, 499, 521, 489, 0, 0, 0, 0, 0, 0, 0, 0, 531, 458, 524, 494, 547, 510, 525, 503, 0, 0, 0, 0, 0, 0, 0, 0, 486, 505, 497, 516, 506, 563, 530, 514, 25, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4110, 4065, 4137, 4110, 4038, 4092, 4142] avg=4096.0000 max=4142 max_rank=7 min=4038 min_rank=5 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[531, 528, 494, 527, 543, 509, 530, 519, 13, 8, 8, 4, 0, 0, 0, 0, 481, 501, 535, 545, 465, 482, 533, 514, 0, 0, 0, 0, 0, 0, 0, 0, 502, 518, 567, 473, 489, 526, 511, 514, 0, 0, 0, 0, 0, 0, 0, 0, 530, 479, 505, 522, 510, 496, 494, 474, 0, 0, 0, 0, 0, 0, 0, 0, 466, 502, 469, 545, 515, 512, 505, 514, 16, 12, 0, 0, 0, 0, 0, 0, 517, 501, 533, 515, 524, 494, 509, 496, 0, 0, 0, 0, 0, 0, 0, 0, 522, 504, 532, 518, 511, 489, 560, 522, 0, 0, 0, 0, 0, 0, 0, 0, 560, 491, 499, 491, 503, 495, 500, 516, 30, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4214, 4056, 4100, 4010, 4056, 4089, 4158, 4085] avg=4096.0000 max=4214 max_rank=0 min=4010 min_rank=3 imbalance_ratio=1.028809 max_load_reduction=28554 max_load_reduction_pct=0.871399 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[514, 533, 516, 472, 538, 489, 492, 520, 7, 3, 2, 1, 0, 0, 0, 0, 535, 523, 517, 510, 498, 490, 494, 526, 0, 0, 0, 0, 0, 0, 0, 0, 506, 491, 500, 514, 503, 499, 526, 509, 0, 0, 0, 0, 0, 0, 0, 0, 492, 546, 497, 523, 488, 494, 528, 512, 0, 0, 0, 0, 0, 0, 0, 0, 540, 490, 521, 546, 526, 493, 482, 480, 13, 7, 0, 0, 0, 0, 0, 0, 476, 498, 534, 547, 516, 536, 493, 516, 0, 0, 0, 0, 0, 0, 0, 0, 498, 496, 535, 509, 531, 530, 506, 563, 0, 0, 0, 0, 0, 0, 0, 0, 488, 486, 510, 499, 515, 515, 491, 547, 27, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4093, 4048, 4080, 4098, 4116, 4168, 4078] avg=4096.0000 max=4168 max_rank=6 min=4048 min_rank=2 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16460 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[477, 512, 526, 542, 517, 533, 488, 487, 6, 3, 2, 0, 0, 0, 0, 0, 485, 546, 499, 512, 503, 500, 500, 513, 0, 0, 0, 0, 0, 0, 0, 0, 494, 527, 501, 569, 519, 476, 483, 538, 0, 0, 0, 0, 0, 0, 0, 0, 458, 514, 519, 484, 493, 540, 511, 519, 0, 0, 0, 0, 0, 0, 0, 0, 541, 524, 526, 522, 509, 506, 521, 533, 13, 7, 0, 0, 0, 0, 0, 0, 551, 503, 512, 510, 490, 510, 532, 505, 0, 0, 0, 0, 0, 0, 0, 0, 495, 490, 532, 533, 518, 515, 505, 504, 0, 0, 0, 0, 0, 0, 0, 0, 515, 490, 513, 490, 531, 476, 498, 526, 26, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4058, 4107, 4038, 4202, 4113, 4092, 4065] avg=4096.0000 max=4202 max_rank=4 min=4038 min_rank=3 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16472 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4352, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 128, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 3968, 4224, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True + [2026-09-21 08:26:50] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 266.1 | throughput per GPU (TFLOP/s/GPU): 121.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.709802E+00 | loss scale: 1.0 | grad norm: 3.276 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[501, 499, 505, 517, 504, 527, 509, 494, 433, 514, 544, 492, 521, 455, 514, 552, 501, 546, 524, 562, 526, 527, 491, 491, 491, 499, 495, 491, 525, 522, 524, 494, 472, 535, 516, 490, 490, 514, 536, 514, 545, 567, 530, 518, 532, 494, 555, 527, 537, 490, 500, 473, 522, 470, 538, 465, 469, 528, 529, 530, 478, 532, 553, 529] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[510, 535, 500, 495, 481, 526, 529, 540, 525, 500, 510, 519, 468, 535, 502, 530, 487, 519, 548, 528, 502, 488, 501, 510, 514, 498, 532, 501, 529, 497, 514, 529, 478, 502, 487, 544, 508, 491, 538, 522, 519, 532, 550, 540, 510, 475, 547, 522, 490, 516, 501, 528, 513, 524, 482, 505, 522, 502, 500, 489, 501, 544, 473, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[525, 509, 526, 502, 522, 522, 501, 501, 522, 526, 514, 487, 522, 510, 523, 537, 485, 508, 465, 494, 535, 538, 477, 518, 529, 518, 502, 517, 502, 516, 522, 530, 570, 477, 534, 486, 498, 486, 504, 467, 562, 522, 514, 519, 527, 481, 521, 512, 501, 535, 505, 529, 483, 512, 555, 496, 478, 487, 506, 545, 509, 492, 535, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[507, 510, 530, 556, 523, 503, 513, 491, 510, 525, 508, 494, 545, 508, 465, 572, 510, 464, 478, 477, 508, 522, 540, 523, 527, 508, 487, 522, 538, 498, 515, 512, 510, 521, 494, 504, 489, 515, 544, 502, 509, 526, 493, 517, 520, 494, 523, 552, 497, 488, 490, 521, 541, 515, 521, 516, 490, 517, 499, 499, 506, 544, 481, 541] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[501, 555, 475, 496, 516, 506, 541, 533, 497, 517, 522, 523, 517, 518, 553, 533, 513, 471, 499, 523, 497, 527, 506, 502, 483, 506, 498, 521, 492, 479, 521, 544, 513, 496, 513, 530, 506, 553, 498, 516, 532, 525, 518, 488, 543, 486, 473, 513, 478, 512, 498, 546, 514, 473, 481, 541, 520, 531, 517, 527, 559, 501, 507, 475] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[487, 530, 487, 455, 529, 497, 509, 530, 480, 478, 500, 536, 544, 528, 492, 494, 508, 516, 528, 485, 556, 541, 551, 537, 546, 513, 502, 497, 555, 501, 528, 485, 526, 494, 477, 522, 493, 488, 527, 549, 492, 539, 530, 526, 479, 477, 512, 526, 550, 521, 515, 489, 513, 501, 487, 488, 506, 509, 551, 508, 502, 545, 488, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[508, 521, 539, 495, 516, 502, 477, 502, 479, 554, 519, 509, 530, 575, 479, 549, 529, 479, 519, 588, 506, 512, 546, 539, 562, 510, 503, 516, 499, 531, 512, 544, 542, 522, 503, 505, 498, 506, 505, 518, 502, 501, 525, 488, 500, 496, 510, 478, 478, 530, 497, 492, 510, 510, 524, 506, 494, 505, 498, 490, 508, 489, 498, 491] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[531, 480, 567, 523, 493, 505, 509, 545, 493, 521, 486, 515, 486, 494, 503, 505, 512, 555, 505, 487, 515, 518, 525, 515, 504, 481, 509, 565, 544, 499, 497, 512, 486, 549, 494, 524, 517, 490, 505, 558, 505, 487, 480, 492, 540, 490, 485, 503, 518, 513, 506, 574, 497, 547, 529, 523, 488, 496, 555, 490, 524, 494, 509, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[510, 535, 500, 495, 481, 526, 529, 540, 0, 0, 0, 0, 0, 0, 0, 0, 525, 500, 510, 519, 468, 535, 502, 525, 0, 0, 0, 0, 0, 0, 0, 0, 487, 519, 548, 528, 502, 473, 501, 510, 0, 0, 0, 0, 0, 0, 0, 0, 514, 498, 532, 501, 519, 497, 514, 529, 0, 0, 0, 0, 0, 0, 0, 0, 478, 502, 487, 544, 508, 491, 538, 522, 7, 5, 0, 0, 0, 0, 0, 0, 519, 517, 550, 540, 510, 475, 547, 522, 0, 0, 0, 0, 0, 0, 0, 0, 490, 516, 501, 528, 513, 524, 482, 505, 15, 1, 0, 0, 0, 0, 0, 0, 522, 502, 500, 489, 501, 544, 473, 511, 14, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4084, 4068, 4104, 4082, 4180, 4075, 4059] avg=4096.0000 max=4180 max_rank=5 min=4059 min_rank=7 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16396 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[501, 499, 505, 517, 504, 527, 509, 494, 0, 0, 0, 0, 0, 0, 0, 0, 433, 514, 544, 492, 521, 455, 514, 547, 0, 0, 0, 0, 0, 0, 0, 0, 501, 546, 524, 562, 526, 510, 491, 491, 0, 0, 0, 0, 0, 0, 0, 0, 491, 499, 495, 491, 515, 522, 524, 494, 0, 0, 0, 0, 0, 0, 0, 0, 472, 535, 516, 490, 490, 514, 536, 514, 7, 5, 0, 0, 0, 0, 0, 0, 545, 551, 530, 518, 532, 494, 555, 527, 0, 0, 0, 0, 0, 0, 0, 0, 537, 490, 500, 473, 522, 470, 538, 465, 17, 1, 0, 0, 0, 0, 0, 0, 469, 528, 529, 530, 478, 532, 553, 529, 15, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4056, 4020, 4151, 4031, 4079, 4252, 4013, 4166] avg=4096.0000 max=4252 max_rank=5 min=4013 min_rank=6 imbalance_ratio=1.038086 max_load_reduction=28516 max_load_reduction_pct=0.870239 moved_to_echo=16510 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[501, 555, 475, 496, 516, 506, 541, 533, 0, 0, 0, 0, 0, 0, 0, 0, 497, 517, 522, 523, 517, 518, 553, 528, 0, 0, 0, 0, 0, 0, 0, 0, 513, 471, 499, 523, 497, 510, 506, 502, 0, 0, 0, 0, 0, 0, 0, 0, 483, 506, 498, 521, 483, 479, 521, 544, 0, 0, 0, 0, 0, 0, 0, 0, 513, 496, 513, 530, 506, 553, 498, 516, 6, 5, 0, 0, 0, 0, 0, 0, 532, 510, 518, 488, 543, 486, 473, 513, 0, 0, 0, 0, 0, 0, 0, 0, 478, 512, 498, 546, 514, 473, 481, 541, 17, 1, 0, 0, 0, 0, 0, 0, 520, 531, 517, 527, 559, 501, 507, 475, 14, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4123, 4175, 4021, 4035, 4136, 4063, 4061, 4154] avg=4096.0000 max=4175 max_rank=1 min=4021 min_rank=2 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16414 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[507, 510, 530, 556, 523, 503, 513, 491, 0, 0, 0, 0, 0, 0, 0, 0, 510, 525, 508, 494, 545, 508, 465, 567, 0, 0, 0, 0, 0, 0, 0, 0, 510, 464, 478, 477, 508, 506, 540, 523, 0, 0, 0, 0, 0, 0, 0, 0, 527, 508, 487, 522, 528, 498, 515, 512, 0, 0, 0, 0, 0, 0, 0, 0, 510, 521, 494, 504, 489, 515, 544, 502, 7, 5, 0, 0, 0, 0, 0, 0, 509, 511, 493, 517, 520, 494, 523, 552, 0, 0, 0, 0, 0, 0, 0, 0, 497, 488, 490, 521, 541, 515, 521, 516, 16, 1, 0, 0, 0, 0, 0, 0, 490, 517, 499, 499, 506, 544, 481, 541, 14, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4133, 4122, 4006, 4097, 4091, 4119, 4106, 4094] avg=4096.0000 max=4133 max_rank=0 min=4006 min_rank=2 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16410 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[508, 521, 539, 495, 516, 502, 477, 502, 0, 0, 0, 0, 0, 0, 0, 0, 479, 554, 519, 509, 530, 575, 479, 544, 0, 0, 0, 0, 0, 0, 0, 0, 529, 479, 519, 588, 506, 496, 546, 539, 0, 0, 0, 0, 0, 0, 0, 0, 562, 510, 503, 516, 489, 531, 512, 544, 0, 0, 0, 0, 0, 0, 0, 0, 542, 522, 503, 505, 498, 506, 505, 518, 7, 5, 0, 0, 0, 0, 0, 0, 502, 487, 525, 488, 500, 496, 510, 478, 0, 0, 0, 0, 0, 0, 0, 0, 478, 530, 497, 492, 510, 510, 524, 506, 16, 1, 0, 0, 0, 0, 0, 0, 494, 505, 498, 490, 508, 489, 498, 491, 13, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4189, 4202, 4167, 4111, 3986, 4064, 3989] avg=4096.0000 max=4202 max_rank=2 min=3986 min_rank=5 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16150 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[487, 525, 487, 455, 529, 497, 509, 530, 0, 0, 0, 0, 0, 0, 0, 0, 480, 478, 500, 536, 544, 528, 492, 486, 0, 0, 0, 0, 0, 0, 0, 0, 508, 516, 528, 485, 556, 520, 551, 537, 0, 0, 0, 0, 0, 0, 0, 0, 546, 513, 502, 497, 535, 501, 528, 485, 0, 0, 0, 0, 0, 0, 0, 0, 526, 494, 477, 522, 493, 488, 527, 549, 11, 8, 5, 0, 0, 0, 0, 0, 492, 515, 530, 526, 479, 477, 512, 526, 0, 0, 0, 0, 0, 0, 0, 0, 550, 521, 515, 489, 513, 501, 487, 488, 21, 6, 0, 0, 0, 0, 0, 0, 506, 509, 551, 508, 502, 545, 488, 513, 18, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4019, 4044, 4201, 4107, 4100, 4057, 4091, 4149] avg=4096.0000 max=4201 max_rank=2 min=4019 min_rank=0 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[525, 509, 526, 502, 522, 522, 501, 501, 0, 0, 0, 0, 0, 0, 0, 0, 522, 526, 514, 487, 522, 510, 523, 532, 0, 0, 0, 0, 0, 0, 0, 0, 485, 508, 465, 494, 535, 521, 477, 518, 0, 0, 0, 0, 0, 0, 0, 0, 529, 518, 502, 517, 492, 516, 522, 530, 0, 0, 0, 0, 0, 0, 0, 0, 570, 477, 534, 486, 498, 486, 504, 467, 7, 5, 0, 0, 0, 0, 0, 0, 562, 507, 514, 519, 527, 481, 521, 512, 0, 0, 0, 0, 0, 0, 0, 0, 501, 535, 505, 529, 483, 512, 555, 496, 17, 1, 0, 0, 0, 0, 0, 0, 478, 487, 506, 545, 509, 492, 535, 515, 14, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4136, 4003, 4126, 4034, 4143, 4134, 4084] avg=4096.0000 max=4143 max_rank=5 min=4003 min_rank=2 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16395 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[531, 480, 567, 523, 493, 505, 509, 545, 0, 0, 0, 0, 0, 0, 0, 0, 493, 521, 486, 515, 486, 494, 503, 500, 0, 0, 0, 0, 0, 0, 0, 0, 512, 555, 505, 487, 515, 502, 525, 515, 0, 0, 0, 0, 0, 0, 0, 0, 504, 481, 509, 565, 534, 499, 497, 512, 0, 0, 0, 0, 0, 0, 0, 0, 486, 549, 494, 524, 517, 490, 505, 558, 7, 5, 0, 0, 0, 0, 0, 0, 505, 473, 480, 492, 540, 490, 485, 503, 0, 0, 0, 0, 0, 0, 0, 0, 518, 513, 506, 574, 497, 547, 529, 523, 16, 1, 0, 0, 0, 0, 0, 0, 488, 496, 555, 490, 524, 494, 509, 501, 13, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4153, 3998, 4116, 4101, 4135, 3968, 4224, 4073] avg=4096.0000 max=4224 max_rank=6 min=3968 min_rank=5 imbalance_ratio=1.031250 max_load_reduction=28544 max_load_reduction_pct=0.871094 moved_to_echo=16400 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3968, 4224, 4224, 4096, 4224, 4224, 4096, 4352, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3968, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 3968, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[532, 452, 509, 526, 528, 529, 493, 522, 493, 554, 497, 517, 531, 488, 518, 525, 494, 518, 508, 523, 533, 516, 511, 533, 519, 502, 473, 567, 490, 536, 531, 477, 508, 506, 515, 565, 543, 505, 488, 499, 516, 523, 492, 482, 476, 517, 494, 532, 476, 544, 515, 514, 485, 508, 495, 537, 531, 489, 506, 512, 537, 496, 506, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[537, 494, 501, 539, 545, 535, 495, 497, 506, 503, 499, 532, 520, 490, 474, 531, 505, 523, 474, 502, 489, 513, 536, 495, 529, 497, 538, 520, 492, 479, 519, 528, 508, 496, 507, 491, 512, 505, 527, 517, 505, 547, 545, 530, 488, 506, 512, 526, 471, 500, 484, 550, 529, 497, 504, 535, 540, 550, 494, 529, 490, 533, 489, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[515, 537, 525, 479, 541, 502, 509, 486, 482, 533, 491, 523, 471, 515, 518, 538, 541, 496, 501, 530, 480, 542, 519, 509, 547, 539, 503, 499, 466, 536, 553, 510, 506, 512, 520, 546, 484, 521, 502, 515, 518, 524, 494, 480, 517, 480, 535, 540, 484, 524, 514, 490, 508, 526, 530, 465, 498, 480, 502, 565, 533, 502, 504, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[446, 520, 536, 498, 527, 536, 526, 507, 536, 523, 517, 514, 509, 513, 506, 539, 524, 507, 480, 526, 488, 514, 512, 501, 553, 500, 510, 505, 490, 485, 511, 505, 508, 461, 488, 521, 502, 509, 506, 512, 547, 534, 530, 515, 503, 468, 533, 515, 519, 525, 510, 478, 533, 530, 524, 519, 509, 553, 503, 507, 504, 499, 524, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[513, 529, 523, 540, 520, 478, 510, 514, 522, 548, 533, 504, 519, 487, 524, 496, 486, 540, 530, 527, 526, 516, 566, 489, 526, 503, 494, 513, 513, 532, 517, 527, 512, 532, 550, 506, 474, 507, 530, 500, 479, 488, 480, 517, 519, 505, 502, 478, 506, 502, 495, 480, 537, 528, 506, 514, 524, 511, 509, 510, 488, 511, 523, 480] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[518, 535, 516, 533, 515, 544, 513, 516, 506, 525, 505, 466, 501, 516, 536, 493, 516, 506, 500, 515, 517, 496, 538, 514, 498, 542, 511, 492, 507, 471, 495, 546, 537, 446, 525, 532, 520, 467, 478, 516, 524, 537, 476, 529, 487, 519, 532, 509, 525, 491, 503, 501, 523, 507, 502, 561, 517, 523, 517, 504, 510, 512, 510, 526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[534, 513, 525, 499, 494, 500, 516, 506, 458, 524, 507, 556, 513, 516, 575, 503, 497, 498, 513, 495, 522, 484, 503, 525, 519, 498, 502, 520, 516, 512, 535, 515, 515, 501, 488, 553, 501, 505, 487, 520, 489, 527, 501, 534, 506, 523, 514, 494, 538, 539, 487, 494, 516, 528, 510, 519, 554, 469, 519, 529, 512, 519, 509, 475] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[520, 478, 483, 479, 549, 539, 506, 515, 512, 507, 473, 538, 484, 514, 467, 544, 520, 528, 517, 505, 506, 562, 510, 490, 525, 515, 506, 513, 522, 499, 537, 484, 478, 515, 521, 541, 518, 506, 490, 529, 489, 521, 515, 513, 505, 509, 508, 508, 510, 522, 540, 521, 503, 538, 514, 527, 497, 517, 525, 505, 509, 531, 504, 492] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[532, 452, 509, 526, 512, 529, 493, 522, 0, 0, 0, 0, 0, 0, 0, 0, 493, 553, 497, 517, 531, 488, 518, 525, 0, 0, 0, 0, 0, 0, 0, 0, 494, 518, 508, 523, 533, 516, 508, 533, 0, 0, 0, 0, 0, 0, 0, 0, 514, 502, 473, 567, 490, 536, 531, 477, 0, 0, 0, 0, 0, 0, 0, 0, 508, 506, 515, 565, 543, 505, 488, 499, 16, 4, 0, 0, 0, 0, 0, 0, 516, 523, 492, 482, 476, 517, 494, 532, 5, 3, 1, 1, 1, 0, 0, 0, 476, 544, 515, 514, 485, 508, 495, 532, 0, 0, 0, 0, 0, 0, 0, 0, 530, 489, 506, 512, 537, 496, 506, 511, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4122, 4133, 4090, 4149, 4043, 4069, 4087] avg=4096.0000 max=4149 max_rank=4 min=4043 min_rank=5 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16348 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[446, 520, 536, 498, 511, 536, 526, 507, 0, 0, 0, 0, 0, 0, 0, 0, 536, 522, 517, 514, 509, 513, 506, 539, 0, 0, 0, 0, 0, 0, 0, 0, 524, 507, 480, 526, 488, 514, 509, 501, 0, 0, 0, 0, 0, 0, 0, 0, 548, 500, 510, 505, 490, 485, 511, 505, 0, 0, 0, 0, 0, 0, 0, 0, 508, 461, 488, 521, 502, 509, 506, 512, 16, 4, 0, 0, 0, 0, 0, 0, 547, 534, 530, 515, 503, 468, 533, 515, 5, 3, 1, 1, 1, 0, 0, 0, 519, 525, 510, 478, 533, 530, 524, 514, 0, 0, 0, 0, 0, 0, 0, 0, 508, 553, 503, 507, 504, 499, 524, 515, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4156, 4049, 4054, 4027, 4156, 4133, 4113] avg=4096.0000 max=4156 max_rank=1 min=4027 min_rank=4 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[537, 494, 501, 539, 529, 535, 495, 497, 0, 0, 0, 0, 0, 0, 0, 0, 506, 502, 499, 532, 520, 490, 474, 531, 0, 0, 0, 0, 0, 0, 0, 0, 505, 523, 474, 502, 489, 513, 532, 495, 0, 0, 0, 0, 0, 0, 0, 0, 524, 497, 538, 520, 492, 479, 519, 528, 0, 0, 0, 0, 0, 0, 0, 0, 508, 496, 507, 491, 512, 505, 527, 517, 16, 4, 0, 0, 0, 0, 0, 0, 505, 547, 545, 530, 488, 506, 512, 526, 5, 4, 1, 1, 1, 0, 0, 0, 471, 500, 484, 550, 529, 497, 504, 530, 0, 0, 0, 0, 0, 0, 0, 0, 539, 550, 494, 529, 490, 533, 489, 514, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4127, 4054, 4033, 4097, 4083, 4171, 4065, 4138] avg=4096.0000 max=4171 max_rank=5 min=4033 min_rank=2 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16457 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[513, 529, 523, 540, 505, 478, 510, 514, 0, 0, 0, 0, 0, 0, 0, 0, 522, 547, 533, 504, 519, 487, 524, 496, 0, 0, 0, 0, 0, 0, 0, 0, 486, 540, 530, 527, 526, 516, 562, 489, 0, 0, 0, 0, 0, 0, 0, 0, 521, 503, 494, 513, 513, 532, 517, 527, 0, 0, 0, 0, 0, 0, 0, 0, 512, 532, 550, 506, 474, 507, 530, 500, 15, 4, 0, 0, 0, 0, 0, 0, 479, 488, 480, 517, 519, 505, 502, 478, 5, 4, 1, 1, 1, 0, 0, 0, 506, 502, 495, 480, 537, 528, 506, 509, 0, 0, 0, 0, 0, 0, 0, 0, 523, 511, 509, 510, 488, 511, 523, 480, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4132, 4176, 4120, 4130, 3980, 4063, 4055] avg=4096.0000 max=4176 max_rank=2 min=3980 min_rank=5 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16228 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[534, 513, 525, 499, 479, 500, 516, 506, 0, 0, 0, 0, 0, 0, 0, 0, 458, 523, 507, 556, 513, 516, 575, 503, 0, 0, 0, 0, 0, 0, 0, 0, 497, 498, 513, 495, 522, 484, 500, 525, 0, 0, 0, 0, 0, 0, 0, 0, 514, 498, 502, 520, 516, 512, 535, 515, 0, 0, 0, 0, 0, 0, 0, 0, 515, 501, 488, 553, 501, 505, 487, 520, 15, 4, 0, 0, 0, 0, 0, 0, 489, 527, 501, 534, 506, 523, 514, 494, 5, 3, 1, 1, 1, 0, 0, 0, 538, 539, 487, 494, 516, 528, 510, 514, 0, 0, 0, 0, 0, 0, 0, 0, 553, 469, 519, 529, 512, 519, 509, 475, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4151, 4034, 4112, 4089, 4099, 4126, 4085] avg=4096.0000 max=4151 max_rank=1 min=4034 min_rank=2 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[515, 537, 525, 479, 521, 502, 509, 486, 0, 0, 0, 0, 0, 0, 0, 0, 482, 530, 491, 523, 471, 515, 518, 538, 0, 0, 0, 0, 0, 0, 0, 0, 541, 496, 501, 530, 480, 542, 511, 509, 0, 0, 0, 0, 0, 0, 0, 0, 536, 539, 503, 499, 466, 536, 553, 510, 0, 0, 0, 0, 0, 0, 0, 0, 506, 512, 520, 546, 484, 521, 502, 515, 20, 6, 0, 0, 0, 0, 0, 0, 518, 524, 494, 480, 517, 480, 535, 540, 7, 8, 5, 4, 3, 0, 0, 0, 484, 524, 514, 490, 508, 526, 530, 458, 0, 0, 0, 0, 0, 0, 0, 0, 494, 480, 502, 565, 533, 502, 504, 513, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4068, 4110, 4142, 4132, 4115, 4034, 4093] avg=4096.0000 max=4142 max_rank=3 min=4034 min_rank=6 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16374 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[518, 535, 516, 533, 500, 544, 513, 516, 0, 0, 0, 0, 0, 0, 0, 0, 506, 524, 505, 466, 501, 516, 536, 493, 0, 0, 0, 0, 0, 0, 0, 0, 516, 506, 500, 515, 517, 496, 534, 514, 0, 0, 0, 0, 0, 0, 0, 0, 493, 542, 511, 492, 507, 471, 495, 546, 0, 0, 0, 0, 0, 0, 0, 0, 537, 446, 525, 532, 520, 467, 478, 516, 15, 4, 0, 0, 0, 0, 0, 0, 524, 537, 476, 529, 487, 519, 532, 509, 5, 4, 1, 1, 1, 0, 0, 0, 525, 491, 503, 501, 523, 507, 502, 556, 0, 0, 0, 0, 0, 0, 0, 0, 516, 523, 517, 504, 510, 512, 510, 526, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4175, 4047, 4098, 4057, 4040, 4125, 4108, 4118] avg=4096.0000 max=4175 max_rank=0 min=4040 min_rank=4 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[520, 478, 483, 479, 533, 539, 506, 515, 0, 0, 0, 0, 0, 0, 0, 0, 512, 506, 473, 538, 484, 514, 467, 544, 0, 0, 0, 0, 0, 0, 0, 0, 520, 528, 517, 505, 506, 562, 507, 490, 0, 0, 0, 0, 0, 0, 0, 0, 520, 515, 506, 513, 522, 499, 537, 484, 0, 0, 0, 0, 0, 0, 0, 0, 478, 515, 521, 541, 518, 506, 490, 529, 16, 4, 0, 0, 0, 0, 0, 0, 489, 521, 515, 513, 505, 509, 508, 508, 5, 3, 1, 1, 1, 0, 0, 0, 510, 522, 540, 521, 503, 538, 514, 522, 0, 0, 0, 0, 0, 0, 0, 0, 496, 517, 525, 505, 509, 531, 504, 492, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4053, 4038, 4135, 4096, 4118, 4079, 4170, 4079] avg=4096.0000 max=4170 max_rank=6 min=4038 min_rank=1 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16446 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33920 observed_loss=-1152 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 128, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4352, 4096, 4096, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33920 slack_slots=1792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[529, 522, 471, 489, 505, 488, 544, 504, 510, 530, 525, 528, 500, 515, 508, 515, 492, 497, 521, 517, 485, 500, 532, 495, 534, 513, 538, 491, 493, 488, 524, 480, 524, 537, 515, 495, 507, 488, 527, 525, 560, 527, 537, 538, 542, 517, 480, 495, 548, 493, 507, 498, 534, 497, 493, 513, 565, 534, 488, 516, 485, 492, 503, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[524, 552, 530, 521, 520, 522, 514, 483, 524, 527, 519, 514, 538, 517, 513, 500, 522, 538, 507, 522, 475, 509, 493, 541, 475, 544, 509, 522, 482, 509, 525, 516, 482, 475, 514, 496, 536, 496, 481, 519, 518, 480, 538, 540, 514, 478, 513, 516, 482, 543, 506, 542, 477, 519, 506, 515, 464, 500, 509, 526, 501, 514, 547, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[513, 513, 535, 498, 492, 483, 498, 535, 518, 557, 515, 475, 509, 504, 545, 512, 524, 547, 502, 475, 502, 530, 511, 523, 499, 505, 540, 538, 551, 498, 498, 530, 510, 481, 475, 498, 530, 526, 502, 533, 478, 461, 506, 473, 529, 537, 524, 509, 559, 517, 486, 502, 552, 516, 490, 545, 540, 484, 527, 508, 517, 494, 492, 492] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[517, 553, 491, 488, 547, 507, 508, 484, 521, 480, 540, 489, 524, 493, 498, 549, 489, 538, 533, 530, 533, 502, 501, 531, 466, 505, 507, 513, 521, 505, 482, 524, 498, 513, 534, 507, 502, 502, 524, 521, 511, 504, 510, 490, 507, 476, 537, 513, 488, 539, 484, 541, 498, 504, 499, 526, 554, 499, 525, 501, 529, 502, 533, 528] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[511, 515, 502, 522, 538, 509, 538, 506, 500, 487, 507, 482, 528, 508, 535, 542, 509, 492, 501, 523, 503, 518, 520, 507, 506, 527, 527, 472, 515, 537, 525, 495, 531, 514, 522, 475, 527, 553, 545, 515, 474, 508, 510, 522, 496, 501, 504, 497, 480, 520, 520, 489, 483, 550, 472, 529, 460, 510, 514, 516, 542, 533, 522, 527] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[529, 514, 499, 496, 473, 500, 527, 541, 519, 483, 521, 503, 492, 516, 519, 491, 567, 510, 530, 513, 493, 521, 491, 495, 574, 519, 493, 501, 547, 543, 516, 543, 482, 515, 531, 492, 513, 510, 502, 492, 508, 489, 511, 524, 503, 477, 517, 540, 525, 493, 525, 520, 512, 486, 503, 528, 511, 552, 471, 480, 526, 489, 531, 531] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[538, 500, 489, 520, 533, 503, 496, 544, 502, 552, 496, 524, 532, 541, 523, 499, 526, 548, 452, 521, 485, 511, 510, 540, 481, 544, 538, 497, 498, 520, 523, 458, 493, 511, 515, 507, 496, 484, 516, 517, 516, 501, 494, 513, 487, 517, 484, 520, 502, 507, 540, 505, 462, 489, 570, 487, 540, 466, 545, 520, 540, 542, 527, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[503, 516, 489, 537, 519, 496, 502, 501, 518, 534, 493, 494, 503, 484, 550, 563, 535, 489, 515, 549, 515, 497, 528, 567, 557, 508, 525, 538, 483, 523, 501, 546, 506, 515, 496, 513, 494, 493, 502, 542, 492, 484, 520, 469, 516, 549, 525, 499, 513, 520, 484, 490, 526, 539, 482, 496, 561, 465, 524, 492, 492, 509, 473, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[524, 550, 530, 521, 520, 522, 514, 483, 0, 0, 0, 0, 0, 0, 0, 0, 524, 527, 519, 514, 538, 517, 491, 500, 0, 0, 0, 0, 0, 0, 0, 0, 522, 538, 507, 522, 475, 509, 493, 525, 0, 0, 0, 0, 0, 0, 0, 0, 475, 544, 493, 522, 482, 509, 525, 516, 0, 0, 0, 0, 0, 0, 0, 0, 482, 475, 514, 496, 536, 496, 481, 519, 12, 9, 0, 0, 0, 0, 0, 0, 518, 480, 538, 540, 514, 478, 513, 516, 22, 7, 0, 0, 0, 0, 0, 0, 482, 543, 506, 542, 477, 519, 506, 515, 5, 4, 2, 0, 0, 0, 0, 0, 459, 500, 509, 526, 501, 514, 547, 514, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4164, 4130, 4091, 4066, 4020, 4126, 4101, 4070] avg=4096.0000 max=4164 max_rank=0 min=4020 min_rank=4 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16317 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[529, 520, 471, 489, 505, 488, 544, 504, 0, 0, 0, 0, 0, 0, 0, 0, 510, 530, 525, 528, 500, 515, 486, 515, 0, 0, 0, 0, 0, 0, 0, 0, 492, 497, 521, 517, 485, 500, 532, 481, 0, 0, 0, 0, 0, 0, 0, 0, 534, 513, 521, 491, 493, 488, 524, 480, 0, 0, 0, 0, 0, 0, 0, 0, 524, 537, 515, 495, 507, 488, 527, 525, 11, 10, 0, 0, 0, 0, 0, 0, 560, 527, 537, 538, 542, 517, 480, 495, 22, 7, 0, 0, 0, 0, 0, 0, 548, 493, 507, 498, 534, 497, 493, 513, 6, 3, 2, 0, 0, 0, 0, 0, 559, 534, 488, 516, 485, 492, 503, 505, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4050, 4109, 4025, 4044, 4139, 4225, 4094, 4082] avg=4096.0000 max=4225 max_rank=5 min=4025 min_rank=2 imbalance_ratio=1.031494 max_load_reduction=28543 max_load_reduction_pct=0.871063 moved_to_echo=16540 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[517, 551, 491, 488, 547, 507, 508, 484, 0, 0, 0, 0, 0, 0, 0, 0, 521, 480, 540, 489, 524, 493, 477, 549, 0, 0, 0, 0, 0, 0, 0, 0, 489, 538, 533, 530, 533, 502, 501, 515, 0, 0, 0, 0, 0, 0, 0, 0, 466, 505, 491, 513, 521, 505, 482, 524, 0, 0, 0, 0, 0, 0, 0, 0, 498, 513, 534, 507, 502, 502, 524, 521, 12, 9, 0, 0, 0, 0, 0, 0, 511, 504, 510, 490, 507, 476, 537, 513, 21, 7, 0, 0, 0, 0, 0, 0, 488, 539, 484, 541, 498, 504, 499, 526, 6, 4, 2, 0, 0, 0, 0, 0, 548, 499, 525, 501, 529, 502, 533, 528, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4073, 4141, 4007, 4122, 4076, 4091, 4165] avg=4096.0000 max=4165 max_rank=7 min=4007 min_rank=3 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16454 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[513, 509, 535, 498, 492, 483, 498, 535, 0, 0, 0, 0, 0, 0, 0, 0, 518, 557, 515, 475, 509, 504, 517, 512, 0, 0, 0, 0, 0, 0, 0, 0, 524, 547, 502, 475, 502, 530, 511, 500, 0, 0, 0, 0, 0, 0, 0, 0, 499, 505, 516, 538, 551, 498, 498, 530, 0, 0, 0, 0, 0, 0, 0, 0, 510, 481, 475, 498, 530, 526, 502, 533, 15, 14, 0, 0, 0, 0, 0, 0, 478, 461, 506, 473, 529, 537, 524, 509, 28, 10, 0, 0, 0, 0, 0, 0, 559, 517, 486, 502, 552, 516, 490, 545, 11, 8, 4, 0, 0, 0, 0, 0, 529, 484, 527, 508, 517, 494, 492, 492, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4107, 4091, 4135, 4084, 4055, 4190, 4043] avg=4096.0000 max=4190 max_rank=6 min=4043 min_rank=7 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[529, 512, 499, 496, 473, 500, 527, 541, 0, 0, 0, 0, 0, 0, 0, 0, 519, 483, 521, 503, 492, 516, 497, 491, 0, 0, 0, 0, 0, 0, 0, 0, 567, 510, 530, 513, 493, 521, 491, 481, 0, 0, 0, 0, 0, 0, 0, 0, 574, 519, 478, 501, 547, 543, 516, 543, 0, 0, 0, 0, 0, 0, 0, 0, 482, 515, 531, 492, 513, 510, 502, 492, 11, 9, 0, 0, 0, 0, 0, 0, 508, 489, 511, 524, 503, 477, 517, 540, 22, 6, 0, 0, 0, 0, 0, 0, 525, 493, 525, 520, 512, 486, 503, 528, 6, 3, 2, 0, 0, 0, 0, 0, 505, 552, 471, 480, 526, 489, 531, 531, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4022, 4106, 4221, 4057, 4097, 4103, 4085] avg=4096.0000 max=4221 max_rank=3 min=4022 min_rank=1 imbalance_ratio=1.030518 max_load_reduction=28547 max_load_reduction_pct=0.871185 moved_to_echo=16342 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[511, 513, 502, 522, 538, 509, 538, 506, 0, 0, 0, 0, 0, 0, 0, 0, 500, 487, 507, 482, 528, 508, 512, 542, 0, 0, 0, 0, 0, 0, 0, 0, 509, 492, 501, 523, 503, 518, 520, 493, 0, 0, 0, 0, 0, 0, 0, 0, 506, 527, 511, 472, 515, 537, 525, 495, 0, 0, 0, 0, 0, 0, 0, 0, 531, 514, 522, 475, 527, 553, 545, 515, 11, 9, 0, 0, 0, 0, 0, 0, 474, 508, 510, 522, 496, 501, 504, 497, 23, 7, 0, 0, 0, 0, 0, 0, 480, 520, 520, 489, 483, 550, 472, 529, 5, 3, 2, 0, 0, 0, 0, 0, 455, 510, 514, 516, 542, 533, 522, 527, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4139, 4066, 4059, 4088, 4202, 4042, 4053, 4119] avg=4096.0000 max=4202 max_rank=4 min=4042 min_rank=5 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16416 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[538, 498, 489, 520, 533, 503, 496, 544, 0, 0, 0, 0, 0, 0, 0, 0, 502, 552, 496, 524, 532, 541, 500, 499, 0, 0, 0, 0, 0, 0, 0, 0, 526, 548, 452, 521, 485, 511, 510, 524, 0, 0, 0, 0, 0, 0, 0, 0, 481, 544, 521, 497, 498, 520, 523, 458, 0, 0, 0, 0, 0, 0, 0, 0, 493, 511, 515, 507, 496, 484, 516, 517, 12, 10, 0, 0, 0, 0, 0, 0, 516, 501, 494, 513, 487, 517, 484, 520, 23, 7, 0, 0, 0, 0, 0, 0, 502, 507, 540, 505, 462, 489, 570, 487, 6, 4, 2, 0, 0, 0, 0, 0, 534, 466, 545, 520, 540, 542, 527, 511, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4146, 4077, 4042, 4061, 4062, 4074, 4185] avg=4096.0000 max=4185 max_rank=7 min=4042 min_rank=3 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[503, 514, 489, 537, 519, 496, 502, 501, 0, 0, 0, 0, 0, 0, 0, 0, 518, 534, 493, 494, 503, 484, 526, 563, 0, 0, 0, 0, 0, 0, 0, 0, 535, 489, 515, 549, 515, 497, 528, 550, 0, 0, 0, 0, 0, 0, 0, 0, 557, 508, 509, 538, 483, 523, 501, 546, 0, 0, 0, 0, 0, 0, 0, 0, 506, 515, 496, 513, 494, 493, 502, 542, 13, 9, 0, 0, 0, 0, 0, 0, 492, 484, 520, 469, 516, 549, 525, 499, 24, 7, 0, 0, 0, 0, 0, 0, 513, 520, 484, 490, 526, 539, 482, 496, 6, 4, 2, 0, 0, 0, 0, 0, 555, 465, 524, 492, 492, 509, 473, 509, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4115, 4178, 4165, 4083, 4085, 4062, 4019] avg=4096.0000 max=4178 max_rank=2 min=4019 min_rank=7 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16249 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 3968, 4224, 4096, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[513, 527, 506, 507, 534, 515, 559, 483, 506, 502, 544, 531, 506, 529, 500, 508, 511, 477, 489, 516, 544, 475, 502, 518, 501, 528, 488, 567, 521, 529, 549, 514, 466, 473, 559, 490, 549, 516, 484, 552, 504, 508, 483, 533, 551, 530, 527, 503, 549, 526, 504, 497, 543, 515, 523, 514, 471, 459, 484, 502, 479, 451, 493, 531] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[514, 504, 491, 523, 531, 496, 539, 535, 512, 508, 517, 482, 501, 533, 477, 524, 522, 525, 509, 527, 495, 504, 500, 517, 528, 539, 545, 518, 512, 498, 509, 479, 505, 516, 520, 531, 463, 515, 509, 533, 505, 507, 509, 512, 531, 507, 518, 502, 491, 529, 505, 532, 514, 496, 527, 511, 525, 498, 512, 538, 483, 485, 519, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[507, 520, 526, 539, 506, 499, 538, 505, 547, 546, 505, 495, 549, 522, 486, 457, 525, 508, 515, 512, 477, 530, 508, 506, 474, 483, 478, 500, 531, 509, 472, 534, 475, 503, 509, 509, 536, 525, 523, 536, 569, 531, 490, 509, 482, 507, 500, 507, 514, 526, 544, 491, 507, 516, 513, 523, 527, 519, 496, 515, 492, 499, 495, 571] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[559, 518, 548, 476, 483, 502, 549, 487, 514, 509, 490, 510, 522, 520, 526, 487, 519, 549, 483, 487, 517, 526, 518, 509, 498, 522, 557, 501, 517, 476, 516, 526, 532, 489, 519, 487, 534, 489, 514, 484, 496, 506, 479, 537, 509, 531, 514, 476, 524, 525, 492, 544, 493, 510, 505, 493, 516, 522, 522, 534, 543, 529, 503, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[536, 471, 506, 503, 568, 504, 531, 450, 538, 520, 510, 484, 505, 488, 513, 542, 521, 506, 531, 492, 508, 516, 516, 535, 512, 521, 501, 505, 477, 512, 517, 514, 533, 503, 517, 538, 512, 548, 493, 456, 516, 516, 483, 530, 522, 502, 546, 508, 487, 524, 505, 514, 515, 518, 505, 543, 484, 479, 532, 535, 478, 525, 542, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[535, 481, 518, 496, 523, 505, 542, 531, 522, 514, 533, 494, 505, 513, 524, 510, 489, 508, 549, 493, 561, 528, 512, 493, 536, 467, 508, 497, 541, 545, 490, 547, 524, 521, 516, 503, 520, 503, 527, 484, 501, 515, 514, 492, 496, 503, 518, 496, 518, 535, 470, 516, 548, 501, 499, 532, 509, 494, 494, 511, 515, 499, 501, 483] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[514, 529, 478, 506, 528, 487, 488, 518, 495, 540, 506, 496, 499, 546, 515, 512, 493, 516, 541, 498, 495, 511, 492, 549, 515, 544, 496, 484, 546, 521, 493, 510, 540, 495, 496, 502, 474, 481, 542, 481, 498, 539, 523, 501, 565, 521, 529, 527, 507, 528, 525, 482, 513, 513, 535, 503, 490, 540, 508, 533, 503, 504, 497, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[539, 529, 489, 509, 551, 483, 486, 514, 504, 501, 537, 539, 500, 531, 526, 528, 511, 518, 514, 512, 479, 513, 503, 484, 544, 513, 518, 467, 543, 488, 512, 486, 542, 508, 497, 492, 522, 510, 486, 489, 505, 520, 509, 531, 525, 523, 477, 548, 478, 505, 557, 492, 506, 520, 470, 492, 559, 505, 477, 541, 537, 510, 515, 549] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[514, 504, 491, 523, 531, 496, 524, 535, 0, 0, 0, 0, 0, 0, 0, 0, 512, 508, 517, 482, 501, 522, 477, 524, 0, 0, 0, 0, 0, 0, 0, 0, 522, 525, 509, 527, 495, 504, 500, 517, 5, 2, 0, 0, 0, 0, 0, 0, 528, 539, 545, 518, 510, 498, 509, 479, 0, 0, 0, 0, 0, 0, 0, 0, 505, 516, 520, 531, 463, 515, 509, 533, 11, 5, 4, 0, 0, 0, 0, 0, 505, 507, 509, 512, 519, 507, 518, 502, 0, 0, 0, 0, 0, 0, 0, 0, 491, 519, 505, 532, 514, 496, 527, 511, 0, 0, 0, 0, 0, 0, 0, 0, 525, 498, 512, 538, 483, 485, 519, 506, 15, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4043, 4106, 4126, 4112, 4079, 4095, 4089] avg=4096.0000 max=4126 max_rank=3 min=4043 min_rank=1 imbalance_ratio=1.007324 max_load_reduction=28642 max_load_reduction_pct=0.874084 moved_to_echo=16375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[559, 518, 548, 476, 483, 502, 534, 487, 0, 0, 0, 0, 0, 0, 0, 0, 514, 509, 490, 510, 522, 510, 526, 487, 0, 0, 0, 0, 0, 0, 0, 0, 519, 549, 483, 487, 517, 526, 518, 509, 5, 2, 0, 0, 0, 0, 0, 0, 498, 522, 557, 501, 515, 476, 516, 526, 0, 0, 0, 0, 0, 0, 0, 0, 532, 489, 519, 487, 534, 489, 514, 484, 10, 5, 4, 0, 0, 0, 0, 0, 496, 506, 479, 537, 497, 531, 514, 476, 0, 0, 0, 0, 0, 0, 0, 0, 524, 515, 492, 544, 493, 510, 505, 493, 0, 0, 0, 0, 0, 0, 0, 0, 516, 522, 522, 534, 543, 529, 503, 496, 15, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4068, 4115, 4111, 4067, 4036, 4076, 4188] avg=4096.0000 max=4188 max_rank=7 min=4036 min_rank=5 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[513, 527, 506, 507, 534, 515, 544, 483, 0, 0, 0, 0, 0, 0, 0, 0, 506, 502, 544, 531, 506, 518, 500, 508, 0, 0, 0, 0, 0, 0, 0, 0, 511, 477, 489, 516, 544, 475, 502, 518, 5, 2, 0, 0, 0, 0, 0, 0, 501, 528, 488, 567, 519, 529, 549, 514, 0, 0, 0, 0, 0, 0, 0, 0, 466, 473, 559, 490, 549, 516, 484, 552, 11, 5, 4, 0, 0, 0, 0, 0, 504, 508, 483, 533, 539, 530, 527, 503, 0, 0, 0, 0, 0, 0, 0, 0, 549, 516, 504, 497, 543, 515, 523, 514, 0, 0, 0, 0, 0, 0, 0, 0, 471, 459, 484, 502, 479, 451, 493, 531, 15, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4129, 4115, 4039, 4195, 4109, 4127, 4161, 3893] avg=4096.0000 max=4195 max_rank=3 min=3893 min_rank=7 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16290 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[507, 520, 526, 539, 506, 499, 519, 505, 0, 0, 0, 0, 0, 0, 0, 0, 547, 546, 505, 495, 549, 509, 486, 457, 0, 0, 0, 0, 0, 0, 0, 0, 525, 508, 515, 512, 477, 530, 508, 506, 7, 7, 0, 0, 0, 0, 0, 0, 474, 483, 478, 500, 524, 509, 472, 534, 0, 0, 0, 0, 0, 0, 0, 0, 475, 503, 509, 509, 536, 525, 523, 536, 13, 9, 8, 0, 0, 0, 0, 0, 569, 531, 490, 509, 463, 507, 500, 507, 0, 0, 0, 0, 0, 0, 0, 0, 514, 510, 544, 491, 507, 516, 513, 523, 0, 0, 0, 0, 0, 0, 0, 0, 527, 519, 496, 515, 492, 499, 495, 571, 19, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4094, 4095, 3974, 4146, 4076, 4118, 4144] avg=4096.0000 max=4146 max_rank=4 min=3974 min_rank=3 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16484 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[536, 471, 506, 503, 568, 504, 517, 450, 0, 0, 0, 0, 0, 0, 0, 0, 538, 520, 510, 484, 505, 478, 513, 542, 0, 0, 0, 0, 0, 0, 0, 0, 521, 506, 531, 492, 508, 516, 516, 535, 4, 2, 0, 0, 0, 0, 0, 0, 512, 521, 501, 505, 475, 512, 517, 514, 0, 0, 0, 0, 0, 0, 0, 0, 533, 503, 517, 538, 512, 548, 493, 456, 10, 5, 4, 0, 0, 0, 0, 0, 516, 516, 483, 530, 510, 502, 546, 508, 0, 0, 0, 0, 0, 0, 0, 0, 487, 515, 505, 514, 515, 518, 505, 543, 0, 0, 0, 0, 0, 0, 0, 0, 484, 479, 532, 535, 478, 525, 542, 506, 14, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4055, 4090, 4131, 4057, 4119, 4111, 4102, 4103] avg=4096.0000 max=4131 max_rank=2 min=4055 min_rank=0 imbalance_ratio=1.008545 max_load_reduction=28637 max_load_reduction_pct=0.873932 moved_to_echo=16435 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[535, 481, 518, 496, 523, 505, 527, 531, 0, 0, 0, 0, 0, 0, 0, 0, 522, 514, 533, 494, 505, 503, 524, 510, 0, 0, 0, 0, 0, 0, 0, 0, 489, 508, 549, 493, 561, 528, 512, 493, 5, 2, 0, 0, 0, 0, 0, 0, 536, 467, 508, 497, 539, 545, 490, 547, 0, 0, 0, 0, 0, 0, 0, 0, 524, 521, 516, 503, 520, 503, 527, 484, 10, 5, 4, 0, 0, 0, 0, 0, 501, 515, 514, 492, 485, 503, 518, 496, 0, 0, 0, 0, 0, 0, 0, 0, 518, 525, 470, 516, 548, 501, 499, 532, 0, 0, 0, 0, 0, 0, 0, 0, 509, 494, 494, 511, 515, 499, 501, 483, 15, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4105, 4140, 4129, 4117, 4024, 4109, 4028] avg=4096.0000 max=4140 max_rank=2 min=4024 min_rank=5 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16278 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[514, 529, 478, 506, 528, 487, 475, 518, 0, 0, 0, 0, 0, 0, 0, 0, 495, 540, 506, 496, 499, 535, 515, 512, 0, 0, 0, 0, 0, 0, 0, 0, 493, 516, 541, 498, 495, 511, 492, 549, 5, 2, 0, 0, 0, 0, 0, 0, 515, 544, 496, 484, 544, 521, 493, 510, 0, 0, 0, 0, 0, 0, 0, 0, 540, 495, 496, 502, 474, 481, 542, 481, 11, 5, 5, 0, 0, 0, 0, 0, 498, 539, 523, 501, 551, 521, 529, 527, 0, 0, 0, 0, 0, 0, 0, 0, 507, 518, 525, 482, 513, 513, 535, 503, 0, 0, 0, 0, 0, 0, 0, 0, 490, 540, 508, 533, 503, 504, 497, 512, 13, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4035, 4098, 4102, 4107, 4032, 4189, 4096, 4109] avg=4096.0000 max=4189 max_rank=5 min=4032 min_rank=4 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16426 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[539, 529, 489, 509, 551, 483, 473, 514, 0, 0, 0, 0, 0, 0, 0, 0, 504, 501, 537, 539, 500, 520, 526, 528, 0, 0, 0, 0, 0, 0, 0, 0, 511, 518, 514, 512, 479, 513, 503, 484, 4, 2, 0, 0, 0, 0, 0, 0, 544, 513, 518, 467, 541, 488, 512, 486, 0, 0, 0, 0, 0, 0, 0, 0, 542, 508, 497, 492, 522, 510, 486, 489, 11, 5, 4, 0, 0, 0, 0, 0, 505, 520, 509, 531, 513, 523, 477, 548, 0, 0, 0, 0, 0, 0, 0, 0, 478, 496, 557, 492, 506, 520, 470, 492, 0, 0, 0, 0, 0, 0, 0, 0, 559, 505, 477, 541, 537, 510, 515, 549, 13, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4155, 4040, 4069, 4066, 4126, 4011, 4214] avg=4096.0000 max=4214 max_rank=7 min=4011 min_rank=6 imbalance_ratio=1.028809 max_load_reduction=28554 max_load_reduction_pct=0.871399 moved_to_echo=16417 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[521, 491, 537, 516, 513, 530, 514, 522, 527, 494, 546, 521, 521, 517, 520, 514, 505, 515, 485, 539, 489, 477, 502, 508, 507, 470, 563, 526, 497, 543, 529, 463, 536, 492, 541, 492, 539, 489, 490, 481, 503, 510, 513, 516, 498, 462, 517, 519, 495, 507, 541, 481, 483, 548, 538, 513, 501, 539, 522, 510, 524, 537, 518, 491] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[525, 519, 507, 487, 498, 480, 507, 504, 505, 531, 485, 511, 506, 488, 531, 491, 537, 486, 460, 524, 513, 534, 514, 514, 501, 531, 494, 539, 524, 530, 484, 522, 549, 509, 523, 544, 501, 530, 529, 502, 553, 509, 512, 491, 507, 492, 528, 496, 483, 516, 517, 550, 549, 490, 509, 485, 499, 504, 525, 503, 515, 517, 546, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[516, 545, 507, 513, 532, 500, 495, 504, 502, 532, 486, 508, 552, 491, 524, 473, 573, 514, 509, 488, 492, 516, 528, 514, 515, 487, 506, 534, 509, 486, 527, 472, 508, 499, 521, 530, 532, 485, 545, 526, 534, 500, 517, 539, 511, 471, 550, 471, 531, 547, 503, 521, 511, 528, 478, 527, 499, 478, 483, 511, 535, 523, 493, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[536, 535, 540, 498, 508, 500, 501, 496, 499, 510, 522, 503, 481, 505, 530, 511, 498, 514, 465, 516, 519, 505, 526, 509, 530, 556, 521, 504, 504, 534, 545, 510, 516, 517, 522, 515, 515, 495, 541, 519, 496, 504, 471, 507, 471, 497, 521, 500, 523, 543, 519, 496, 498, 559, 521, 517, 477, 543, 527, 518, 489, 531, 471, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[522, 539, 473, 524, 544, 514, 533, 503, 562, 489, 491, 505, 506, 498, 539, 502, 528, 513, 530, 551, 506, 536, 498, 508, 491, 490, 501, 514, 469, 511, 519, 508, 518, 515, 505, 501, 520, 502, 524, 504, 512, 500, 499, 542, 506, 528, 523, 491, 524, 538, 522, 505, 527, 523, 500, 545, 491, 496, 520, 523, 491, 483, 505, 468] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[503, 499, 489, 474, 491, 504, 492, 494, 500, 498, 537, 521, 557, 504, 491, 473, 516, 489, 466, 533, 506, 514, 517, 513, 525, 521, 545, 519, 497, 489, 531, 548, 497, 507, 537, 502, 493, 524, 526, 531, 519, 486, 507, 525, 488, 531, 513, 495, 501, 519, 561, 519, 551, 514, 541, 522, 538, 507, 510, 547, 512, 514, 456, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[509, 549, 460, 506, 542, 512, 500, 481, 506, 544, 554, 528, 520, 526, 513, 540, 497, 552, 549, 499, 523, 478, 442, 558, 537, 509, 487, 526, 585, 510, 524, 492, 535, 504, 524, 558, 514, 493, 502, 475, 533, 508, 522, 501, 507, 485, 512, 542, 525, 473, 505, 541, 482, 512, 492, 495, 535, 536, 478, 483, 469, 509, 465, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[547, 516, 532, 496, 483, 515, 512, 517, 509, 508, 485, 527, 506, 519, 515, 536, 508, 473, 484, 527, 483, 512, 512, 518, 544, 504, 498, 501, 533, 510, 513, 503, 511, 492, 471, 492, 511, 534, 545, 483, 529, 540, 515, 498, 524, 512, 500, 530, 510, 507, 501, 537, 520, 520, 526, 525, 511, 510, 502, 539, 497, 521, 475, 534] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[521, 491, 537, 516, 513, 530, 514, 522, 9, 2, 0, 0, 0, 0, 0, 0, 527, 494, 546, 521, 521, 517, 511, 514, 0, 0, 0, 0, 0, 0, 0, 0, 505, 515, 485, 539, 489, 477, 502, 508, 11, 5, 0, 0, 0, 0, 0, 0, 507, 470, 563, 526, 497, 543, 511, 463, 0, 0, 0, 0, 0, 0, 0, 0, 536, 492, 541, 492, 539, 489, 477, 481, 0, 0, 0, 0, 0, 0, 0, 0, 503, 510, 513, 516, 498, 462, 517, 519, 13, 7, 0, 0, 0, 0, 0, 0, 495, 507, 541, 481, 483, 504, 538, 513, 0, 0, 0, 0, 0, 0, 0, 0, 501, 539, 522, 510, 524, 537, 518, 491, 37, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4155, 4151, 4036, 4080, 4047, 4058, 4062, 4179] avg=4096.0000 max=4179 max_rank=7 min=4036 min_rank=2 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16346 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[516, 545, 507, 513, 532, 500, 495, 504, 9, 2, 0, 0, 0, 0, 0, 0, 502, 532, 486, 508, 552, 491, 515, 473, 0, 0, 0, 0, 0, 0, 0, 0, 573, 514, 509, 488, 492, 516, 528, 514, 12, 5, 0, 0, 0, 0, 0, 0, 515, 487, 506, 534, 509, 486, 509, 472, 0, 0, 0, 0, 0, 0, 0, 0, 508, 499, 521, 530, 532, 485, 531, 526, 0, 0, 0, 0, 0, 0, 0, 0, 534, 500, 517, 539, 511, 471, 550, 471, 13, 6, 0, 0, 0, 0, 0, 0, 531, 547, 503, 521, 511, 486, 478, 527, 0, 0, 0, 0, 0, 0, 0, 0, 499, 478, 483, 511, 535, 523, 493, 511, 36, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4123, 4059, 4151, 4018, 4132, 4112, 4104, 4069] avg=4096.0000 max=4151 max_rank=2 min=4018 min_rank=3 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16417 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[525, 519, 507, 487, 498, 480, 507, 504, 9, 2, 0, 0, 0, 0, 0, 0, 505, 531, 485, 511, 506, 488, 522, 491, 0, 0, 0, 0, 0, 0, 0, 0, 537, 486, 460, 524, 513, 534, 514, 514, 12, 5, 0, 0, 0, 0, 0, 0, 501, 531, 494, 539, 524, 530, 467, 522, 0, 0, 0, 0, 0, 0, 0, 0, 549, 509, 523, 544, 501, 530, 515, 502, 0, 0, 0, 0, 0, 0, 0, 0, 553, 509, 512, 491, 507, 492, 528, 496, 12, 6, 0, 0, 0, 0, 0, 0, 483, 516, 517, 550, 549, 451, 509, 485, 0, 0, 0, 0, 0, 0, 0, 0, 499, 504, 525, 503, 515, 517, 546, 503, 33, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4038, 4039, 4099, 4108, 4173, 4106, 4060, 4145] avg=4096.0000 max=4173 max_rank=4 min=4038 min_rank=0 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16484 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[536, 535, 540, 498, 508, 500, 501, 496, 9, 2, 0, 0, 0, 0, 0, 0, 499, 510, 522, 503, 481, 505, 521, 511, 0, 0, 0, 0, 0, 0, 0, 0, 498, 514, 465, 516, 519, 505, 526, 509, 12, 5, 0, 0, 0, 0, 0, 0, 530, 556, 521, 504, 504, 534, 527, 510, 0, 0, 0, 0, 0, 0, 0, 0, 516, 517, 522, 515, 515, 495, 527, 519, 0, 0, 0, 0, 0, 0, 0, 0, 496, 504, 471, 507, 471, 497, 521, 500, 13, 7, 0, 0, 0, 0, 0, 0, 523, 543, 519, 496, 498, 514, 521, 517, 0, 0, 0, 0, 0, 0, 0, 0, 477, 543, 527, 518, 489, 531, 471, 498, 38, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4052, 4069, 4186, 4126, 3987, 4131, 4092] avg=4096.0000 max=4186 max_rank=3 min=3987 min_rank=5 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16336 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[503, 499, 489, 474, 491, 504, 492, 494, 9, 2, 0, 0, 0, 0, 0, 0, 500, 498, 537, 521, 557, 504, 482, 473, 0, 0, 0, 0, 0, 0, 0, 0, 516, 489, 466, 533, 506, 514, 517, 513, 12, 5, 0, 0, 0, 0, 0, 0, 525, 521, 545, 519, 497, 489, 513, 548, 0, 0, 0, 0, 0, 0, 0, 0, 497, 507, 537, 502, 493, 524, 512, 531, 0, 0, 0, 0, 0, 0, 0, 0, 519, 486, 507, 525, 488, 531, 513, 495, 13, 6, 0, 0, 0, 0, 0, 0, 501, 519, 561, 519, 551, 473, 541, 522, 0, 0, 0, 0, 0, 0, 0, 0, 538, 507, 510, 547, 512, 514, 456, 519, 35, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3957, 4072, 4071, 4157, 4103, 4083, 4187, 4138] avg=4096.0000 max=4187 max_rank=6 min=3957 min_rank=0 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16511 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[509, 549, 460, 506, 542, 512, 500, 481, 14, 4, 0, 0, 0, 0, 0, 0, 506, 544, 554, 528, 520, 526, 499, 540, 0, 0, 0, 0, 0, 0, 0, 0, 497, 552, 549, 499, 523, 478, 442, 558, 14, 9, 0, 0, 0, 0, 0, 0, 537, 509, 487, 526, 585, 510, 499, 492, 0, 0, 0, 0, 0, 0, 0, 0, 535, 504, 524, 558, 514, 493, 484, 475, 0, 0, 0, 0, 0, 0, 0, 0, 533, 508, 522, 501, 507, 485, 512, 542, 16, 10, 0, 0, 0, 0, 0, 0, 525, 473, 505, 541, 482, 463, 492, 495, 0, 0, 0, 0, 0, 0, 0, 0, 535, 536, 478, 483, 469, 509, 465, 495, 39, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4217, 4121, 4145, 4087, 4136, 3976, 4009] avg=4096.0000 max=4217 max_rank=1 min=3976 min_rank=6 imbalance_ratio=1.029541 max_load_reduction=28551 max_load_reduction_pct=0.871307 moved_to_echo=16208 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[522, 539, 473, 524, 544, 514, 533, 503, 10, 2, 0, 0, 0, 0, 0, 0, 562, 489, 491, 505, 506, 498, 529, 502, 0, 0, 0, 0, 0, 0, 0, 0, 528, 513, 530, 551, 506, 536, 498, 508, 12, 5, 0, 0, 0, 0, 0, 0, 491, 490, 501, 514, 469, 511, 501, 508, 0, 0, 0, 0, 0, 0, 0, 0, 518, 515, 505, 501, 520, 502, 510, 504, 0, 0, 0, 0, 0, 0, 0, 0, 512, 500, 499, 542, 506, 528, 523, 491, 13, 6, 0, 0, 0, 0, 0, 0, 524, 538, 522, 505, 527, 482, 500, 545, 0, 0, 0, 0, 0, 0, 0, 0, 491, 496, 520, 523, 491, 483, 505, 468, 35, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4164, 4082, 4187, 3985, 4075, 4120, 4143, 4012] avg=4096.0000 max=4187 max_rank=2 min=3985 min_rank=3 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16350 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[547, 516, 532, 496, 483, 515, 512, 517, 9, 2, 0, 0, 0, 0, 0, 0, 509, 508, 485, 527, 506, 519, 506, 536, 0, 0, 0, 0, 0, 0, 0, 0, 508, 473, 484, 527, 483, 512, 512, 518, 12, 5, 0, 0, 0, 0, 0, 0, 544, 504, 498, 501, 533, 510, 496, 503, 0, 0, 0, 0, 0, 0, 0, 0, 511, 492, 471, 492, 511, 534, 531, 483, 0, 0, 0, 0, 0, 0, 0, 0, 529, 540, 515, 498, 524, 512, 500, 530, 12, 6, 0, 0, 0, 0, 0, 0, 510, 507, 501, 537, 520, 479, 526, 525, 0, 0, 0, 0, 0, 0, 0, 0, 511, 510, 502, 539, 497, 521, 475, 534, 35, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4129, 4096, 4034, 4089, 4025, 4166, 4105, 4124] avg=4096.0000 max=4166 max_rank=5 min=4025 min_rank=4 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16420 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 3968, 4224, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 3968, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 3968, 4096, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[501, 510, 515, 538, 530, 516, 498, 533, 511, 511, 519, 519, 519, 496, 525, 532, 511, 523, 509, 537, 527, 504, 514, 493, 547, 521, 492, 522, 523, 513, 539, 513, 502, 508, 510, 506, 499, 508, 511, 548, 533, 521, 498, 540, 487, 493, 489, 491, 508, 495, 495, 494, 464, 491, 459, 521, 481, 541, 492, 510, 538, 527, 539, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[465, 498, 510, 545, 472, 557, 487, 489, 510, 488, 534, 517, 504, 529, 473, 481, 504, 521, 499, 525, 540, 513, 557, 530, 543, 543, 502, 516, 503, 514, 547, 490, 525, 507, 534, 503, 511, 472, 525, 510, 522, 493, 506, 546, 482, 496, 520, 514, 540, 522, 511, 519, 521, 501, 507, 514, 503, 509, 541, 489, 513, 478, 503, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[536, 488, 509, 487, 528, 533, 509, 540, 506, 495, 482, 520, 518, 496, 547, 504, 506, 517, 499, 524, 473, 546, 530, 495, 532, 521, 490, 532, 521, 511, 540, 485, 505, 520, 519, 520, 500, 515, 527, 488, 495, 505, 542, 533, 511, 520, 497, 520, 534, 516, 509, 527, 482, 545, 516, 528, 519, 512, 509, 500, 457, 491, 464, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[493, 486, 528, 507, 517, 538, 502, 528, 499, 508, 526, 508, 492, 483, 529, 496, 510, 513, 515, 513, 517, 511, 537, 510, 503, 477, 542, 541, 482, 530, 480, 500, 553, 515, 492, 535, 543, 516, 513, 519, 545, 494, 526, 519, 473, 539, 466, 490, 484, 540, 505, 525, 498, 507, 493, 530, 521, 493, 555, 494, 515, 506, 499, 544] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[524, 512, 498, 516, 529, 520, 494, 495, 490, 535, 515, 467, 519, 506, 470, 524, 504, 493, 482, 509, 540, 534, 464, 509, 507, 515, 528, 516, 512, 521, 506, 494, 509, 505, 541, 510, 529, 504, 519, 487, 524, 517, 530, 515, 523, 523, 518, 519, 523, 536, 510, 509, 520, 558, 515, 492, 518, 499, 541, 494, 496, 480, 536, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[493, 520, 512, 519, 522, 482, 485, 543, 505, 548, 482, 526, 521, 538, 489, 504, 510, 522, 523, 485, 480, 491, 507, 515, 522, 507, 521, 551, 505, 512, 536, 484, 524, 502, 535, 522, 501, 506, 517, 500, 501, 542, 482, 522, 508, 531, 511, 531, 477, 489, 499, 509, 499, 509, 526, 478, 521, 538, 498, 497, 565, 527, 528, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[506, 501, 492, 539, 508, 488, 558, 508, 503, 538, 529, 489, 510, 453, 492, 509, 494, 512, 498, 528, 500, 494, 505, 465, 498, 526, 503, 529, 505, 553, 523, 551, 540, 521, 480, 511, 503, 495, 535, 513, 516, 542, 469, 531, 515, 536, 504, 543, 499, 497, 524, 514, 522, 495, 516, 543, 522, 495, 522, 489, 519, 505, 530, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[512, 558, 526, 542, 517, 521, 484, 515, 502, 515, 491, 495, 523, 475, 460, 503, 514, 499, 529, 514, 524, 534, 516, 548, 529, 525, 522, 493, 510, 524, 495, 527, 530, 527, 492, 494, 483, 551, 503, 505, 512, 536, 544, 512, 498, 572, 518, 493, 545, 478, 496, 508, 500, 517, 508, 495, 471, 503, 491, 492, 510, 522, 524, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[501, 510, 515, 531, 530, 516, 498, 533, 0, 0, 0, 0, 0, 0, 0, 0, 511, 511, 519, 519, 519, 496, 525, 532, 36, 8, 0, 0, 0, 0, 0, 0, 511, 523, 509, 537, 527, 504, 514, 493, 0, 0, 0, 0, 0, 0, 0, 0, 547, 521, 492, 486, 523, 513, 539, 513, 0, 0, 0, 0, 0, 0, 0, 0, 493, 508, 510, 506, 499, 508, 511, 548, 0, 0, 0, 0, 0, 0, 0, 0, 533, 521, 498, 519, 487, 493, 489, 491, 0, 0, 0, 0, 0, 0, 0, 0, 508, 495, 495, 494, 464, 491, 459, 521, 13, 6, 0, 0, 0, 0, 0, 0, 481, 541, 492, 510, 538, 527, 539, 508, 7, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4134, 4176, 4118, 4134, 4083, 4031, 3946, 4146] avg=4096.0000 max=4176 max_rank=1 min=3946 min_rank=6 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16206 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[536, 488, 509, 480, 528, 533, 509, 540, 0, 0, 0, 0, 0, 0, 0, 0, 506, 495, 482, 520, 518, 496, 547, 504, 37, 8, 0, 0, 0, 0, 0, 0, 506, 517, 499, 524, 473, 546, 530, 495, 0, 0, 0, 0, 0, 0, 0, 0, 532, 521, 490, 495, 521, 511, 540, 485, 0, 0, 0, 0, 0, 0, 0, 0, 496, 520, 519, 520, 500, 515, 527, 488, 0, 0, 0, 0, 0, 0, 0, 0, 495, 505, 542, 512, 511, 520, 497, 520, 0, 0, 0, 0, 0, 0, 0, 0, 534, 516, 509, 527, 482, 545, 516, 528, 13, 6, 0, 0, 0, 0, 0, 0, 519, 512, 509, 500, 457, 491, 464, 522, 7, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4123, 4113, 4090, 4095, 4085, 4102, 4176, 3984] avg=4096.0000 max=4176 max_rank=6 min=3984 min_rank=7 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[465, 498, 510, 538, 472, 557, 487, 489, 0, 0, 0, 0, 0, 0, 0, 0, 510, 488, 534, 517, 504, 529, 473, 481, 36, 8, 0, 0, 0, 0, 0, 0, 504, 521, 499, 525, 540, 513, 557, 530, 0, 0, 0, 0, 0, 0, 0, 0, 543, 543, 502, 480, 503, 514, 547, 490, 0, 0, 0, 0, 0, 0, 0, 0, 515, 507, 534, 503, 511, 472, 525, 510, 0, 0, 0, 0, 0, 0, 0, 0, 522, 493, 506, 525, 482, 496, 520, 514, 0, 0, 0, 0, 0, 0, 0, 0, 540, 522, 511, 519, 521, 501, 507, 514, 13, 6, 0, 0, 0, 0, 0, 0, 503, 509, 541, 489, 513, 478, 503, 525, 7, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4016, 4080, 4189, 4122, 4077, 4058, 4154, 4072] avg=4096.0000 max=4189 max_rank=2 min=4016 min_rank=0 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16361 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[524, 512, 498, 501, 529, 520, 494, 495, 0, 0, 0, 0, 0, 0, 0, 0, 490, 535, 515, 467, 519, 506, 470, 524, 41, 12, 0, 0, 0, 0, 0, 0, 504, 493, 482, 509, 540, 534, 464, 509, 3, 0, 0, 0, 0, 0, 0, 0, 507, 515, 528, 475, 512, 521, 506, 494, 0, 0, 0, 0, 0, 0, 0, 0, 492, 505, 541, 510, 529, 504, 519, 487, 0, 0, 0, 0, 0, 0, 0, 0, 524, 517, 530, 486, 523, 523, 518, 519, 0, 0, 0, 0, 0, 0, 0, 0, 523, 536, 510, 509, 520, 558, 515, 492, 17, 11, 0, 0, 0, 0, 0, 0, 518, 499, 541, 494, 496, 480, 536, 520, 12, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4079, 4038, 4058, 4087, 4140, 4191, 4102] avg=4096.0000 max=4191 max_rank=6 min=4038 min_rank=2 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16520 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[493, 486, 528, 500, 517, 538, 502, 528, 0, 0, 0, 0, 0, 0, 0, 0, 499, 508, 526, 508, 492, 483, 529, 496, 38, 8, 0, 0, 0, 0, 0, 0, 510, 513, 515, 513, 517, 511, 537, 510, 0, 0, 0, 0, 0, 0, 0, 0, 503, 477, 542, 503, 482, 530, 480, 500, 0, 0, 0, 0, 0, 0, 0, 0, 543, 515, 492, 535, 543, 516, 513, 519, 0, 0, 0, 0, 0, 0, 0, 0, 545, 494, 526, 498, 473, 539, 466, 490, 0, 0, 0, 0, 0, 0, 0, 0, 484, 540, 505, 525, 498, 507, 493, 530, 13, 6, 0, 0, 0, 0, 0, 0, 521, 493, 555, 494, 515, 506, 499, 544, 7, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4087, 4126, 4017, 4176, 4031, 4101, 4138] avg=4096.0000 max=4176 max_rank=4 min=4017 min_rank=3 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16446 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[493, 520, 512, 512, 522, 482, 485, 543, 0, 0, 0, 0, 0, 0, 0, 0, 505, 548, 482, 526, 521, 538, 489, 504, 38, 8, 0, 0, 0, 0, 0, 0, 510, 522, 523, 485, 480, 491, 507, 515, 0, 0, 0, 0, 0, 0, 0, 0, 522, 507, 521, 513, 505, 512, 536, 484, 0, 0, 0, 0, 0, 0, 0, 0, 514, 502, 535, 522, 501, 506, 517, 500, 0, 0, 0, 0, 0, 0, 0, 0, 501, 542, 482, 501, 508, 531, 511, 531, 0, 0, 0, 0, 0, 0, 0, 0, 477, 489, 499, 509, 499, 509, 526, 478, 13, 6, 0, 0, 0, 0, 0, 0, 521, 538, 498, 497, 565, 527, 528, 513, 7, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4159, 4033, 4100, 4097, 4107, 4005, 4198] avg=4096.0000 max=4198 max_rank=7 min=4005 min_rank=6 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16407 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[506, 501, 492, 532, 508, 488, 558, 508, 0, 0, 0, 0, 0, 0, 0, 0, 503, 538, 529, 489, 510, 453, 492, 509, 37, 8, 0, 0, 0, 0, 0, 0, 494, 512, 498, 528, 500, 494, 505, 465, 0, 0, 0, 0, 0, 0, 0, 0, 498, 526, 503, 492, 505, 553, 523, 551, 0, 0, 0, 0, 0, 0, 0, 0, 530, 521, 480, 511, 503, 495, 535, 513, 0, 0, 0, 0, 0, 0, 0, 0, 516, 542, 469, 510, 515, 536, 504, 543, 0, 0, 0, 0, 0, 0, 0, 0, 499, 497, 524, 514, 522, 495, 516, 543, 13, 6, 0, 0, 0, 0, 0, 0, 522, 495, 522, 489, 519, 505, 530, 515, 7, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4068, 3996, 4151, 4088, 4135, 4129, 4108] avg=4096.0000 max=4151 max_rank=3 min=3996 min_rank=2 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16460 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[512, 558, 526, 535, 517, 521, 484, 515, 0, 0, 0, 0, 0, 0, 0, 0, 502, 515, 491, 495, 523, 475, 460, 503, 34, 8, 0, 0, 0, 0, 0, 0, 514, 499, 529, 514, 524, 534, 516, 548, 0, 0, 0, 0, 0, 0, 0, 0, 529, 525, 522, 459, 510, 524, 495, 527, 0, 0, 0, 0, 0, 0, 0, 0, 520, 527, 492, 494, 483, 551, 503, 505, 0, 0, 0, 0, 0, 0, 0, 0, 512, 536, 544, 491, 498, 572, 518, 493, 0, 0, 0, 0, 0, 0, 0, 0, 545, 478, 496, 508, 500, 517, 508, 495, 13, 6, 0, 0, 0, 0, 0, 0, 471, 503, 491, 492, 510, 522, 524, 496, 7, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4168, 4006, 4178, 4091, 4075, 4164, 4066, 4020] avg=4096.0000 max=4178 max_rank=2 min=4006 min_rank=1 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16325 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 3968, 4224, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 3968, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[528, 542, 516, 522, 537, 485, 517, 559, 479, 490, 501, 509, 508, 503, 490, 502, 519, 506, 530, 536, 509, 511, 502, 496, 515, 493, 518, 489, 537, 502, 523, 494, 559, 493, 527, 503, 526, 521, 508, 516, 500, 503, 523, 508, 484, 528, 534, 484, 499, 507, 498, 532, 492, 501, 487, 536, 522, 508, 531, 537, 507, 512, 498, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[495, 515, 511, 507, 549, 516, 507, 531, 481, 538, 485, 476, 510, 556, 504, 486, 482, 515, 481, 495, 530, 514, 512, 516, 524, 508, 514, 553, 511, 538, 531, 556, 540, 528, 499, 526, 499, 526, 507, 543, 483, 521, 508, 525, 492, 450, 515, 505, 510, 539, 502, 505, 512, 489, 505, 534, 506, 541, 518, 517, 512, 491, 493, 480] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[518, 496, 538, 515, 513, 523, 548, 499, 503, 484, 540, 501, 512, 501, 477, 478, 510, 507, 507, 502, 561, 526, 500, 523, 503, 494, 493, 529, 494, 517, 514, 505, 484, 531, 567, 514, 481, 557, 553, 507, 491, 547, 499, 555, 506, 502, 521, 487, 518, 527, 524, 473, 533, 547, 521, 490, 480, 500, 523, 478, 492, 518, 498, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[507, 481, 505, 498, 511, 511, 508, 546, 543, 521, 520, 515, 494, 528, 486, 506, 515, 516, 532, 490, 493, 489, 509, 471, 512, 504, 513, 519, 534, 477, 513, 535, 540, 483, 508, 527, 536, 532, 578, 477, 522, 538, 538, 488, 475, 507, 520, 494, 507, 523, 490, 528, 522, 577, 516, 474, 501, 525, 495, 484, 533, 485, 513, 530] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[500, 488, 506, 513, 506, 488, 528, 542, 548, 521, 501, 548, 509, 498, 523, 549, 530, 546, 490, 541, 481, 546, 495, 501, 535, 513, 551, 504, 495, 482, 512, 494, 508, 487, 498, 474, 477, 526, 468, 530, 475, 525, 511, 508, 508, 516, 527, 498, 524, 534, 524, 506, 493, 506, 546, 523, 496, 497, 495, 524, 511, 538, 489, 542] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[534, 487, 473, 530, 497, 528, 517, 515, 511, 456, 498, 515, 482, 544, 535, 530, 532, 508, 526, 503, 487, 517, 513, 469, 501, 500, 515, 518, 518, 493, 515, 520, 519, 501, 488, 529, 525, 505, 515, 535, 501, 533, 547, 534, 496, 531, 548, 501, 504, 519, 529, 518, 520, 509, 470, 490, 511, 528, 521, 510, 511, 494, 514, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[547, 515, 518, 508, 461, 515, 521, 509, 489, 522, 498, 519, 484, 501, 480, 505, 509, 499, 527, 506, 491, 518, 504, 521, 520, 515, 510, 520, 529, 511, 521, 520, 527, 508, 530, 445, 538, 499, 506, 514, 486, 530, 529, 522, 473, 512, 558, 502, 513, 533, 496, 503, 496, 518, 534, 526, 531, 517, 562, 532, 507, 491, 529, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[498, 483, 520, 519, 496, 519, 497, 493, 495, 493, 509, 518, 485, 512, 506, 549, 505, 523, 538, 523, 495, 516, 508, 526, 464, 494, 497, 479, 529, 502, 513, 519, 517, 497, 508, 535, 537, 526, 503, 525, 542, 558, 472, 524, 486, 540, 487, 540, 546, 522, 457, 488, 538, 517, 529, 531, 494, 506, 523, 507, 547, 533, 516, 494] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[528, 542, 516, 522, 537, 485, 517, 553, 0, 0, 0, 0, 0, 0, 0, 0, 479, 490, 501, 509, 508, 503, 490, 502, 30, 9, 0, 0, 0, 0, 0, 0, 519, 506, 530, 536, 509, 511, 502, 496, 4, 3, 0, 0, 0, 0, 0, 0, 515, 493, 518, 489, 537, 502, 523, 494, 0, 0, 0, 0, 0, 0, 0, 0, 529, 493, 527, 503, 526, 521, 508, 516, 0, 0, 0, 0, 0, 0, 0, 0, 500, 503, 523, 508, 484, 528, 534, 484, 0, 0, 0, 0, 0, 0, 0, 0, 499, 495, 498, 532, 492, 501, 487, 536, 0, 0, 0, 0, 0, 0, 0, 0, 522, 508, 531, 537, 507, 512, 498, 516, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4200, 4021, 4116, 4071, 4123, 4064, 4040, 4133] avg=4096.0000 max=4200 max_rank=0 min=4021 min_rank=1 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16360 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[495, 515, 511, 507, 549, 516, 507, 525, 0, 0, 0, 0, 0, 0, 0, 0, 481, 538, 485, 476, 510, 556, 504, 486, 29, 10, 0, 0, 0, 0, 0, 0, 482, 515, 481, 495, 530, 514, 512, 516, 4, 4, 0, 0, 0, 0, 0, 0, 524, 508, 514, 553, 511, 538, 531, 556, 0, 0, 0, 0, 0, 0, 0, 0, 511, 528, 499, 526, 499, 526, 507, 543, 0, 0, 0, 0, 0, 0, 0, 0, 483, 521, 508, 525, 492, 450, 515, 505, 0, 0, 0, 0, 0, 0, 0, 0, 510, 525, 502, 505, 512, 489, 505, 534, 0, 0, 0, 0, 0, 0, 0, 0, 506, 541, 518, 517, 512, 491, 493, 480, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4075, 4053, 4235, 4139, 3999, 4082, 4060] avg=4096.0000 max=4235 max_rank=3 min=3999 min_rank=5 imbalance_ratio=1.033936 max_load_reduction=28533 max_load_reduction_pct=0.870758 moved_to_echo=16280 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[500, 488, 506, 513, 506, 488, 528, 536, 0, 0, 0, 0, 0, 0, 0, 0, 548, 521, 501, 548, 509, 498, 523, 549, 27, 10, 0, 0, 0, 0, 0, 0, 530, 546, 490, 541, 481, 546, 495, 501, 4, 4, 0, 0, 0, 0, 0, 0, 535, 513, 551, 504, 495, 482, 512, 494, 0, 0, 0, 0, 0, 0, 0, 0, 481, 487, 498, 474, 477, 526, 468, 530, 0, 0, 0, 0, 0, 0, 0, 0, 475, 525, 511, 508, 508, 516, 527, 498, 0, 0, 0, 0, 0, 0, 0, 0, 524, 520, 524, 506, 493, 506, 546, 523, 0, 0, 0, 0, 0, 0, 0, 0, 496, 497, 495, 524, 511, 538, 489, 542, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4065, 4234, 4138, 4086, 3941, 4068, 4142, 4094] avg=4096.0000 max=4234 max_rank=1 min=3941 min_rank=4 imbalance_ratio=1.033691 max_load_reduction=28534 max_load_reduction_pct=0.870789 moved_to_echo=16245 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[507, 481, 505, 498, 511, 511, 508, 531, 0, 0, 0, 0, 0, 0, 0, 0, 543, 521, 520, 515, 494, 528, 486, 506, 32, 13, 0, 0, 0, 0, 0, 0, 515, 516, 532, 490, 493, 489, 509, 471, 9, 7, 0, 0, 0, 0, 0, 0, 512, 504, 513, 519, 531, 477, 513, 535, 0, 0, 0, 0, 0, 0, 0, 0, 508, 483, 508, 527, 536, 532, 578, 477, 0, 0, 0, 0, 0, 0, 0, 0, 522, 533, 538, 488, 475, 507, 520, 494, 0, 0, 0, 0, 0, 0, 0, 0, 507, 503, 490, 528, 522, 577, 516, 474, 0, 0, 0, 0, 0, 0, 0, 0, 501, 525, 495, 484, 533, 485, 513, 530, 6, 5, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4052, 4158, 4031, 4104, 4149, 4077, 4117, 4080] avg=4096.0000 max=4158 max_rank=1 min=4031 min_rank=2 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16423 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[518, 496, 538, 515, 513, 523, 548, 493, 0, 0, 0, 0, 0, 0, 0, 0, 503, 484, 540, 501, 512, 501, 477, 478, 26, 10, 0, 0, 0, 0, 0, 0, 510, 507, 507, 502, 561, 526, 500, 523, 4, 4, 0, 0, 0, 0, 0, 0, 503, 494, 493, 529, 494, 517, 514, 505, 0, 0, 0, 0, 0, 0, 0, 0, 458, 531, 567, 514, 481, 557, 553, 507, 0, 0, 0, 0, 0, 0, 0, 0, 491, 547, 499, 555, 506, 502, 521, 487, 0, 0, 0, 0, 0, 0, 0, 0, 518, 513, 524, 473, 533, 547, 521, 490, 0, 0, 0, 0, 0, 0, 0, 0, 480, 500, 523, 478, 492, 518, 498, 513, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4144, 4032, 4144, 4049, 4168, 4108, 4119, 4004] avg=4096.0000 max=4168 max_rank=4 min=4004 min_rank=7 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[534, 487, 473, 530, 497, 528, 517, 509, 0, 0, 0, 0, 0, 0, 0, 0, 511, 456, 498, 515, 482, 544, 535, 530, 28, 9, 0, 0, 0, 0, 0, 0, 532, 508, 526, 503, 487, 517, 513, 469, 4, 3, 0, 0, 0, 0, 0, 0, 501, 500, 515, 518, 518, 493, 515, 520, 0, 0, 0, 0, 0, 0, 0, 0, 491, 501, 488, 529, 525, 505, 515, 535, 0, 0, 0, 0, 0, 0, 0, 0, 501, 533, 547, 534, 496, 531, 548, 501, 0, 0, 0, 0, 0, 0, 0, 0, 504, 507, 529, 518, 520, 509, 470, 490, 0, 0, 0, 0, 0, 0, 0, 0, 511, 528, 521, 510, 511, 494, 514, 525, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4108, 4062, 4080, 4089, 4191, 4047, 4116] avg=4096.0000 max=4191 max_rank=5 min=4047 min_rank=6 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16443 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[498, 483, 520, 519, 496, 519, 497, 487, 0, 0, 0, 0, 0, 0, 0, 0, 495, 493, 509, 518, 485, 512, 506, 549, 28, 9, 0, 0, 0, 0, 0, 0, 505, 523, 538, 523, 495, 516, 508, 526, 4, 3, 0, 0, 0, 0, 0, 0, 464, 494, 497, 479, 529, 502, 513, 519, 0, 0, 0, 0, 0, 0, 0, 0, 489, 497, 508, 535, 537, 526, 503, 525, 0, 0, 0, 0, 0, 0, 0, 0, 542, 558, 472, 524, 486, 540, 487, 540, 0, 0, 0, 0, 0, 0, 0, 0, 546, 510, 457, 488, 538, 517, 529, 531, 0, 0, 0, 0, 0, 0, 0, 0, 494, 506, 523, 507, 547, 533, 516, 494, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4019, 4104, 4141, 3997, 4120, 4149, 4116, 4122] avg=4096.0000 max=4149 max_rank=5 min=3997 min_rank=3 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16507 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[547, 515, 518, 508, 461, 515, 521, 503, 0, 0, 0, 0, 0, 0, 0, 0, 489, 522, 498, 519, 484, 501, 480, 505, 28, 10, 0, 0, 0, 0, 0, 0, 509, 499, 527, 506, 491, 518, 504, 521, 4, 4, 0, 0, 0, 0, 0, 0, 520, 515, 510, 520, 529, 511, 521, 520, 0, 0, 0, 0, 0, 0, 0, 0, 499, 508, 530, 445, 538, 499, 506, 514, 0, 0, 0, 0, 0, 0, 0, 0, 486, 530, 529, 522, 473, 512, 558, 502, 0, 0, 0, 0, 0, 0, 0, 0, 513, 519, 496, 503, 496, 518, 534, 526, 0, 0, 0, 0, 0, 0, 0, 0, 531, 517, 562, 532, 507, 491, 529, 488, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4036, 4083, 4146, 4039, 4112, 4105, 4159] avg=4096.0000 max=4159 max_rank=7 min=4036 min_rank=1 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16415 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3968, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4352, 4224, 4224, 3968, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[553, 508, 513, 490, 514, 540, 508, 521, 479, 481, 517, 517, 486, 539, 489, 525, 487, 520, 491, 504, 568, 502, 509, 477, 512, 528, 460, 500, 501, 484, 491, 566, 517, 508, 508, 497, 509, 507, 491, 507, 533, 566, 495, 497, 480, 477, 479, 546, 568, 544, 539, 553, 505, 518, 528, 511, 513, 518, 549, 502, 501, 503, 518, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[524, 486, 554, 543, 515, 513, 530, 531, 516, 547, 498, 482, 540, 508, 483, 532, 464, 518, 502, 532, 547, 476, 503, 507, 494, 473, 508, 495, 494, 503, 509, 497, 505, 494, 529, 538, 511, 542, 556, 506, 518, 467, 485, 550, 509, 522, 527, 526, 473, 500, 526, 496, 517, 527, 493, 515, 524, 541, 505, 499, 507, 507, 513, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[487, 496, 506, 506, 495, 494, 528, 524, 484, 501, 529, 526, 540, 534, 518, 495, 508, 508, 514, 516, 505, 502, 546, 509, 517, 506, 538, 527, 548, 503, 511, 520, 495, 520, 511, 511, 495, 519, 508, 524, 531, 494, 482, 495, 529, 547, 502, 518, 532, 509, 500, 512, 484, 515, 475, 522, 520, 501, 520, 500, 512, 526, 519, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[499, 530, 548, 527, 537, 512, 477, 488, 518, 536, 529, 555, 487, 530, 479, 484, 491, 536, 524, 520, 509, 522, 509, 495, 534, 556, 538, 536, 478, 493, 494, 446, 526, 490, 546, 510, 527, 543, 572, 491, 514, 523, 522, 475, 495, 487, 501, 528, 488, 483, 516, 523, 499, 530, 483, 488, 505, 491, 510, 513, 550, 528, 514, 480] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[534, 521, 551, 522, 513, 534, 515, 510, 484, 494, 516, 535, 495, 509, 527, 527, 518, 522, 525, 485, 515, 523, 489, 534, 520, 535, 512, 488, 514, 522, 543, 531, 527, 489, 533, 497, 509, 508, 509, 473, 542, 485, 509, 493, 483, 528, 506, 525, 492, 494, 531, 489, 514, 525, 487, 499, 502, 530, 476, 488, 493, 504, 535, 530] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[512, 496, 497, 536, 504, 545, 525, 511, 508, 527, 479, 513, 561, 525, 522, 490, 525, 455, 520, 493, 505, 506, 509, 482, 504, 529, 531, 519, 515, 512, 547, 514, 517, 527, 506, 516, 534, 550, 511, 494, 537, 520, 470, 466, 504, 513, 534, 513, 525, 535, 495, 524, 517, 531, 483, 503, 485, 495, 548, 494, 531, 491, 479, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[497, 484, 506, 515, 501, 489, 549, 504, 527, 534, 486, 548, 521, 503, 516, 494, 520, 479, 503, 515, 482, 534, 498, 531, 516, 484, 489, 540, 519, 527, 493, 521, 564, 512, 495, 519, 550, 548, 514, 492, 495, 521, 546, 500, 495, 484, 514, 527, 497, 507, 516, 534, 505, 516, 522, 523, 501, 529, 515, 501, 472, 527, 485, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[509, 502, 510, 543, 503, 530, 496, 484, 549, 502, 513, 461, 524, 485, 506, 500, 495, 555, 537, 515, 514, 498, 517, 531, 501, 489, 523, 531, 499, 495, 577, 514, 501, 507, 552, 529, 519, 516, 513, 466, 542, 522, 482, 502, 520, 535, 526, 489, 490, 493, 518, 532, 527, 508, 519, 510, 481, 500, 503, 516, 438, 522, 547, 535] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[524, 486, 531, 543, 515, 513, 530, 531, 0, 0, 0, 0, 0, 0, 0, 0, 516, 547, 498, 482, 540, 508, 483, 532, 0, 0, 0, 0, 0, 0, 0, 0, 464, 518, 502, 532, 547, 476, 503, 507, 14, 0, 0, 0, 0, 0, 0, 0, 494, 473, 508, 495, 494, 503, 505, 497, 0, 0, 0, 0, 0, 0, 0, 0, 505, 494, 529, 538, 511, 512, 556, 506, 0, 0, 0, 0, 0, 0, 0, 0, 518, 467, 485, 550, 509, 522, 527, 526, 9, 6, 0, 0, 0, 0, 0, 0, 473, 500, 526, 496, 517, 527, 493, 515, 4, 0, 0, 0, 0, 0, 0, 0, 524, 541, 505, 499, 507, 507, 513, 516, 24, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4173, 4106, 4063, 3969, 4151, 4119, 4051, 4136] avg=4096.0000 max=4173 max_rank=0 min=3969 min_rank=3 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16457 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[553, 508, 492, 490, 514, 540, 508, 521, 0, 0, 0, 0, 0, 0, 0, 0, 479, 481, 517, 517, 486, 539, 489, 525, 0, 0, 0, 0, 0, 0, 0, 0, 487, 520, 491, 504, 568, 502, 509, 477, 13, 0, 0, 0, 0, 0, 0, 0, 512, 528, 460, 500, 501, 484, 487, 566, 0, 0, 0, 0, 0, 0, 0, 0, 517, 508, 508, 497, 509, 479, 491, 507, 0, 0, 0, 0, 0, 0, 0, 0, 533, 566, 495, 497, 480, 477, 479, 546, 8, 6, 0, 0, 0, 0, 0, 0, 568, 544, 539, 553, 505, 518, 528, 511, 4, 0, 0, 0, 0, 0, 0, 0, 513, 518, 549, 502, 501, 503, 518, 501, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4033, 4071, 4038, 4016, 4087, 4270, 4127] avg=4096.0000 max=4270 max_rank=6 min=4016 min_rank=4 imbalance_ratio=1.042480 max_load_reduction=28498 max_load_reduction_pct=0.869690 moved_to_echo=16500 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[487, 496, 485, 506, 495, 494, 528, 524, 0, 0, 0, 0, 0, 0, 0, 0, 484, 501, 529, 526, 540, 534, 518, 495, 0, 0, 0, 0, 0, 0, 0, 0, 508, 508, 514, 516, 505, 502, 546, 509, 13, 0, 0, 0, 0, 0, 0, 0, 517, 506, 538, 527, 548, 503, 507, 520, 0, 0, 0, 0, 0, 0, 0, 0, 495, 520, 511, 511, 495, 490, 508, 524, 0, 0, 0, 0, 0, 0, 0, 0, 531, 494, 482, 495, 529, 547, 502, 518, 8, 6, 0, 0, 0, 0, 0, 0, 532, 509, 500, 512, 484, 515, 475, 522, 4, 0, 0, 0, 0, 0, 0, 0, 520, 501, 520, 500, 512, 526, 519, 499, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4015, 4127, 4121, 4166, 4054, 4112, 4053, 4120] avg=4096.0000 max=4166 max_rank=3 min=4015 min_rank=0 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16339 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[499, 530, 526, 527, 537, 512, 477, 488, 0, 0, 0, 0, 0, 0, 0, 0, 518, 536, 529, 555, 487, 530, 479, 484, 0, 0, 0, 0, 0, 0, 0, 0, 491, 536, 524, 520, 509, 522, 509, 495, 14, 0, 0, 0, 0, 0, 0, 0, 534, 556, 538, 536, 478, 493, 490, 446, 0, 0, 0, 0, 0, 0, 0, 0, 526, 490, 546, 510, 527, 513, 572, 491, 0, 0, 0, 0, 0, 0, 0, 0, 514, 523, 522, 475, 495, 487, 501, 528, 8, 6, 0, 0, 0, 0, 0, 0, 488, 483, 516, 523, 499, 530, 483, 488, 4, 0, 0, 0, 0, 0, 0, 0, 505, 491, 510, 513, 550, 528, 514, 480, 24, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4118, 4120, 4071, 4175, 4059, 4014, 4115] avg=4096.0000 max=4175 max_rank=4 min=4014 min_rank=6 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[512, 496, 471, 536, 504, 545, 525, 511, 0, 0, 0, 0, 0, 0, 0, 0, 508, 527, 479, 513, 561, 525, 522, 490, 3, 0, 0, 0, 0, 0, 0, 0, 525, 455, 520, 493, 505, 506, 509, 482, 15, 6, 0, 0, 0, 0, 0, 0, 504, 529, 531, 519, 515, 512, 531, 514, 0, 0, 0, 0, 0, 0, 0, 0, 517, 527, 506, 516, 534, 512, 511, 494, 0, 0, 0, 0, 0, 0, 0, 0, 537, 520, 470, 466, 504, 513, 534, 513, 11, 10, 0, 0, 0, 0, 0, 0, 525, 535, 495, 524, 517, 531, 483, 503, 7, 0, 0, 0, 0, 0, 0, 0, 485, 495, 548, 494, 531, 491, 479, 503, 28, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4128, 4016, 4155, 4117, 4078, 4120, 4054] avg=4096.0000 max=4155 max_rank=3 min=4016 min_rank=2 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16369 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[534, 521, 529, 522, 513, 534, 515, 510, 0, 0, 0, 0, 0, 0, 0, 0, 484, 494, 516, 535, 495, 509, 527, 527, 0, 0, 0, 0, 0, 0, 0, 0, 518, 522, 525, 485, 515, 523, 489, 534, 14, 1, 0, 0, 0, 0, 0, 0, 520, 535, 512, 488, 514, 522, 538, 531, 0, 0, 0, 0, 0, 0, 0, 0, 527, 489, 533, 497, 509, 480, 509, 473, 0, 0, 0, 0, 0, 0, 0, 0, 542, 485, 509, 493, 483, 528, 506, 525, 8, 6, 0, 0, 0, 0, 0, 0, 492, 494, 531, 489, 514, 525, 487, 499, 4, 0, 0, 0, 0, 0, 0, 0, 502, 530, 476, 488, 493, 504, 535, 530, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4178, 4087, 4126, 4160, 4017, 4085, 4035, 4080] avg=4096.0000 max=4178 max_rank=0 min=4017 min_rank=4 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16217 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[497, 484, 485, 515, 501, 489, 549, 504, 0, 0, 0, 0, 0, 0, 0, 0, 527, 534, 486, 548, 521, 503, 516, 494, 0, 0, 0, 0, 0, 0, 0, 0, 520, 479, 503, 515, 482, 534, 498, 531, 13, 0, 0, 0, 0, 0, 0, 0, 516, 484, 489, 540, 519, 527, 489, 521, 0, 0, 0, 0, 0, 0, 0, 0, 564, 512, 495, 519, 550, 518, 514, 492, 0, 0, 0, 0, 0, 0, 0, 0, 495, 521, 546, 500, 495, 484, 514, 527, 8, 6, 0, 0, 0, 0, 0, 0, 497, 507, 516, 534, 505, 516, 522, 523, 4, 0, 0, 0, 0, 0, 0, 0, 501, 529, 515, 501, 472, 527, 485, 517, 24, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4024, 4129, 4075, 4085, 4164, 4096, 4124, 4071] avg=4096.0000 max=4164 max_rank=4 min=4024 min_rank=0 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16455 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[509, 502, 489, 543, 503, 530, 496, 484, 0, 0, 0, 0, 0, 0, 0, 0, 549, 502, 513, 461, 524, 485, 506, 500, 0, 0, 0, 0, 0, 0, 0, 0, 495, 555, 537, 515, 514, 498, 517, 531, 13, 1, 0, 0, 0, 0, 0, 0, 501, 489, 523, 531, 499, 495, 572, 514, 0, 0, 0, 0, 0, 0, 0, 0, 501, 507, 552, 529, 519, 487, 513, 466, 0, 0, 0, 0, 0, 0, 0, 0, 542, 522, 482, 502, 520, 535, 526, 489, 8, 6, 0, 0, 0, 0, 0, 0, 490, 493, 518, 532, 527, 508, 519, 510, 4, 0, 0, 0, 0, 0, 0, 0, 481, 500, 503, 516, 438, 522, 547, 535, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4056, 4040, 4176, 4124, 4074, 4132, 4101, 4065] avg=4096.0000 max=4176 max_rank=2 min=4040 min_rank=1 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4224, 3968, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[507, 517, 492, 530, 507, 497, 511, 555, 494, 506, 535, 466, 521, 487, 540, 518, 535, 492, 513, 533, 514, 462, 501, 563, 524, 526, 487, 478, 484, 515, 495, 495, 544, 541, 480, 491, 490, 562, 500, 517, 507, 502, 497, 491, 542, 532, 521, 546, 513, 524, 525, 511, 490, 516, 500, 533, 488, 496, 511, 497, 545, 480, 558, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[509, 536, 562, 535, 518, 516, 501, 489, 500, 500, 489, 517, 529, 517, 493, 523, 510, 498, 538, 513, 514, 513, 521, 500, 513, 498, 526, 520, 491, 499, 521, 508, 526, 525, 519, 530, 519, 541, 496, 480, 472, 503, 519, 489, 500, 496, 461, 493, 502, 504, 526, 501, 537, 516, 497, 534, 542, 497, 513, 534, 507, 511, 529, 532] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[522, 509, 498, 517, 509, 514, 505, 503, 509, 536, 492, 522, 495, 540, 499, 529, 518, 493, 517, 523, 503, 514, 542, 538, 519, 550, 500, 502, 536, 542, 513, 501, 506, 484, 504, 523, 492, 550, 476, 464, 510, 521, 542, 467, 521, 489, 486, 505, 467, 509, 511, 508, 534, 561, 539, 500, 535, 515, 499, 519, 527, 467, 532, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[494, 491, 544, 522, 494, 527, 502, 535, 504, 477, 502, 496, 513, 475, 559, 515, 536, 538, 507, 526, 500, 522, 507, 532, 465, 514, 520, 533, 525, 510, 485, 543, 502, 543, 552, 521, 507, 504, 522, 509, 462, 505, 501, 552, 514, 483, 483, 505, 557, 507, 500, 503, 524, 506, 503, 495, 499, 489, 527, 505, 519, 542, 514, 500] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[487, 521, 510, 479, 504, 520, 517, 563, 514, 520, 552, 545, 507, 506, 507, 504, 540, 516, 518, 523, 511, 480, 520, 494, 501, 503, 533, 518, 502, 493, 512, 490, 531, 478, 486, 504, 514, 495, 499, 527, 548, 537, 518, 492, 489, 490, 517, 497, 514, 500, 512, 469, 525, 565, 524, 479, 531, 526, 533, 514, 511, 505, 530, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[494, 527, 533, 496, 532, 523, 509, 530, 525, 539, 545, 519, 465, 528, 515, 507, 517, 514, 517, 485, 538, 499, 490, 496, 507, 552, 495, 464, 493, 530, 494, 530, 506, 484, 513, 537, 509, 541, 486, 518, 486, 521, 522, 542, 515, 498, 485, 523, 539, 498, 518, 506, 494, 496, 545, 520, 499, 508, 479, 487, 544, 514, 504, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[472, 511, 500, 517, 521, 491, 492, 529, 507, 529, 521, 539, 505, 503, 488, 481, 510, 542, 508, 542, 541, 525, 539, 514, 534, 521, 488, 471, 509, 505, 517, 459, 501, 508, 542, 498, 486, 503, 477, 520, 499, 498, 522, 535, 511, 525, 550, 510, 527, 519, 514, 521, 485, 534, 500, 501, 540, 526, 527, 528, 509, 482, 506, 533] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[496, 508, 502, 501, 510, 509, 501, 555, 477, 502, 540, 564, 490, 503, 506, 485, 534, 510, 533, 501, 502, 491, 530, 493, 495, 487, 512, 493, 473, 522, 531, 483, 503, 526, 523, 526, 529, 551, 536, 498, 536, 466, 520, 537, 519, 488, 533, 507, 532, 515, 506, 508, 525, 522, 517, 516, 508, 526, 482, 554, 491, 489, 534, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[507, 517, 492, 530, 507, 497, 511, 548, 0, 0, 0, 0, 0, 0, 0, 0, 494, 506, 535, 466, 521, 487, 540, 518, 3, 0, 0, 0, 0, 0, 0, 0, 505, 492, 513, 533, 514, 462, 501, 563, 0, 0, 0, 0, 0, 0, 0, 0, 524, 526, 487, 478, 484, 515, 495, 495, 30, 8, 0, 0, 0, 0, 0, 0, 544, 541, 480, 491, 490, 562, 500, 517, 0, 0, 0, 0, 0, 0, 0, 0, 507, 502, 497, 491, 542, 532, 521, 546, 16, 7, 4, 0, 0, 0, 0, 0, 513, 524, 525, 511, 490, 501, 500, 533, 0, 0, 0, 0, 0, 0, 0, 0, 488, 496, 511, 497, 545, 480, 542, 518, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4070, 4083, 4042, 4125, 4165, 4097, 4077] avg=4096.0000 max=4165 max_rank=5 min=4042 min_rank=3 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16464 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[509, 536, 562, 535, 518, 516, 501, 483, 0, 0, 0, 0, 0, 0, 0, 0, 500, 500, 489, 517, 529, 517, 493, 523, 2, 0, 0, 0, 0, 0, 0, 0, 481, 498, 538, 513, 514, 513, 521, 500, 0, 0, 0, 0, 0, 0, 0, 0, 513, 498, 526, 520, 491, 499, 521, 508, 29, 8, 0, 0, 0, 0, 0, 0, 526, 525, 519, 530, 519, 541, 496, 480, 0, 0, 0, 0, 0, 0, 0, 0, 472, 503, 519, 489, 500, 496, 461, 493, 15, 7, 4, 0, 0, 0, 0, 0, 502, 504, 526, 501, 537, 501, 497, 534, 0, 0, 0, 0, 0, 0, 0, 0, 542, 497, 513, 534, 507, 511, 514, 532, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4160, 4070, 4078, 4113, 4136, 3959, 4102, 4150] avg=4096.0000 max=4160 max_rank=0 min=3959 min_rank=5 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[487, 521, 510, 479, 504, 520, 517, 556, 0, 0, 0, 0, 0, 0, 0, 0, 514, 520, 552, 545, 507, 506, 507, 504, 3, 0, 0, 0, 0, 0, 0, 0, 510, 516, 518, 523, 511, 480, 520, 494, 0, 0, 0, 0, 0, 0, 0, 0, 501, 503, 533, 518, 502, 493, 512, 490, 30, 9, 0, 0, 0, 0, 0, 0, 531, 478, 486, 504, 514, 495, 499, 527, 0, 0, 0, 0, 0, 0, 0, 0, 548, 537, 518, 492, 489, 490, 517, 497, 15, 7, 4, 0, 0, 0, 0, 0, 514, 500, 512, 469, 525, 549, 524, 479, 0, 0, 0, 0, 0, 0, 0, 0, 531, 526, 533, 514, 511, 505, 515, 498, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4158, 4072, 4091, 4034, 4114, 4072, 4133] avg=4096.0000 max=4158 max_rank=1 min=4034 min_rank=4 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[522, 509, 498, 517, 509, 514, 505, 497, 0, 0, 0, 0, 0, 0, 0, 0, 509, 536, 492, 522, 495, 540, 499, 529, 2, 0, 0, 0, 0, 0, 0, 0, 489, 493, 517, 523, 503, 514, 542, 538, 0, 0, 0, 0, 0, 0, 0, 0, 519, 550, 500, 502, 536, 542, 513, 501, 29, 9, 0, 0, 0, 0, 0, 0, 506, 484, 504, 523, 492, 550, 476, 464, 0, 0, 0, 0, 0, 0, 0, 0, 510, 521, 542, 467, 521, 489, 486, 505, 15, 7, 4, 0, 0, 0, 0, 0, 467, 509, 511, 508, 534, 545, 539, 500, 0, 0, 0, 0, 0, 0, 0, 0, 535, 515, 499, 519, 527, 467, 517, 495, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4071, 4124, 4119, 4201, 3999, 4067, 4113, 4074] avg=4096.0000 max=4201 max_rank=3 min=3999 min_rank=4 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16253 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[494, 527, 533, 496, 532, 523, 509, 523, 0, 0, 0, 0, 0, 0, 0, 0, 525, 539, 545, 519, 465, 528, 515, 507, 3, 0, 0, 0, 0, 0, 0, 0, 488, 514, 517, 485, 538, 499, 490, 496, 0, 0, 0, 0, 0, 0, 0, 0, 507, 552, 495, 464, 493, 530, 494, 530, 29, 8, 0, 0, 0, 0, 0, 0, 506, 484, 513, 537, 509, 541, 486, 518, 0, 0, 0, 0, 0, 0, 0, 0, 486, 521, 522, 542, 515, 498, 485, 523, 14, 6, 4, 0, 0, 0, 0, 0, 539, 498, 518, 506, 494, 482, 545, 520, 0, 0, 0, 0, 0, 0, 0, 0, 499, 508, 479, 487, 544, 514, 490, 523, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4137, 4146, 4027, 4102, 4094, 4116, 4102, 4044] avg=4096.0000 max=4146 max_rank=1 min=4027 min_rank=2 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16356 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[494, 491, 544, 522, 494, 527, 502, 522, 0, 0, 0, 0, 0, 0, 0, 0, 504, 477, 502, 496, 513, 475, 559, 515, 6, 7, 0, 0, 0, 0, 0, 0, 501, 538, 507, 526, 500, 522, 507, 532, 0, 0, 0, 0, 0, 0, 0, 0, 465, 514, 520, 533, 525, 510, 485, 543, 35, 13, 0, 0, 0, 0, 0, 0, 502, 543, 552, 521, 507, 497, 522, 509, 0, 0, 0, 0, 0, 0, 0, 0, 462, 505, 501, 552, 514, 483, 483, 505, 17, 11, 7, 0, 0, 0, 0, 0, 557, 507, 500, 503, 524, 482, 503, 495, 0, 0, 0, 0, 0, 0, 0, 0, 499, 489, 527, 505, 519, 542, 497, 500, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4054, 4133, 4143, 4153, 4040, 4071, 4078] avg=4096.0000 max=4153 max_rank=4 min=4040 min_rank=5 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16342 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[472, 511, 500, 517, 521, 491, 492, 522, 0, 0, 0, 0, 0, 0, 0, 0, 507, 529, 521, 539, 505, 503, 488, 481, 3, 0, 0, 0, 0, 0, 0, 0, 481, 542, 508, 542, 541, 525, 539, 514, 0, 0, 0, 0, 0, 0, 0, 0, 534, 521, 488, 471, 509, 505, 517, 459, 29, 9, 0, 0, 0, 0, 0, 0, 501, 508, 542, 498, 486, 503, 477, 520, 0, 0, 0, 0, 0, 0, 0, 0, 499, 498, 522, 535, 511, 525, 550, 510, 14, 7, 4, 0, 0, 0, 0, 0, 527, 519, 514, 521, 485, 518, 500, 501, 0, 0, 0, 0, 0, 0, 0, 0, 540, 526, 527, 528, 509, 482, 492, 533, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4026, 4076, 4192, 4042, 4035, 4175, 4085, 4137] avg=4096.0000 max=4192 max_rank=2 min=4026 min_rank=0 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16432 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[496, 508, 502, 501, 510, 509, 501, 548, 0, 0, 0, 0, 0, 0, 0, 0, 477, 502, 540, 564, 490, 503, 506, 485, 3, 0, 0, 0, 0, 0, 0, 0, 504, 510, 533, 501, 502, 491, 530, 493, 0, 0, 0, 0, 0, 0, 0, 0, 495, 487, 512, 493, 473, 522, 531, 483, 30, 8, 0, 0, 0, 0, 0, 0, 503, 526, 523, 526, 529, 551, 536, 498, 0, 0, 0, 0, 0, 0, 0, 0, 536, 466, 520, 537, 519, 488, 533, 507, 15, 7, 4, 0, 0, 0, 0, 0, 532, 515, 506, 508, 525, 507, 517, 516, 0, 0, 0, 0, 0, 0, 0, 0, 508, 526, 482, 554, 491, 489, 519, 506, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4070, 4064, 4034, 4192, 4132, 4126, 4075] avg=4096.0000 max=4192 max_rank=4 min=4034 min_rank=3 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16525 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4352, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3968, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[544, 505, 473, 516, 487, 520, 544, 525, 530, 517, 517, 480, 493, 480, 503, 483, 532, 536, 489, 528, 513, 492, 491, 541, 510, 547, 507, 549, 484, 515, 517, 500, 522, 502, 496, 492, 523, 525, 561, 501, 489, 520, 530, 508, 523, 516, 513, 502, 503, 503, 513, 507, 497, 477, 516, 502, 511, 480, 509, 550, 521, 517, 539, 532] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[498, 507, 542, 533, 476, 522, 533, 528, 475, 497, 496, 480, 534, 514, 530, 528, 520, 525, 498, 522, 527, 512, 481, 504, 537, 563, 499, 475, 568, 496, 487, 508, 501, 497, 512, 502, 536, 522, 509, 529, 509, 506, 478, 495, 532, 496, 489, 516, 516, 530, 503, 499, 532, 520, 489, 521, 547, 519, 514, 508, 475, 500, 510, 541] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[500, 523, 539, 536, 510, 505, 520, 461, 508, 547, 474, 513, 486, 511, 504, 497, 529, 539, 550, 522, 527, 528, 486, 560, 482, 571, 484, 514, 507, 493, 490, 495, 478, 557, 532, 508, 509, 513, 484, 505, 544, 480, 505, 557, 507, 474, 491, 495, 522, 522, 482, 479, 511, 532, 523, 494, 530, 501, 532, 534, 530, 492, 523, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[527, 533, 508, 521, 502, 515, 534, 463, 549, 543, 517, 527, 499, 527, 539, 533, 498, 527, 526, 528, 518, 514, 540, 478, 498, 512, 534, 549, 505, 505, 500, 533, 472, 505, 506, 499, 497, 500, 497, 481, 512, 491, 483, 517, 508, 562, 488, 490, 531, 516, 545, 538, 498, 504, 492, 562, 472, 478, 470, 492, 521, 486, 530, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[520, 476, 537, 538, 542, 467, 515, 483, 510, 527, 506, 500, 497, 498, 511, 514, 500, 525, 524, 512, 523, 503, 496, 530, 517, 529, 508, 493, 502, 499, 524, 515, 541, 507, 500, 522, 498, 504, 520, 501, 525, 475, 533, 500, 538, 475, 481, 496, 559, 498, 537, 544, 525, 525, 494, 522, 502, 526, 505, 492, 494, 491, 566, 531] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[487, 532, 491, 505, 494, 496, 533, 508, 502, 510, 520, 495, 528, 480, 532, 514, 498, 499, 507, 543, 491, 487, 508, 481, 517, 531, 484, 503, 535, 517, 516, 524, 503, 507, 525, 522, 504, 515, 529, 514, 526, 514, 567, 505, 514, 534, 475, 491, 537, 486, 539, 539, 543, 540, 540, 527, 516, 468, 518, 502, 487, 507, 480, 526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[490, 530, 493, 556, 500, 518, 527, 556, 471, 557, 500, 472, 499, 482, 492, 533, 536, 544, 491, 496, 494, 475, 556, 541, 525, 482, 500, 479, 521, 492, 543, 576, 541, 501, 478, 529, 505, 504, 508, 530, 432, 539, 501, 546, 540, 525, 509, 521, 509, 525, 483, 519, 520, 483, 521, 507, 495, 490, 522, 523, 531, 495, 504, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[500, 514, 538, 523, 515, 523, 524, 466, 519, 494, 510, 534, 493, 494, 526, 497, 504, 541, 503, 474, 531, 541, 516, 510, 507, 524, 509, 528, 508, 547, 514, 546, 493, 510, 516, 534, 455, 513, 500, 558, 532, 520, 510, 518, 523, 524, 510, 524, 515, 483, 484, 501, 491, 493, 466, 496, 540, 519, 529, 526, 521, 486, 514, 491] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[498, 507, 542, 533, 476, 522, 524, 528, 0, 0, 0, 0, 0, 0, 0, 0, 475, 497, 496, 480, 534, 514, 530, 528, 23, 3, 0, 0, 0, 0, 0, 0, 520, 502, 498, 522, 527, 512, 481, 504, 0, 0, 0, 0, 0, 0, 0, 0, 537, 541, 499, 475, 568, 496, 487, 508, 0, 0, 0, 0, 0, 0, 0, 0, 501, 497, 512, 502, 536, 522, 509, 529, 9, 4, 0, 0, 0, 0, 0, 0, 509, 506, 478, 495, 532, 496, 489, 516, 15, 0, 0, 0, 0, 0, 0, 0, 509, 530, 503, 499, 532, 520, 489, 521, 0, 0, 0, 0, 0, 0, 0, 0, 547, 519, 514, 508, 475, 500, 510, 541, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4130, 4080, 4066, 4111, 4121, 4036, 4103, 4121] avg=4096.0000 max=4130 max_rank=0 min=4036 min_rank=5 imbalance_ratio=1.008301 max_load_reduction=28638 max_load_reduction_pct=0.873962 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[527, 533, 508, 521, 502, 515, 525, 463, 0, 0, 0, 0, 0, 0, 0, 0, 549, 543, 517, 527, 499, 527, 539, 533, 24, 3, 0, 0, 0, 0, 0, 0, 498, 503, 526, 528, 518, 514, 540, 478, 0, 0, 0, 0, 0, 0, 0, 0, 498, 491, 534, 549, 505, 505, 500, 533, 0, 0, 0, 0, 0, 0, 0, 0, 472, 505, 506, 499, 497, 500, 497, 481, 9, 4, 0, 0, 0, 0, 0, 0, 512, 491, 483, 517, 508, 562, 488, 490, 14, 0, 0, 0, 0, 0, 0, 0, 524, 516, 545, 538, 498, 504, 492, 562, 0, 0, 0, 0, 0, 0, 0, 0, 472, 478, 470, 492, 521, 486, 530, 523, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4261, 4105, 4115, 3970, 4065, 4179, 3979] avg=4096.0000 max=4261 max_rank=1 min=3970 min_rank=4 imbalance_ratio=1.040283 max_load_reduction=28507 max_load_reduction_pct=0.869965 moved_to_echo=16193 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[544, 505, 473, 516, 487, 520, 535, 525, 0, 0, 0, 0, 0, 0, 0, 0, 530, 517, 517, 480, 493, 480, 503, 483, 24, 3, 0, 0, 0, 0, 0, 0, 532, 512, 489, 528, 513, 492, 491, 541, 0, 0, 0, 0, 0, 0, 0, 0, 510, 525, 507, 549, 484, 515, 517, 500, 0, 0, 0, 0, 0, 0, 0, 0, 522, 502, 496, 492, 523, 525, 561, 501, 9, 4, 0, 0, 0, 0, 0, 0, 489, 520, 530, 508, 523, 516, 513, 502, 15, 0, 0, 0, 0, 0, 0, 0, 496, 503, 513, 507, 497, 477, 516, 502, 0, 0, 0, 0, 0, 0, 0, 0, 511, 480, 509, 550, 521, 517, 539, 532, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4030, 4098, 4107, 4135, 4116, 4011, 4166] avg=4096.0000 max=4166 max_rank=7 min=4011 min_rank=6 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16428 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[520, 476, 537, 538, 542, 467, 498, 483, 0, 0, 0, 0, 0, 0, 0, 0, 510, 527, 506, 500, 497, 498, 511, 514, 28, 6, 0, 0, 0, 0, 0, 0, 500, 497, 524, 512, 523, 503, 496, 530, 0, 0, 0, 0, 0, 0, 0, 0, 517, 496, 508, 493, 502, 499, 524, 515, 0, 0, 0, 0, 0, 0, 0, 0, 541, 507, 500, 522, 498, 504, 520, 501, 11, 8, 0, 0, 0, 0, 0, 0, 525, 475, 533, 500, 538, 475, 481, 496, 19, 0, 0, 0, 0, 0, 0, 0, 546, 498, 537, 544, 525, 525, 494, 522, 0, 0, 0, 0, 0, 0, 0, 0, 502, 526, 505, 492, 494, 491, 566, 531, 13, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4097, 4085, 4054, 4112, 4042, 4191, 4126] avg=4096.0000 max=4191 max_rank=6 min=4042 min_rank=5 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16471 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[500, 523, 539, 536, 510, 505, 512, 461, 0, 0, 0, 0, 0, 0, 0, 0, 508, 547, 474, 513, 486, 511, 504, 497, 24, 3, 0, 0, 0, 0, 0, 0, 529, 515, 550, 522, 527, 528, 486, 560, 0, 0, 0, 0, 0, 0, 0, 0, 482, 549, 484, 514, 507, 493, 490, 495, 0, 0, 0, 0, 0, 0, 0, 0, 478, 557, 532, 508, 509, 513, 484, 505, 8, 4, 0, 0, 0, 0, 0, 0, 544, 480, 505, 557, 507, 474, 491, 495, 15, 0, 0, 0, 0, 0, 0, 0, 515, 522, 482, 479, 511, 532, 523, 494, 0, 0, 0, 0, 0, 0, 0, 0, 530, 501, 532, 534, 530, 492, 523, 511, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4067, 4217, 4014, 4098, 4068, 4058, 4160] avg=4096.0000 max=4217 max_rank=2 min=4014 min_rank=3 imbalance_ratio=1.029541 max_load_reduction=28551 max_load_reduction_pct=0.871307 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[490, 530, 493, 556, 500, 518, 518, 556, 0, 0, 0, 0, 0, 0, 0, 0, 471, 557, 500, 472, 499, 482, 492, 533, 24, 3, 0, 0, 0, 0, 0, 0, 536, 520, 491, 496, 494, 475, 556, 541, 0, 0, 0, 0, 0, 0, 0, 0, 525, 463, 500, 479, 521, 492, 543, 576, 0, 0, 0, 0, 0, 0, 0, 0, 541, 501, 478, 529, 505, 504, 508, 530, 9, 3, 0, 0, 0, 0, 0, 0, 432, 539, 501, 546, 540, 525, 509, 521, 13, 0, 0, 0, 0, 0, 0, 0, 502, 525, 483, 519, 520, 483, 521, 507, 0, 0, 0, 0, 0, 0, 0, 0, 495, 490, 522, 523, 531, 495, 504, 505, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4161, 4033, 4109, 4099, 4108, 4126, 4060, 4072] avg=4096.0000 max=4161 max_rank=0 min=4033 min_rank=1 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[487, 532, 491, 505, 494, 496, 524, 508, 0, 0, 0, 0, 0, 0, 0, 0, 502, 510, 520, 495, 528, 480, 532, 514, 22, 3, 0, 0, 0, 0, 0, 0, 498, 477, 507, 543, 491, 487, 508, 481, 0, 0, 0, 0, 0, 0, 0, 0, 517, 510, 484, 503, 535, 517, 516, 524, 0, 0, 0, 0, 0, 0, 0, 0, 503, 507, 525, 522, 504, 515, 529, 514, 9, 4, 0, 0, 0, 0, 0, 0, 526, 514, 567, 505, 514, 534, 475, 491, 14, 0, 0, 0, 0, 0, 0, 0, 530, 486, 539, 539, 543, 540, 540, 527, 0, 0, 0, 0, 0, 0, 0, 0, 516, 468, 518, 502, 487, 507, 480, 526, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4037, 4106, 3992, 4106, 4132, 4140, 4244, 4011] avg=4096.0000 max=4244 max_rank=6 min=3992 min_rank=2 imbalance_ratio=1.036133 max_load_reduction=28524 max_load_reduction_pct=0.870483 moved_to_echo=16527 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[500, 514, 538, 523, 515, 523, 515, 466, 0, 0, 0, 0, 0, 0, 0, 0, 519, 494, 510, 534, 493, 494, 526, 497, 24, 3, 0, 0, 0, 0, 0, 0, 504, 517, 503, 474, 531, 541, 516, 510, 0, 0, 0, 0, 0, 0, 0, 0, 507, 503, 509, 528, 508, 547, 514, 546, 0, 0, 0, 0, 0, 0, 0, 0, 493, 510, 516, 534, 455, 513, 500, 558, 9, 4, 0, 0, 0, 0, 0, 0, 532, 520, 510, 518, 523, 524, 510, 524, 14, 0, 0, 0, 0, 0, 0, 0, 508, 483, 484, 501, 491, 493, 466, 496, 0, 0, 0, 0, 0, 0, 0, 0, 540, 519, 529, 526, 521, 486, 514, 491, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4094, 4096, 4162, 4092, 4175, 3922, 4133] avg=4096.0000 max=4175 max_rank=5 min=3922 min_rank=6 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16322 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4352, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 3968, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[512, 523, 522, 508, 509, 466, 483, 515, 527, 497, 505, 522, 513, 512, 499, 511, 441, 520, 522, 481, 543, 554, 492, 525, 527, 497, 527, 519, 503, 506, 516, 537, 506, 493, 522, 486, 494, 510, 524, 504, 521, 550, 506, 509, 515, 513, 491, 530, 520, 514, 520, 490, 525, 510, 553, 533, 496, 541, 474, 510, 537, 484, 535, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[516, 493, 501, 533, 517, 509, 501, 510, 505, 497, 526, 520, 523, 515, 502, 524, 496, 527, 509, 500, 521, 537, 525, 491, 512, 523, 515, 501, 487, 521, 542, 495, 502, 511, 528, 532, 521, 465, 515, 538, 531, 500, 558, 519, 538, 516, 507, 519, 529, 492, 507, 485, 524, 486, 481, 484, 502, 526, 519, 507, 515, 507, 475, 535] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[524, 516, 523, 513, 483, 490, 493, 504, 547, 530, 492, 519, 503, 482, 532, 466, 500, 553, 514, 530, 501, 562, 498, 511, 505, 527, 490, 468, 500, 551, 521, 537, 502, 530, 516, 504, 529, 500, 488, 483, 501, 533, 508, 499, 463, 506, 503, 560, 486, 525, 522, 517, 520, 478, 513, 557, 524, 519, 529, 519, 502, 489, 536, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[526, 549, 510, 506, 537, 504, 489, 493, 510, 502, 526, 536, 480, 500, 514, 551, 491, 491, 514, 504, 552, 502, 526, 486, 509, 500, 518, 527, 534, 537, 491, 527, 500, 537, 495, 496, 505, 497, 517, 518, 506, 484, 512, 577, 479, 496, 479, 491, 572, 522, 509, 520, 502, 459, 501, 525, 544, 520, 521, 493, 526, 515, 517, 491] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[508, 548, 479, 521, 515, 473, 521, 517, 499, 532, 461, 505, 488, 521, 527, 482, 512, 532, 536, 512, 530, 519, 516, 523, 485, 534, 492, 499, 520, 500, 485, 513, 518, 482, 502, 527, 531, 500, 529, 529, 532, 554, 504, 495, 519, 533, 507, 487, 508, 544, 504, 499, 529, 516, 509, 492, 503, 508, 551, 514, 537, 495, 498, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[555, 511, 478, 509, 522, 523, 493, 440, 518, 458, 536, 488, 519, 508, 530, 520, 495, 537, 499, 531, 528, 483, 527, 528, 521, 510, 503, 513, 504, 528, 536, 531, 477, 485, 494, 509, 558, 510, 522, 464, 561, 552, 517, 519, 521, 522, 512, 515, 515, 491, 523, 464, 531, 515, 518, 505, 498, 518, 543, 519, 507, 485, 511, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[511, 504, 507, 542, 503, 471, 532, 530, 500, 489, 505, 490, 493, 498, 512, 524, 512, 519, 496, 568, 523, 489, 504, 521, 525, 543, 490, 499, 502, 539, 506, 513, 494, 552, 502, 531, 483, 512, 538, 522, 509, 517, 507, 530, 543, 526, 539, 495, 518, 493, 498, 551, 494, 532, 491, 517, 516, 476, 514, 496, 475, 506, 531, 500] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[534, 529, 545, 521, 514, 514, 486, 521, 515, 508, 492, 517, 497, 513, 494, 506, 502, 545, 503, 528, 499, 514, 505, 515, 520, 535, 459, 544, 496, 535, 532, 440, 512, 524, 489, 501, 496, 541, 513, 516, 524, 523, 501, 517, 495, 528, 510, 525, 515, 508, 484, 516, 501, 519, 479, 533, 499, 502, 512, 545, 524, 508, 532, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[516, 493, 501, 533, 517, 509, 501, 510, 14, 0, 0, 0, 0, 0, 0, 0, 505, 497, 526, 520, 523, 515, 502, 524, 27, 0, 0, 0, 0, 0, 0, 0, 496, 503, 509, 500, 521, 537, 525, 491, 0, 0, 0, 0, 0, 0, 0, 0, 512, 523, 515, 501, 487, 515, 542, 495, 0, 0, 0, 0, 0, 0, 0, 0, 502, 511, 528, 532, 521, 465, 515, 538, 10, 6, 0, 0, 0, 0, 0, 0, 531, 467, 558, 519, 538, 516, 507, 519, 0, 0, 0, 0, 0, 0, 0, 0, 529, 492, 507, 485, 524, 486, 481, 484, 6, 1, 0, 0, 0, 0, 0, 0, 502, 526, 518, 507, 515, 507, 475, 535, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4139, 4082, 4090, 4128, 4155, 3995, 4085] avg=4096.0000 max=4155 max_rank=5 min=3995 min_rank=6 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[524, 516, 523, 513, 483, 490, 493, 504, 14, 0, 0, 0, 0, 0, 0, 0, 547, 530, 492, 519, 503, 482, 532, 466, 29, 0, 0, 0, 0, 0, 0, 0, 500, 528, 514, 530, 501, 562, 498, 511, 0, 0, 0, 0, 0, 0, 0, 0, 505, 527, 490, 468, 500, 544, 521, 537, 0, 0, 0, 0, 0, 0, 0, 0, 502, 530, 516, 504, 529, 500, 488, 483, 10, 7, 0, 0, 0, 0, 0, 0, 501, 497, 508, 499, 463, 506, 503, 560, 0, 0, 0, 0, 0, 0, 0, 0, 486, 525, 522, 517, 520, 478, 513, 557, 7, 1, 1, 0, 0, 0, 0, 0, 524, 519, 528, 519, 502, 489, 536, 522, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4100, 4144, 4092, 4069, 4037, 4127, 4139] avg=4096.0000 max=4144 max_rank=2 min=4037 min_rank=5 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[526, 549, 510, 506, 537, 504, 489, 493, 17, 0, 0, 0, 0, 0, 0, 0, 510, 502, 526, 536, 480, 500, 514, 551, 32, 0, 0, 0, 0, 0, 0, 0, 491, 457, 514, 504, 552, 502, 526, 486, 0, 0, 0, 0, 0, 0, 0, 0, 509, 500, 518, 527, 534, 526, 491, 527, 0, 0, 0, 0, 0, 0, 0, 0, 500, 537, 495, 496, 505, 497, 517, 518, 13, 10, 0, 0, 0, 0, 0, 0, 506, 442, 512, 577, 479, 496, 479, 491, 0, 0, 0, 0, 0, 0, 0, 0, 572, 522, 509, 520, 502, 459, 501, 525, 11, 6, 4, 0, 0, 0, 0, 0, 544, 520, 515, 493, 526, 515, 517, 491, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4131, 4151, 4032, 4132, 4088, 3982, 4131, 4121] avg=4096.0000 max=4151 max_rank=1 min=3982 min_rank=5 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16322 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[508, 548, 479, 521, 515, 473, 521, 517, 14, 0, 0, 0, 0, 0, 0, 0, 499, 532, 461, 505, 488, 521, 527, 482, 30, 0, 0, 0, 0, 0, 0, 0, 512, 507, 536, 512, 530, 519, 516, 523, 0, 0, 0, 0, 0, 0, 0, 0, 485, 534, 492, 499, 520, 494, 485, 513, 0, 0, 0, 0, 0, 0, 0, 0, 518, 482, 502, 527, 531, 500, 529, 529, 10, 7, 0, 0, 0, 0, 0, 0, 532, 517, 504, 495, 519, 533, 507, 487, 0, 0, 0, 0, 0, 0, 0, 0, 508, 544, 504, 499, 529, 516, 509, 492, 6, 1, 1, 0, 0, 0, 0, 0, 503, 508, 550, 514, 537, 495, 498, 507, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4045, 4155, 4022, 4135, 4094, 4109, 4112] avg=4096.0000 max=4155 max_rank=2 min=4022 min_rank=3 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16450 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[512, 523, 522, 508, 509, 466, 483, 515, 13, 0, 0, 0, 0, 0, 0, 0, 527, 497, 505, 522, 513, 512, 499, 511, 30, 0, 0, 0, 0, 0, 0, 0, 441, 498, 522, 481, 543, 554, 492, 525, 0, 0, 0, 0, 0, 0, 0, 0, 527, 497, 527, 519, 503, 500, 516, 537, 0, 0, 0, 0, 0, 0, 0, 0, 506, 493, 522, 486, 494, 510, 524, 504, 9, 7, 0, 0, 0, 0, 0, 0, 521, 513, 506, 509, 515, 513, 491, 530, 0, 0, 0, 0, 0, 0, 0, 0, 520, 514, 520, 490, 525, 510, 553, 533, 6, 1, 0, 0, 0, 0, 0, 0, 496, 541, 473, 510, 537, 484, 535, 518, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4051, 4116, 4056, 4126, 4055, 4098, 4172, 4094] avg=4096.0000 max=4172 max_rank=6 min=4051 min_rank=0 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16419 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[555, 511, 478, 509, 522, 523, 493, 440, 14, 0, 0, 0, 0, 0, 0, 0, 518, 458, 536, 488, 519, 508, 530, 520, 30, 0, 0, 0, 0, 0, 0, 0, 495, 512, 499, 531, 528, 483, 527, 528, 0, 0, 0, 0, 0, 0, 0, 0, 521, 510, 503, 513, 504, 522, 536, 531, 0, 0, 0, 0, 0, 0, 0, 0, 477, 485, 494, 509, 558, 510, 522, 464, 10, 7, 0, 0, 0, 0, 0, 0, 561, 515, 517, 519, 521, 522, 512, 515, 0, 0, 0, 0, 0, 0, 0, 0, 515, 491, 523, 464, 531, 515, 518, 505, 6, 1, 1, 0, 0, 0, 0, 0, 498, 518, 542, 519, 507, 485, 511, 505, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4045, 4107, 4103, 4140, 4036, 4182, 4070, 4085] avg=4096.0000 max=4182 max_rank=5 min=4036 min_rank=4 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[511, 504, 507, 542, 503, 471, 532, 530, 13, 0, 0, 0, 0, 0, 0, 0, 500, 489, 505, 490, 493, 498, 512, 524, 28, 0, 0, 0, 0, 0, 0, 0, 512, 497, 496, 568, 523, 489, 504, 521, 0, 0, 0, 0, 0, 0, 0, 0, 525, 543, 490, 499, 502, 533, 506, 513, 0, 0, 0, 0, 0, 0, 0, 0, 494, 552, 502, 531, 483, 512, 538, 522, 9, 6, 0, 0, 0, 0, 0, 0, 509, 483, 507, 530, 543, 526, 539, 495, 0, 0, 0, 0, 0, 0, 0, 0, 518, 493, 498, 551, 494, 532, 491, 517, 6, 1, 0, 0, 0, 0, 0, 0, 516, 476, 513, 496, 475, 506, 531, 500, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4039, 4110, 4111, 4149, 4132, 4101, 4013] avg=4096.0000 max=4149 max_rank=4 min=4013 min_rank=7 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16395 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[534, 529, 545, 521, 514, 514, 486, 521, 14, 0, 0, 0, 0, 0, 0, 0, 515, 508, 492, 517, 497, 513, 494, 506, 29, 0, 0, 0, 0, 0, 0, 0, 502, 520, 503, 528, 499, 514, 505, 515, 0, 0, 0, 0, 0, 0, 0, 0, 520, 535, 459, 544, 496, 529, 532, 440, 0, 0, 0, 0, 0, 0, 0, 0, 512, 524, 489, 501, 496, 541, 513, 516, 10, 6, 0, 0, 0, 0, 0, 0, 524, 488, 501, 517, 495, 528, 510, 525, 0, 0, 0, 0, 0, 0, 0, 0, 515, 508, 484, 516, 501, 519, 479, 533, 6, 1, 1, 0, 0, 0, 0, 0, 499, 502, 511, 545, 524, 508, 532, 498, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4178, 4071, 4086, 4055, 4108, 4088, 4063, 4119] avg=4096.0000 max=4178 max_rank=0 min=4055 min_rank=3 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 3968, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3968, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 3968, 4224, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[494, 500, 571, 521, 519, 592, 474, 517, 535, 546, 516, 479, 512, 498, 504, 471, 533, 522, 532, 526, 521, 469, 504, 587, 527, 520, 498, 541, 540, 462, 500, 518, 502, 510, 474, 519, 470, 495, 528, 497, 533, 516, 531, 537, 520, 511, 492, 502, 487, 480, 492, 487, 478, 500, 545, 504, 507, 476, 521, 527, 484, 521, 527, 546] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[517, 522, 537, 526, 534, 532, 528, 526, 494, 568, 524, 514, 484, 498, 503, 526, 489, 509, 451, 493, 470, 535, 457, 559, 493, 510, 514, 476, 479, 524, 517, 541, 511, 530, 522, 508, 506, 502, 493, 487, 479, 532, 539, 519, 481, 526, 513, 554, 534, 503, 527, 503, 514, 504, 512, 510, 536, 492, 509, 553, 507, 516, 502, 494] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[520, 544, 522, 541, 468, 488, 555, 544, 491, 550, 513, 503, 513, 497, 509, 547, 530, 517, 534, 528, 529, 516, 515, 508, 487, 487, 505, 498, 502, 512, 521, 516, 503, 500, 510, 522, 488, 489, 555, 508, 519, 509, 479, 493, 512, 494, 513, 505, 500, 508, 525, 502, 499, 490, 499, 498, 515, 520, 523, 486, 520, 532, 531, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[505, 508, 508, 486, 514, 526, 545, 534, 504, 531, 495, 540, 507, 506, 494, 481, 514, 497, 521, 501, 548, 531, 533, 526, 534, 533, 524, 512, 530, 514, 535, 485, 496, 509, 550, 503, 506, 470, 503, 484, 506, 535, 496, 498, 523, 527, 524, 528, 508, 464, 504, 486, 501, 520, 529, 485, 508, 497, 530, 523, 506, 518, 508, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[511, 537, 493, 539, 515, 500, 491, 525, 533, 517, 527, 488, 517, 503, 497, 515, 529, 504, 543, 521, 477, 494, 485, 530, 502, 516, 497, 528, 530, 528, 494, 479, 485, 476, 525, 532, 542, 507, 520, 487, 546, 494, 515, 536, 514, 512, 530, 499, 469, 528, 514, 502, 503, 522, 506, 500, 492, 528, 544, 511, 537, 511, 504, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[471, 499, 520, 497, 506, 489, 515, 472, 511, 472, 540, 498, 540, 513, 499, 532, 509, 504, 518, 503, 578, 533, 519, 515, 538, 513, 513, 469, 515, 515, 490, 510, 506, 534, 536, 514, 504, 505, 526, 514, 519, 517, 520, 526, 513, 504, 492, 541, 475, 517, 507, 510, 508, 535, 513, 476, 519, 485, 506, 545, 504, 505, 520, 556] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[496, 474, 516, 505, 527, 490, 510, 538, 521, 516, 562, 505, 551, 522, 531, 504, 515, 513, 500, 494, 514, 540, 496, 526, 511, 507, 533, 496, 516, 514, 531, 456, 493, 518, 508, 506, 520, 538, 466, 503, 511, 514, 505, 514, 519, 479, 493, 532, 531, 542, 511, 533, 503, 516, 532, 474, 510, 471, 512, 504, 517, 535, 484, 544] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[501, 510, 528, 474, 510, 495, 512, 511, 533, 511, 519, 530, 530, 548, 530, 506, 511, 530, 537, 518, 520, 480, 512, 493, 534, 506, 521, 526, 533, 500, 493, 517, 472, 485, 534, 515, 502, 519, 483, 486, 528, 551, 495, 494, 527, 494, 470, 477, 519, 510, 521, 517, 518, 510, 513, 536, 497, 511, 534, 529, 529, 503, 531, 479] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[494, 500, 554, 521, 519, 592, 474, 517, 0, 0, 0, 0, 0, 0, 0, 0, 535, 520, 516, 479, 512, 498, 504, 471, 0, 0, 0, 0, 0, 0, 0, 0, 533, 522, 532, 526, 521, 469, 504, 561, 0, 0, 0, 0, 0, 0, 0, 0, 527, 520, 498, 541, 540, 462, 500, 518, 7, 3, 0, 0, 0, 0, 0, 0, 502, 510, 474, 519, 470, 495, 528, 497, 26, 20, 0, 0, 0, 0, 0, 0, 533, 509, 531, 537, 520, 511, 492, 502, 0, 0, 0, 0, 0, 0, 0, 0, 487, 480, 492, 487, 478, 500, 545, 504, 18, 14, 6, 0, 0, 0, 0, 0, 507, 476, 503, 527, 484, 521, 527, 546, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4171, 4035, 4168, 4116, 4041, 4135, 4011, 4091] avg=4096.0000 max=4171 max_rank=0 min=4011 min_rank=6 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16278 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[505, 508, 494, 486, 514, 526, 545, 534, 0, 0, 0, 0, 0, 0, 0, 0, 504, 506, 495, 540, 507, 506, 494, 481, 0, 0, 0, 0, 0, 0, 0, 0, 514, 497, 521, 501, 548, 531, 533, 503, 0, 0, 0, 0, 0, 0, 0, 0, 534, 533, 524, 512, 530, 514, 535, 485, 7, 2, 0, 0, 0, 0, 0, 0, 496, 509, 550, 503, 506, 470, 503, 484, 25, 18, 0, 0, 0, 0, 0, 0, 506, 528, 496, 498, 523, 527, 524, 528, 0, 0, 0, 0, 0, 0, 0, 0, 508, 464, 504, 486, 501, 520, 529, 485, 18, 12, 5, 0, 0, 0, 0, 0, 508, 497, 512, 523, 506, 518, 508, 501, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4033, 4148, 4176, 4064, 4130, 4032, 4073] avg=4096.0000 max=4176 max_rank=3 min=4032 min_rank=6 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16299 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[520, 544, 508, 541, 468, 488, 555, 544, 0, 0, 0, 0, 0, 0, 0, 0, 491, 524, 513, 503, 513, 497, 509, 547, 0, 0, 0, 0, 0, 0, 0, 0, 530, 517, 534, 528, 529, 516, 515, 485, 0, 0, 0, 0, 0, 0, 0, 0, 487, 487, 505, 498, 502, 512, 521, 516, 7, 2, 0, 0, 0, 0, 0, 0, 503, 500, 510, 522, 488, 489, 555, 508, 26, 18, 0, 0, 0, 0, 0, 0, 519, 502, 479, 493, 512, 494, 513, 505, 0, 0, 0, 0, 0, 0, 0, 0, 500, 508, 525, 502, 499, 490, 499, 498, 18, 12, 5, 0, 0, 0, 0, 0, 515, 520, 505, 486, 520, 532, 531, 511, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4168, 4097, 4154, 4037, 4119, 4017, 4056, 4120] avg=4096.0000 max=4168 max_rank=0 min=4017 min_rank=5 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16312 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[517, 522, 522, 526, 534, 532, 528, 526, 0, 0, 0, 0, 0, 0, 0, 0, 494, 541, 524, 514, 484, 498, 503, 526, 0, 0, 0, 0, 0, 0, 0, 0, 489, 509, 451, 493, 470, 535, 457, 534, 0, 0, 0, 0, 0, 0, 0, 0, 493, 510, 514, 476, 479, 524, 517, 541, 7, 2, 0, 0, 0, 0, 0, 0, 511, 530, 522, 508, 506, 502, 493, 487, 27, 19, 0, 0, 0, 0, 0, 0, 479, 525, 539, 519, 481, 526, 513, 554, 0, 0, 0, 0, 0, 0, 0, 0, 534, 503, 527, 503, 514, 504, 512, 510, 18, 13, 6, 0, 0, 0, 0, 0, 536, 492, 491, 553, 507, 516, 502, 494, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4207, 4084, 3938, 4063, 4105, 4136, 4144, 4091] avg=4096.0000 max=4207 max_rank=0 min=3938 min_rank=2 imbalance_ratio=1.027100 max_load_reduction=28561 max_load_reduction_pct=0.871613 moved_to_echo=16476 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[511, 537, 469, 539, 515, 500, 491, 525, 0, 0, 0, 0, 0, 0, 0, 0, 533, 487, 527, 488, 517, 503, 497, 515, 0, 0, 0, 0, 0, 0, 0, 0, 529, 504, 543, 521, 477, 494, 485, 497, 0, 0, 0, 0, 0, 0, 0, 0, 502, 516, 497, 528, 530, 528, 494, 479, 10, 8, 0, 0, 0, 0, 0, 0, 485, 476, 525, 532, 542, 507, 520, 487, 30, 23, 0, 0, 0, 0, 0, 0, 546, 484, 515, 536, 514, 512, 530, 499, 0, 0, 0, 0, 0, 0, 0, 0, 469, 528, 514, 502, 503, 522, 506, 500, 23, 16, 10, 0, 0, 0, 0, 0, 492, 528, 521, 511, 537, 511, 504, 512, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4067, 4050, 4092, 4127, 4136, 4093, 4116] avg=4096.0000 max=4136 max_rank=5 min=4050 min_rank=2 imbalance_ratio=1.009766 max_load_reduction=28632 max_load_reduction_pct=0.873779 moved_to_echo=16472 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[471, 499, 506, 497, 506, 489, 515, 472, 0, 0, 0, 0, 0, 0, 0, 0, 511, 449, 540, 498, 540, 513, 499, 532, 0, 0, 0, 0, 0, 0, 0, 0, 509, 504, 518, 503, 578, 533, 519, 492, 0, 0, 0, 0, 0, 0, 0, 0, 538, 513, 513, 469, 515, 515, 490, 510, 7, 2, 0, 0, 0, 0, 0, 0, 506, 534, 536, 514, 504, 505, 526, 514, 23, 18, 0, 0, 0, 0, 0, 0, 519, 510, 520, 526, 513, 504, 492, 541, 0, 0, 0, 0, 0, 0, 0, 0, 475, 517, 507, 510, 508, 535, 513, 476, 17, 12, 5, 0, 0, 0, 0, 0, 519, 485, 489, 545, 504, 505, 520, 556, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3955, 4082, 4156, 4072, 4180, 4125, 4075, 4123] avg=4096.0000 max=4180 max_rank=4 min=3955 min_rank=0 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16503 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[501, 510, 513, 474, 510, 495, 512, 511, 0, 0, 0, 0, 0, 0, 0, 0, 533, 487, 519, 530, 530, 548, 530, 506, 0, 0, 0, 0, 0, 0, 0, 0, 511, 530, 537, 518, 520, 480, 512, 471, 0, 0, 0, 0, 0, 0, 0, 0, 534, 506, 521, 526, 533, 500, 493, 517, 7, 2, 0, 0, 0, 0, 0, 0, 472, 485, 534, 515, 502, 519, 483, 486, 24, 17, 0, 0, 0, 0, 0, 0, 528, 544, 495, 494, 527, 494, 470, 477, 0, 0, 0, 0, 0, 0, 0, 0, 519, 510, 521, 517, 518, 510, 513, 536, 18, 13, 5, 0, 0, 0, 0, 0, 497, 511, 516, 529, 529, 503, 531, 479, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4026, 4183, 4079, 4139, 4037, 4029, 4180, 4095] avg=4096.0000 max=4183 max_rank=1 min=4026 min_rank=0 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16341 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[496, 474, 502, 505, 527, 490, 510, 538, 0, 0, 0, 0, 0, 0, 0, 0, 521, 491, 562, 505, 551, 522, 531, 504, 0, 0, 0, 0, 0, 0, 0, 0, 515, 513, 500, 494, 514, 540, 496, 503, 0, 0, 0, 0, 0, 0, 0, 0, 511, 507, 533, 496, 516, 514, 531, 456, 7, 2, 0, 0, 0, 0, 0, 0, 493, 518, 508, 506, 520, 538, 466, 503, 25, 18, 0, 0, 0, 0, 0, 0, 511, 507, 505, 514, 519, 479, 493, 532, 0, 0, 0, 0, 0, 0, 0, 0, 531, 542, 511, 533, 503, 516, 532, 474, 18, 12, 5, 0, 0, 0, 0, 0, 510, 471, 494, 504, 517, 535, 484, 544, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4042, 4187, 4075, 4073, 4095, 4060, 4177, 4059] avg=4096.0000 max=4187 max_rank=1 min=4042 min_rank=0 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3968, 4096, 4224, 4224, 4096, 4096, 4096, 3968, 256, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[541, 539, 498, 505, 504, 502, 522, 512, 508, 501, 482, 501, 526, 465, 499, 526, 508, 512, 493, 519, 499, 508, 461, 520, 481, 542, 531, 534, 506, 509, 509, 495, 521, 543, 487, 524, 488, 530, 522, 508, 481, 503, 471, 543, 548, 526, 529, 502, 524, 504, 565, 523, 531, 516, 506, 513, 546, 497, 507, 500, 533, 508, 525, 486] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[497, 536, 534, 493, 503, 518, 513, 551, 451, 503, 515, 494, 545, 485, 500, 541, 509, 518, 522, 522, 542, 511, 488, 508, 523, 502, 515, 517, 530, 515, 542, 475, 509, 522, 524, 506, 505, 510, 477, 464, 536, 504, 502, 513, 525, 498, 471, 523, 505, 547, 496, 500, 495, 534, 530, 549, 510, 518, 489, 535, 485, 551, 508, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[526, 490, 542, 492, 520, 502, 494, 514, 498, 479, 517, 506, 537, 504, 514, 534, 497, 528, 507, 512, 504, 551, 530, 510, 511, 499, 526, 523, 497, 532, 544, 509, 510, 539, 540, 497, 496, 516, 515, 504, 521, 485, 529, 511, 529, 510, 479, 499, 532, 511, 506, 525, 496, 465, 488, 506, 506, 496, 519, 507, 533, 516, 507, 526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[447, 470, 531, 518, 530, 552, 501, 505, 487, 526, 511, 513, 532, 509, 491, 530, 524, 539, 499, 515, 531, 486, 519, 513, 538, 497, 557, 504, 498, 549, 524, 510, 483, 524, 502, 546, 564, 523, 537, 515, 543, 506, 468, 505, 503, 503, 517, 496, 503, 542, 498, 514, 490, 505, 496, 504, 500, 477, 519, 451, 503, 503, 543, 529] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[490, 505, 455, 515, 546, 459, 502, 452, 487, 479, 504, 519, 517, 555, 491, 504, 547, 525, 500, 536, 518, 513, 523, 520, 501, 537, 559, 551, 507, 492, 557, 512, 506, 530, 514, 501, 495, 529, 481, 531, 505, 480, 525, 534, 541, 491, 522, 519, 522, 493, 550, 489, 505, 497, 529, 523, 497, 543, 498, 506, 492, 503, 547, 492] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[561, 486, 469, 469, 537, 537, 497, 490, 508, 507, 518, 489, 541, 523, 510, 511, 544, 482, 519, 526, 485, 480, 491, 493, 499, 498, 514, 489, 508, 517, 524, 530, 498, 511, 527, 517, 520, 491, 492, 499, 517, 543, 531, 520, 563, 508, 469, 503, 499, 521, 505, 497, 478, 530, 499, 525, 532, 533, 544, 563, 503, 527, 531, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[497, 530, 485, 530, 513, 492, 507, 522, 515, 503, 478, 541, 541, 515, 524, 525, 524, 532, 521, 501, 482, 536, 521, 522, 579, 498, 497, 503, 508, 463, 515, 503, 515, 476, 473, 478, 519, 496, 496, 517, 531, 518, 500, 533, 539, 498, 519, 527, 515, 498, 540, 515, 521, 491, 518, 549, 515, 476, 492, 518, 499, 534, 506, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[548, 534, 526, 498, 512, 539, 535, 520, 496, 489, 518, 490, 498, 490, 518, 486, 536, 534, 506, 521, 511, 513, 465, 525, 484, 478, 514, 517, 497, 509, 534, 498, 527, 502, 498, 541, 510, 492, 486, 531, 526, 487, 490, 512, 522, 496, 506, 535, 537, 510, 498, 528, 554, 494, 489, 532, 537, 515, 576, 485, 503, 491, 534, 485] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[526, 490, 542, 492, 520, 502, 494, 514, 10, 2, 0, 0, 0, 0, 0, 0, 498, 479, 517, 506, 537, 504, 514, 534, 21, 9, 0, 0, 0, 0, 0, 0, 487, 528, 507, 512, 504, 551, 530, 510, 0, 0, 0, 0, 0, 0, 0, 0, 511, 499, 526, 523, 497, 532, 523, 509, 0, 0, 0, 0, 0, 0, 0, 0, 510, 539, 540, 497, 496, 516, 515, 504, 11, 2, 0, 0, 0, 0, 0, 0, 521, 485, 529, 511, 527, 510, 479, 499, 0, 0, 0, 0, 0, 0, 0, 0, 532, 511, 506, 525, 496, 465, 488, 495, 0, 0, 0, 0, 0, 0, 0, 0, 506, 496, 519, 507, 533, 516, 496, 526, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4119, 4129, 4120, 4130, 4061, 4018, 4099] avg=4096.0000 max=4130 max_rank=4 min=4018 min_rank=6 imbalance_ratio=1.008301 max_load_reduction=28638 max_load_reduction_pct=0.873962 moved_to_echo=16308 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[541, 539, 498, 505, 504, 502, 522, 512, 10, 2, 0, 0, 0, 0, 0, 0, 508, 501, 482, 501, 526, 465, 499, 526, 20, 9, 0, 0, 0, 0, 0, 0, 498, 512, 493, 519, 499, 508, 461, 520, 0, 0, 0, 0, 0, 0, 0, 0, 481, 542, 531, 534, 506, 509, 489, 495, 0, 0, 0, 0, 0, 0, 0, 0, 521, 543, 487, 524, 488, 530, 522, 508, 11, 2, 0, 0, 0, 0, 0, 0, 481, 503, 471, 543, 546, 526, 529, 502, 0, 0, 0, 0, 0, 0, 0, 0, 524, 504, 565, 523, 531, 516, 506, 502, 0, 0, 0, 0, 0, 0, 0, 0, 546, 497, 507, 500, 533, 508, 514, 486, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4135, 4037, 4010, 4087, 4136, 4101, 4171, 4091] avg=4096.0000 max=4171 max_rank=6 min=4010 min_rank=2 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16499 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[447, 470, 531, 518, 530, 552, 501, 505, 10, 2, 0, 0, 0, 0, 0, 0, 487, 526, 511, 513, 532, 509, 491, 530, 20, 9, 0, 0, 0, 0, 0, 0, 514, 539, 499, 515, 531, 486, 519, 513, 0, 0, 0, 0, 0, 0, 0, 0, 538, 497, 557, 504, 498, 549, 504, 510, 0, 0, 0, 0, 0, 0, 0, 0, 483, 524, 502, 546, 564, 523, 537, 515, 12, 2, 0, 0, 0, 0, 0, 0, 543, 506, 468, 505, 501, 503, 517, 496, 0, 0, 0, 0, 0, 0, 0, 0, 503, 542, 498, 514, 490, 505, 496, 493, 0, 0, 0, 0, 0, 0, 0, 0, 500, 477, 519, 451, 503, 503, 531, 529, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 4128, 4116, 4157, 4208, 4039, 4041, 4013] avg=4096.0000 max=4208 max_rank=4 min=4013 min_rank=7 imbalance_ratio=1.027344 max_load_reduction=28560 max_load_reduction_pct=0.871582 moved_to_echo=16301 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[497, 536, 534, 493, 503, 518, 513, 551, 10, 2, 0, 0, 0, 0, 0, 0, 451, 503, 515, 494, 545, 485, 500, 541, 21, 10, 0, 0, 0, 0, 0, 0, 499, 518, 522, 522, 542, 511, 488, 508, 0, 0, 0, 0, 0, 0, 0, 0, 523, 502, 515, 517, 530, 515, 521, 475, 0, 0, 0, 0, 0, 0, 0, 0, 509, 522, 524, 506, 505, 510, 477, 464, 11, 2, 0, 0, 0, 0, 0, 0, 536, 504, 502, 513, 523, 498, 471, 523, 0, 0, 0, 0, 0, 0, 0, 0, 505, 547, 496, 500, 495, 534, 530, 537, 0, 0, 0, 0, 0, 0, 0, 0, 510, 518, 489, 535, 485, 551, 497, 509, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4157, 4065, 4110, 4098, 4030, 4070, 4144, 4094] avg=4096.0000 max=4157 max_rank=0 min=4030 min_rank=4 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16338 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[561, 486, 469, 469, 537, 537, 497, 490, 15, 7, 0, 0, 0, 0, 0, 0, 508, 507, 518, 489, 541, 523, 510, 511, 23, 14, 0, 0, 0, 0, 0, 0, 527, 482, 519, 526, 485, 480, 491, 493, 0, 0, 0, 0, 0, 0, 0, 0, 499, 498, 514, 489, 508, 517, 501, 530, 0, 0, 0, 0, 0, 0, 0, 0, 498, 511, 527, 517, 520, 491, 492, 499, 15, 8, 2, 0, 0, 0, 0, 0, 517, 543, 531, 520, 556, 508, 469, 503, 0, 0, 0, 0, 0, 0, 0, 0, 499, 521, 505, 497, 478, 530, 499, 503, 0, 0, 0, 0, 0, 0, 0, 0, 532, 533, 544, 563, 503, 527, 516, 520, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4144, 4003, 4056, 4080, 4147, 4032, 4238] avg=4096.0000 max=4238 max_rank=7 min=4003 min_rank=2 imbalance_ratio=1.034668 max_load_reduction=28530 max_load_reduction_pct=0.870667 moved_to_echo=16497 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[490, 505, 455, 515, 546, 459, 502, 452, 11, 2, 0, 0, 0, 0, 0, 0, 487, 479, 504, 519, 517, 555, 491, 504, 22, 9, 0, 0, 0, 0, 0, 0, 536, 525, 500, 536, 518, 513, 523, 520, 0, 0, 0, 0, 0, 0, 0, 0, 501, 537, 559, 551, 507, 492, 535, 512, 0, 0, 0, 0, 0, 0, 0, 0, 506, 530, 514, 501, 495, 529, 481, 531, 12, 2, 0, 0, 0, 0, 0, 0, 505, 480, 525, 534, 539, 491, 522, 519, 0, 0, 0, 0, 0, 0, 0, 0, 522, 493, 550, 489, 505, 497, 529, 512, 0, 0, 0, 0, 0, 0, 0, 0, 497, 543, 498, 506, 492, 503, 535, 492, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3937, 4087, 4171, 4194, 4101, 4115, 4097, 4066] avg=4096.0000 max=4194 max_rank=3 min=3937 min_rank=0 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[548, 534, 526, 498, 512, 539, 535, 520, 11, 2, 0, 0, 0, 0, 0, 0, 496, 489, 518, 490, 498, 490, 518, 486, 21, 10, 0, 0, 0, 0, 0, 0, 525, 534, 506, 521, 511, 513, 465, 525, 0, 0, 0, 0, 0, 0, 0, 0, 484, 478, 514, 517, 497, 509, 513, 498, 0, 0, 0, 0, 0, 0, 0, 0, 527, 502, 498, 541, 510, 492, 486, 531, 11, 2, 0, 0, 0, 0, 0, 0, 526, 487, 490, 512, 520, 496, 506, 535, 0, 0, 0, 0, 0, 0, 0, 0, 537, 510, 498, 528, 554, 494, 489, 520, 0, 0, 0, 0, 0, 0, 0, 0, 537, 515, 576, 485, 503, 491, 523, 485, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4225, 4016, 4100, 4010, 4100, 4072, 4130, 4115] avg=4096.0000 max=4225 max_rank=0 min=4010 min_rank=3 imbalance_ratio=1.031494 max_load_reduction=28543 max_load_reduction_pct=0.871063 moved_to_echo=16417 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[497, 530, 485, 530, 513, 492, 507, 522, 10, 2, 0, 0, 0, 0, 0, 0, 515, 503, 478, 541, 541, 515, 524, 525, 20, 10, 0, 0, 0, 0, 0, 0, 514, 532, 521, 501, 482, 536, 521, 522, 0, 0, 0, 0, 0, 0, 0, 0, 579, 498, 497, 503, 508, 463, 495, 503, 0, 0, 0, 0, 0, 0, 0, 0, 515, 476, 473, 478, 519, 496, 496, 517, 11, 2, 0, 0, 0, 0, 0, 0, 531, 518, 500, 533, 537, 498, 519, 527, 0, 0, 0, 0, 0, 0, 0, 0, 515, 498, 540, 515, 521, 491, 518, 537, 0, 0, 0, 0, 0, 0, 0, 0, 515, 476, 492, 518, 499, 534, 495, 523, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4172, 4129, 4046, 3983, 4163, 4135, 4052] avg=4096.0000 max=4172 max_rank=1 min=3983 min_rank=4 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16333 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4352, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3968, 4096, 4096, 4096, 4352, 4096, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[554, 544, 500, 550, 527, 557, 488, 466, 477, 503, 530, 466, 514, 489, 473, 509, 484, 543, 500, 491, 505, 506, 519, 574, 487, 519, 506, 479, 531, 546, 506, 570, 510, 551, 522, 507, 515, 522, 498, 501, 477, 513, 500, 533, 518, 529, 484, 504, 538, 481, 520, 536, 503, 539, 497, 512, 525, 492, 496, 488, 473, 500, 560, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[511, 530, 529, 485, 511, 532, 528, 477, 519, 497, 548, 494, 495, 542, 504, 489, 515, 524, 511, 488, 516, 514, 484, 525, 486, 534, 516, 462, 536, 502, 511, 483, 502, 519, 527, 516, 495, 525, 491, 536, 532, 483, 482, 532, 533, 490, 510, 507, 489, 519, 544, 553, 512, 527, 501, 496, 525, 514, 500, 482, 564, 504, 518, 542] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[481, 499, 504, 511, 527, 516, 497, 540, 518, 524, 493, 508, 519, 509, 542, 498, 495, 491, 505, 497, 557, 514, 499, 528, 545, 515, 468, 556, 532, 507, 508, 536, 526, 525, 482, 500, 533, 474, 507, 502, 561, 479, 530, 506, 508, 511, 508, 534, 516, 498, 532, 495, 509, 499, 477, 540, 483, 518, 476, 535, 517, 520, 518, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[504, 497, 510, 487, 497, 491, 532, 530, 545, 541, 521, 515, 510, 501, 535, 516, 501, 528, 507, 523, 521, 510, 496, 482, 532, 495, 507, 472, 487, 504, 547, 522, 478, 478, 497, 510, 494, 493, 494, 512, 524, 562, 535, 503, 516, 521, 523, 510, 477, 538, 531, 525, 543, 526, 525, 507, 498, 509, 487, 468, 541, 516, 513, 548] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[507, 515, 495, 484, 473, 495, 511, 499, 513, 500, 506, 510, 495, 520, 541, 541, 509, 507, 527, 505, 548, 508, 500, 498, 540, 493, 509, 554, 506, 483, 531, 490, 532, 502, 480, 485, 491, 517, 494, 521, 504, 500, 513, 584, 511, 506, 520, 530, 534, 505, 547, 524, 531, 514, 485, 486, 502, 475, 510, 541, 557, 532, 522, 500] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[530, 484, 498, 500, 521, 454, 519, 483, 503, 498, 505, 544, 536, 480, 507, 494, 537, 541, 531, 548, 530, 491, 485, 533, 505, 521, 512, 497, 514, 488, 567, 511, 532, 499, 525, 529, 520, 472, 536, 482, 530, 528, 515, 517, 508, 492, 532, 523, 511, 505, 522, 495, 550, 536, 505, 524, 486, 520, 503, 493, 497, 494, 528, 492] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[486, 483, 507, 515, 493, 540, 530, 475, 500, 518, 528, 533, 555, 524, 506, 517, 516, 481, 514, 511, 502, 549, 516, 473, 512, 515, 494, 479, 508, 527, 507, 510, 547, 517, 526, 531, 525, 474, 518, 480, 517, 511, 517, 491, 483, 530, 482, 546, 549, 505, 519, 527, 513, 485, 504, 523, 493, 499, 515, 511, 510, 532, 540, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[565, 537, 498, 480, 516, 525, 540, 512, 517, 528, 507, 516, 500, 521, 488, 517, 474, 533, 513, 520, 498, 506, 515, 528, 479, 521, 497, 507, 505, 498, 526, 498, 480, 494, 554, 510, 516, 500, 534, 551, 512, 528, 518, 521, 478, 475, 543, 499, 524, 520, 525, 547, 522, 511, 462, 492, 516, 542, 501, 481, 521, 486, 473, 547] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[554, 544, 500, 550, 527, 557, 488, 466, 22, 0, 0, 0, 0, 0, 0, 0, 477, 503, 525, 466, 514, 489, 473, 509, 0, 0, 0, 0, 0, 0, 0, 0, 484, 543, 500, 491, 502, 506, 519, 574, 0, 0, 0, 0, 0, 0, 0, 0, 487, 519, 506, 479, 531, 546, 506, 570, 3, 3, 0, 0, 0, 0, 0, 0, 510, 551, 522, 507, 515, 522, 498, 501, 12, 6, 0, 0, 0, 0, 0, 0, 477, 513, 500, 514, 518, 529, 484, 504, 0, 0, 0, 0, 0, 0, 0, 0, 538, 481, 492, 536, 503, 539, 497, 512, 0, 0, 0, 0, 0, 0, 0, 0, 525, 492, 496, 488, 473, 500, 560, 511, 7, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4208, 3956, 4119, 4150, 4144, 4039, 4098, 4054] avg=4096.0000 max=4208 max_rank=0 min=3956 min_rank=1 imbalance_ratio=1.027344 max_load_reduction=28560 max_load_reduction_pct=0.871582 moved_to_echo=16335 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[481, 499, 504, 511, 527, 516, 497, 540, 23, 0, 0, 0, 0, 0, 0, 0, 518, 524, 489, 508, 519, 509, 542, 498, 0, 0, 0, 0, 0, 0, 0, 0, 495, 491, 505, 497, 553, 514, 499, 528, 0, 0, 0, 0, 0, 0, 0, 0, 545, 515, 468, 556, 532, 507, 508, 536, 4, 2, 0, 0, 0, 0, 0, 0, 526, 525, 482, 500, 533, 474, 507, 502, 11, 6, 0, 0, 0, 0, 0, 0, 561, 479, 530, 489, 508, 511, 508, 534, 0, 0, 0, 0, 0, 0, 0, 0, 516, 498, 503, 495, 509, 499, 477, 540, 0, 0, 0, 0, 0, 0, 0, 0, 483, 518, 476, 535, 517, 520, 518, 510, 6, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4107, 4082, 4173, 4066, 4120, 4037, 4085] avg=4096.0000 max=4173 max_rank=3 min=4037 min_rank=6 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16308 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[507, 515, 495, 484, 473, 495, 511, 499, 23, 0, 0, 0, 0, 0, 0, 0, 513, 500, 501, 510, 495, 520, 541, 541, 0, 0, 0, 0, 0, 0, 0, 0, 509, 507, 527, 505, 544, 508, 500, 498, 0, 0, 0, 0, 0, 0, 0, 0, 540, 493, 509, 554, 506, 483, 531, 490, 4, 3, 0, 0, 0, 0, 0, 0, 532, 502, 480, 485, 491, 517, 494, 521, 13, 6, 0, 0, 0, 0, 0, 0, 504, 500, 513, 564, 511, 506, 520, 530, 0, 0, 0, 0, 0, 0, 0, 0, 534, 505, 518, 524, 531, 514, 485, 486, 0, 0, 0, 0, 0, 0, 0, 0, 502, 475, 510, 541, 557, 532, 522, 500, 7, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4002, 4121, 4098, 4113, 4041, 4148, 4097, 4148] avg=4096.0000 max=4148 max_rank=5 min=4002 min_rank=0 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16434 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[504, 497, 510, 487, 497, 491, 532, 530, 23, 0, 0, 0, 0, 0, 0, 0, 545, 541, 516, 515, 510, 501, 535, 516, 0, 0, 0, 0, 0, 0, 0, 0, 501, 528, 507, 523, 518, 510, 496, 482, 0, 0, 0, 0, 0, 0, 0, 0, 532, 495, 507, 472, 487, 504, 547, 522, 3, 3, 0, 0, 0, 0, 0, 0, 478, 478, 497, 510, 494, 493, 494, 512, 11, 6, 0, 0, 0, 0, 0, 0, 524, 562, 535, 486, 516, 521, 523, 510, 0, 0, 0, 0, 0, 0, 0, 0, 477, 538, 502, 525, 543, 526, 525, 507, 0, 0, 0, 0, 0, 0, 0, 0, 498, 509, 487, 468, 541, 516, 513, 548, 6, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4071, 4179, 4065, 4072, 3973, 4177, 4143, 4088] avg=4096.0000 max=4179 max_rank=1 min=3973 min_rank=4 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[530, 484, 498, 500, 521, 454, 519, 483, 27, 0, 0, 0, 0, 0, 0, 0, 503, 498, 495, 544, 536, 480, 507, 494, 0, 0, 0, 0, 0, 0, 0, 0, 537, 541, 531, 548, 521, 491, 485, 533, 0, 0, 0, 0, 0, 0, 0, 0, 505, 521, 512, 497, 514, 488, 567, 511, 9, 5, 0, 0, 0, 0, 0, 0, 532, 499, 525, 529, 520, 472, 536, 482, 17, 11, 0, 0, 0, 0, 0, 0, 530, 528, 515, 489, 508, 492, 532, 523, 0, 0, 0, 0, 0, 0, 0, 0, 511, 505, 484, 495, 550, 536, 505, 524, 0, 0, 0, 0, 0, 0, 0, 0, 486, 520, 503, 493, 497, 494, 528, 492, 11, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4016, 4057, 4187, 4129, 4123, 4117, 4110, 4029] avg=4096.0000 max=4187 max_rank=2 min=4016 min_rank=0 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[486, 483, 507, 515, 493, 540, 530, 475, 22, 0, 0, 0, 0, 0, 0, 0, 500, 518, 523, 533, 555, 524, 506, 517, 0, 0, 0, 0, 0, 0, 0, 0, 516, 481, 514, 511, 499, 549, 516, 473, 0, 0, 0, 0, 0, 0, 0, 0, 512, 515, 494, 479, 508, 527, 507, 510, 3, 3, 0, 0, 0, 0, 0, 0, 547, 517, 526, 531, 525, 474, 518, 480, 11, 6, 0, 0, 0, 0, 0, 0, 517, 511, 517, 474, 483, 530, 482, 546, 0, 0, 0, 0, 0, 0, 0, 0, 549, 505, 491, 527, 513, 485, 504, 523, 0, 0, 0, 0, 0, 0, 0, 0, 493, 499, 515, 511, 510, 532, 540, 524, 6, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4051, 4176, 4059, 4058, 4135, 4060, 4097, 4132] avg=4096.0000 max=4176 max_rank=1 min=4051 min_rank=0 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16424 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[565, 537, 498, 480, 516, 525, 540, 512, 23, 0, 0, 0, 0, 0, 0, 0, 517, 528, 502, 516, 500, 521, 488, 517, 0, 0, 0, 0, 0, 0, 0, 0, 474, 533, 513, 520, 495, 506, 515, 528, 0, 0, 0, 0, 0, 0, 0, 0, 479, 521, 497, 507, 505, 498, 526, 498, 3, 3, 0, 0, 0, 0, 0, 0, 480, 494, 554, 510, 516, 500, 534, 551, 12, 6, 0, 0, 0, 0, 0, 0, 512, 528, 518, 503, 478, 475, 543, 499, 0, 0, 0, 0, 0, 0, 0, 0, 524, 520, 496, 547, 522, 511, 462, 492, 0, 0, 0, 0, 0, 0, 0, 0, 516, 542, 501, 481, 521, 486, 473, 547, 6, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4196, 4089, 4084, 4037, 4157, 4056, 4074, 4075] avg=4096.0000 max=4196 max_rank=0 min=4037 min_rank=3 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16362 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[511, 530, 529, 485, 511, 532, 528, 477, 23, 0, 0, 0, 0, 0, 0, 0, 519, 497, 543, 494, 495, 542, 504, 489, 0, 0, 0, 0, 0, 0, 0, 0, 515, 524, 511, 488, 513, 514, 484, 525, 0, 0, 0, 0, 0, 0, 0, 0, 486, 534, 516, 462, 536, 502, 511, 483, 3, 3, 0, 0, 0, 0, 0, 0, 502, 519, 527, 516, 495, 525, 491, 536, 12, 6, 0, 0, 0, 0, 0, 0, 532, 483, 482, 514, 533, 490, 510, 507, 0, 0, 0, 0, 0, 0, 0, 0, 489, 519, 515, 553, 512, 527, 501, 496, 0, 0, 0, 0, 0, 0, 0, 0, 525, 514, 500, 482, 564, 504, 518, 542, 6, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4083, 4074, 4036, 4129, 4051, 4112, 4157] avg=4096.0000 max=4157 max_rank=7 min=4036 min_rank=3 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16449 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 3968, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33024 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[508, 509, 534, 506, 500, 478, 494, 522, 509, 491, 504, 526, 491, 483, 532, 524, 517, 500, 538, 504, 496, 476, 510, 518, 479, 505, 529, 527, 496, 543, 526, 511, 498, 490, 528, 517, 505, 492, 500, 553, 535, 520, 504, 532, 516, 536, 546, 499, 526, 495, 491, 508, 543, 524, 506, 508, 554, 536, 503, 535, 521, 505, 494, 462] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[500, 562, 502, 530, 504, 477, 526, 487, 504, 446, 530, 518, 523, 520, 504, 532, 542, 497, 531, 481, 462, 531, 501, 521, 530, 531, 499, 534, 538, 510, 515, 456, 527, 499, 515, 522, 506, 505, 525, 536, 534, 499, 506, 524, 530, 517, 498, 508, 513, 477, 499, 489, 496, 503, 543, 517, 511, 509, 483, 510, 526, 526, 549, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[584, 519, 523, 520, 532, 509, 510, 525, 516, 515, 505, 462, 487, 514, 500, 567, 516, 514, 550, 503, 497, 507, 507, 480, 531, 531, 477, 501, 516, 484, 528, 519, 506, 513, 509, 525, 530, 455, 509, 560, 537, 494, 541, 542, 509, 512, 495, 491, 564, 494, 532, 480, 516, 514, 509, 532, 482, 496, 508, 491, 509, 492, 507, 465] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[544, 531, 510, 502, 560, 508, 509, 537, 519, 475, 504, 545, 512, 491, 504, 515, 495, 500, 540, 531, 535, 484, 483, 509, 478, 518, 517, 503, 532, 486, 521, 523, 542, 527, 552, 524, 520, 522, 533, 470, 504, 511, 515, 539, 530, 499, 530, 534, 492, 496, 499, 503, 491, 508, 513, 501, 496, 509, 509, 502, 501, 483, 509, 483] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[483, 512, 502, 520, 524, 488, 530, 499, 513, 542, 507, 544, 523, 485, 485, 539, 508, 517, 554, 533, 479, 518, 502, 488, 484, 549, 546, 514, 533, 526, 501, 527, 497, 488, 514, 508, 516, 507, 518, 555, 546, 527, 513, 494, 510, 515, 475, 511, 516, 533, 506, 527, 498, 516, 485, 491, 500, 505, 490, 466, 536, 485, 530, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[559, 504, 528, 517, 496, 516, 506, 503, 510, 500, 501, 500, 512, 527, 508, 508, 540, 508, 489, 491, 528, 516, 506, 515, 521, 545, 557, 522, 508, 523, 507, 489, 508, 486, 506, 499, 525, 505, 482, 524, 493, 488, 533, 523, 483, 516, 471, 509, 508, 515, 509, 537, 504, 527, 555, 512, 491, 518, 552, 513, 497, 497, 515, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[505, 532, 501, 526, 524, 474, 547, 483, 495, 492, 469, 509, 505, 518, 487, 516, 540, 523, 505, 495, 528, 487, 523, 497, 518, 511, 498, 522, 531, 518, 525, 495, 486, 494, 495, 487, 526, 508, 565, 472, 512, 485, 558, 492, 499, 544, 497, 501, 490, 510, 534, 509, 496, 542, 521, 577, 510, 503, 540, 495, 548, 537, 518, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[508, 508, 525, 504, 497, 486, 513, 509, 513, 563, 485, 529, 494, 538, 485, 522, 488, 510, 489, 507, 523, 539, 502, 528, 515, 543, 493, 534, 498, 570, 483, 501, 511, 546, 545, 485, 505, 511, 525, 521, 500, 480, 533, 519, 492, 535, 509, 495, 535, 497, 475, 546, 520, 483, 512, 533, 497, 497, 506, 520, 522, 512, 502, 497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[495, 509, 534, 506, 500, 478, 494, 522, 0, 0, 0, 0, 0, 0, 0, 0, 509, 491, 504, 526, 491, 483, 532, 524, 13, 7, 0, 0, 0, 0, 0, 0, 517, 500, 538, 504, 496, 476, 510, 518, 8, 4, 1, 0, 0, 0, 0, 0, 479, 478, 529, 527, 496, 543, 526, 511, 0, 0, 0, 0, 0, 0, 0, 0, 498, 490, 528, 517, 505, 492, 500, 545, 0, 0, 0, 0, 0, 0, 0, 0, 535, 520, 496, 532, 516, 536, 546, 499, 0, 0, 0, 0, 0, 0, 0, 0, 526, 495, 491, 508, 543, 524, 506, 504, 0, 0, 0, 0, 0, 0, 0, 0, 554, 536, 503, 535, 521, 505, 494, 462, 27, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4038, 4080, 4072, 4089, 4075, 4180, 4097, 4137] avg=4096.0000 max=4180 max_rank=5 min=4038 min_rank=0 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16489 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[487, 562, 502, 530, 504, 477, 526, 487, 0, 0, 0, 0, 0, 0, 0, 0, 504, 446, 530, 518, 523, 520, 504, 532, 13, 7, 0, 0, 0, 0, 0, 0, 542, 497, 531, 481, 462, 531, 501, 521, 8, 4, 1, 0, 0, 0, 0, 0, 530, 502, 499, 534, 538, 510, 515, 456, 0, 0, 0, 0, 0, 0, 0, 0, 527, 499, 515, 522, 506, 505, 525, 528, 0, 0, 0, 0, 0, 0, 0, 0, 534, 499, 498, 524, 530, 517, 498, 508, 0, 0, 0, 0, 0, 0, 0, 0, 513, 477, 499, 489, 496, 503, 543, 513, 0, 0, 0, 0, 0, 0, 0, 0, 511, 509, 483, 510, 526, 526, 549, 522, 29, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4097, 4079, 4084, 4127, 4108, 4033, 4165] avg=4096.0000 max=4165 max_rank=7 min=4033 min_rank=6 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16433 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[567, 519, 523, 520, 532, 509, 510, 525, 0, 0, 0, 0, 0, 0, 0, 0, 516, 515, 505, 462, 487, 514, 500, 567, 16, 7, 0, 0, 0, 0, 0, 0, 516, 514, 550, 503, 497, 507, 507, 480, 8, 4, 2, 0, 0, 0, 0, 0, 531, 502, 477, 501, 516, 484, 528, 519, 0, 0, 0, 0, 0, 0, 0, 0, 506, 513, 509, 525, 530, 455, 509, 552, 0, 0, 0, 0, 0, 0, 0, 0, 537, 494, 532, 542, 509, 512, 495, 491, 0, 0, 0, 0, 0, 0, 0, 0, 564, 494, 532, 480, 516, 514, 509, 528, 0, 0, 0, 0, 0, 0, 0, 0, 482, 496, 508, 491, 509, 492, 507, 465, 29, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4205, 4089, 4088, 4058, 4099, 4112, 4137, 3980] avg=4096.0000 max=4205 max_rank=0 min=3980 min_rank=7 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16328 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[519, 531, 510, 502, 560, 508, 509, 537, 0, 0, 0, 0, 0, 0, 0, 0, 519, 475, 504, 545, 512, 491, 504, 515, 19, 11, 0, 0, 0, 0, 0, 0, 495, 500, 540, 531, 535, 484, 483, 509, 12, 6, 5, 0, 0, 0, 0, 0, 478, 486, 517, 503, 532, 486, 521, 523, 0, 0, 0, 0, 0, 0, 0, 0, 542, 527, 552, 524, 520, 522, 533, 458, 0, 0, 0, 0, 0, 0, 0, 0, 504, 511, 499, 539, 530, 499, 530, 534, 0, 0, 0, 0, 0, 0, 0, 0, 492, 496, 499, 503, 491, 508, 513, 495, 0, 0, 0, 0, 0, 0, 0, 0, 496, 509, 509, 502, 501, 483, 509, 483, 32, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4176, 4095, 4100, 4046, 4178, 4146, 3997, 4030] avg=4096.0000 max=4178 max_rank=4 min=3997 min_rank=6 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16351 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[470, 512, 502, 520, 524, 488, 530, 499, 0, 0, 0, 0, 0, 0, 0, 0, 513, 542, 507, 544, 523, 485, 485, 539, 13, 7, 0, 0, 0, 0, 0, 0, 508, 517, 554, 533, 479, 518, 502, 488, 8, 3, 1, 0, 0, 0, 0, 0, 484, 519, 546, 514, 533, 526, 501, 527, 0, 0, 0, 0, 0, 0, 0, 0, 497, 488, 514, 508, 516, 507, 518, 547, 0, 0, 0, 0, 0, 0, 0, 0, 546, 527, 505, 494, 510, 515, 475, 511, 0, 0, 0, 0, 0, 0, 0, 0, 516, 533, 506, 527, 498, 516, 485, 488, 0, 0, 0, 0, 0, 0, 0, 0, 500, 505, 490, 466, 536, 485, 530, 515, 30, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4045, 4158, 4111, 4150, 4095, 4083, 4069, 4057] avg=4096.0000 max=4158 max_rank=1 min=4045 min_rank=0 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16304 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[543, 504, 528, 517, 496, 516, 506, 503, 0, 0, 0, 0, 0, 0, 0, 0, 510, 500, 501, 500, 512, 527, 508, 508, 15, 7, 0, 0, 0, 0, 0, 0, 540, 508, 489, 491, 528, 516, 506, 515, 8, 4, 2, 0, 0, 0, 0, 0, 521, 516, 557, 522, 508, 523, 507, 489, 0, 0, 0, 0, 0, 0, 0, 0, 508, 486, 506, 499, 525, 505, 482, 516, 0, 0, 0, 0, 0, 0, 0, 0, 493, 488, 524, 523, 483, 516, 471, 509, 0, 0, 0, 0, 0, 0, 0, 0, 508, 515, 509, 537, 504, 527, 555, 508, 0, 0, 0, 0, 0, 0, 0, 0, 491, 518, 552, 513, 497, 497, 515, 507, 29, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4088, 4107, 4143, 4027, 4007, 4163, 4120] avg=4096.0000 max=4163 max_rank=6 min=4007 min_rank=5 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16317 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[492, 532, 501, 526, 524, 474, 547, 483, 0, 0, 0, 0, 0, 0, 0, 0, 495, 492, 469, 509, 505, 518, 487, 516, 13, 8, 0, 0, 0, 0, 0, 0, 540, 523, 505, 495, 528, 487, 523, 497, 7, 4, 2, 0, 0, 0, 0, 0, 518, 483, 498, 522, 531, 518, 525, 495, 0, 0, 0, 0, 0, 0, 0, 0, 486, 494, 495, 487, 526, 508, 565, 465, 0, 0, 0, 0, 0, 0, 0, 0, 512, 485, 548, 492, 499, 544, 497, 501, 0, 0, 0, 0, 0, 0, 0, 0, 490, 510, 534, 509, 496, 542, 521, 573, 0, 0, 0, 0, 0, 0, 0, 0, 510, 503, 540, 495, 548, 537, 518, 518, 28, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4012, 4111, 4090, 4026, 4078, 4175, 4197] avg=4096.0000 max=4197 max_rank=7 min=4012 min_rank=1 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16476 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[495, 508, 525, 504, 497, 486, 513, 509, 0, 0, 0, 0, 0, 0, 0, 0, 513, 563, 485, 529, 494, 538, 485, 522, 13, 7, 0, 0, 0, 0, 0, 0, 488, 510, 489, 507, 523, 539, 502, 528, 8, 4, 2, 0, 0, 0, 0, 0, 515, 514, 493, 534, 498, 570, 483, 501, 0, 0, 0, 0, 0, 0, 0, 0, 511, 546, 545, 485, 505, 511, 525, 513, 0, 0, 0, 0, 0, 0, 0, 0, 500, 480, 524, 519, 492, 535, 509, 495, 0, 0, 0, 0, 0, 0, 0, 0, 535, 497, 475, 546, 520, 483, 512, 529, 0, 0, 0, 0, 0, 0, 0, 0, 497, 497, 506, 520, 522, 512, 502, 497, 29, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4037, 4149, 4100, 4108, 4141, 4054, 4097, 4082] avg=4096.0000 max=4149 max_rank=1 min=4037 min_rank=0 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16374 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 3968, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[487, 477, 538, 525, 513, 471, 522, 532, 535, 552, 501, 489, 515, 481, 555, 527, 503, 514, 509, 535, 515, 497, 512, 515, 499, 549, 503, 452, 513, 495, 530, 494, 513, 563, 534, 503, 536, 532, 495, 528, 502, 512, 523, 480, 526, 506, 491, 506, 499, 507, 530, 499, 458, 535, 543, 496, 525, 485, 513, 546, 543, 509, 471, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[518, 511, 512, 509, 488, 484, 504, 521, 504, 518, 515, 489, 520, 516, 504, 527, 477, 548, 533, 520, 502, 516, 478, 538, 473, 479, 523, 527, 499, 501, 515, 526, 471, 525, 491, 527, 486, 557, 497, 508, 538, 551, 520, 508, 487, 539, 518, 508, 527, 496, 470, 498, 516, 500, 543, 519, 547, 502, 513, 537, 514, 512, 537, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[503, 497, 493, 509, 509, 482, 534, 522, 516, 513, 527, 500, 525, 493, 512, 517, 502, 484, 493, 500, 523, 515, 505, 532, 525, 527, 524, 501, 532, 523, 502, 515, 493, 513, 525, 522, 505, 509, 495, 539, 483, 527, 497, 529, 498, 512, 544, 522, 540, 515, 526, 533, 519, 503, 514, 526, 467, 503, 512, 490, 496, 487, 540, 529] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[514, 465, 519, 489, 519, 505, 508, 488, 523, 505, 502, 487, 516, 535, 505, 497, 528, 487, 495, 496, 515, 495, 512, 530, 526, 516, 527, 514, 516, 540, 535, 523, 541, 545, 520, 495, 517, 528, 538, 521, 522, 512, 527, 539, 511, 488, 500, 489, 477, 505, 505, 529, 515, 511, 503, 520, 506, 501, 558, 476, 487, 495, 530, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[522, 467, 518, 522, 549, 515, 522, 542, 494, 515, 510, 501, 520, 494, 511, 528, 532, 502, 488, 491, 485, 503, 493, 513, 490, 499, 538, 498, 550, 527, 479, 509, 555, 568, 518, 483, 494, 504, 517, 552, 501, 537, 514, 527, 496, 466, 531, 509, 518, 497, 534, 484, 511, 509, 507, 498, 510, 507, 516, 497, 513, 528, 537, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[464, 501, 506, 497, 480, 506, 499, 501, 556, 487, 549, 551, 496, 498, 489, 489, 543, 488, 510, 505, 508, 555, 510, 482, 490, 542, 532, 499, 507, 491, 528, 514, 481, 488, 529, 517, 517, 542, 547, 513, 532, 503, 487, 478, 506, 516, 498, 524, 539, 527, 549, 508, 523, 529, 495, 508, 513, 512, 503, 533, 516, 495, 542, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[480, 511, 514, 513, 488, 503, 523, 524, 489, 511, 529, 527, 525, 518, 538, 525, 524, 510, 490, 481, 571, 494, 555, 546, 491, 538, 516, 521, 527, 528, 487, 486, 516, 526, 499, 549, 525, 449, 515, 505, 493, 492, 518, 496, 504, 525, 505, 450, 532, 528, 522, 517, 516, 486, 550, 531, 479, 497, 513, 527, 506, 499, 533, 482] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[511, 518, 516, 542, 525, 528, 512, 511, 466, 537, 550, 506, 534, 519, 507, 472, 496, 487, 503, 512, 466, 505, 515, 496, 512, 522, 548, 510, 522, 471, 478, 523, 507, 561, 484, 512, 518, 522, 507, 495, 519, 500, 540, 559, 493, 490, 534, 546, 524, 503, 496, 533, 513, 495, 538, 504, 494, 505, 507, 492, 484, 497, 544, 532] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[487, 477, 538, 525, 513, 471, 522, 532, 35, 0, 0, 0, 0, 0, 0, 0, 535, 552, 491, 489, 515, 481, 555, 527, 0, 0, 0, 0, 0, 0, 0, 0, 503, 514, 509, 535, 515, 497, 512, 515, 17, 6, 0, 0, 0, 0, 0, 0, 499, 549, 500, 452, 513, 495, 530, 494, 0, 0, 0, 0, 0, 0, 0, 0, 513, 522, 534, 503, 536, 532, 495, 528, 0, 0, 0, 0, 0, 0, 0, 0, 502, 512, 523, 480, 526, 506, 491, 506, 7, 0, 0, 0, 0, 0, 0, 0, 499, 507, 530, 499, 458, 535, 526, 496, 0, 0, 0, 0, 0, 0, 0, 0, 525, 485, 513, 546, 543, 509, 471, 504, 3, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4145, 4123, 4032, 4163, 4053, 4050, 4102] avg=4096.0000 max=4163 max_rank=4 min=4032 min_rank=3 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[518, 511, 512, 509, 488, 484, 504, 521, 33, 0, 0, 0, 0, 0, 0, 0, 504, 518, 505, 489, 520, 516, 504, 527, 0, 0, 0, 0, 0, 0, 0, 0, 477, 548, 533, 520, 502, 516, 478, 538, 17, 5, 0, 0, 0, 0, 0, 0, 473, 479, 520, 527, 499, 501, 515, 526, 0, 0, 0, 0, 0, 0, 0, 0, 471, 487, 491, 527, 486, 557, 497, 508, 0, 0, 0, 0, 0, 0, 0, 0, 538, 551, 520, 508, 487, 539, 518, 508, 7, 0, 0, 0, 0, 0, 0, 0, 527, 496, 470, 498, 516, 500, 526, 519, 0, 0, 0, 0, 0, 0, 0, 0, 547, 502, 513, 537, 514, 512, 537, 511, 3, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4083, 4134, 4040, 4024, 4176, 4052, 4179] avg=4096.0000 max=4179 max_rank=7 min=4024 min_rank=4 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[522, 467, 518, 522, 549, 515, 522, 542, 35, 0, 0, 0, 0, 0, 0, 0, 494, 515, 500, 501, 520, 494, 511, 528, 0, 0, 0, 0, 0, 0, 0, 0, 532, 502, 488, 491, 485, 503, 493, 513, 16, 6, 0, 0, 0, 0, 0, 0, 490, 499, 535, 498, 550, 527, 479, 509, 0, 0, 0, 0, 0, 0, 0, 0, 555, 527, 518, 483, 494, 504, 517, 552, 0, 0, 0, 0, 0, 0, 0, 0, 501, 537, 514, 527, 496, 466, 531, 509, 7, 0, 0, 0, 0, 0, 0, 0, 518, 497, 534, 484, 511, 509, 491, 498, 0, 0, 0, 0, 0, 0, 0, 0, 510, 507, 516, 497, 513, 528, 537, 503, 3, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4192, 4063, 4029, 4087, 4150, 4088, 4042, 4117] avg=4096.0000 max=4192 max_rank=0 min=4029 min_rank=2 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[503, 497, 493, 509, 509, 482, 534, 522, 35, 0, 0, 0, 0, 0, 0, 0, 516, 513, 506, 500, 525, 493, 512, 517, 0, 0, 0, 0, 0, 0, 0, 0, 502, 484, 493, 500, 523, 515, 505, 532, 19, 9, 3, 0, 0, 0, 0, 0, 525, 527, 516, 501, 532, 523, 502, 515, 0, 0, 0, 0, 0, 0, 0, 0, 493, 469, 525, 522, 505, 509, 495, 539, 0, 0, 0, 0, 0, 0, 0, 0, 483, 527, 497, 529, 498, 512, 544, 522, 12, 0, 0, 0, 0, 0, 0, 0, 540, 515, 526, 533, 519, 503, 495, 526, 0, 0, 0, 0, 0, 0, 0, 0, 467, 503, 512, 490, 496, 487, 540, 529, 8, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4082, 4085, 4141, 4057, 4124, 4157, 4038] avg=4096.0000 max=4157 max_rank=6 min=4038 min_rank=7 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16376 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[514, 465, 519, 489, 519, 505, 508, 488, 34, 0, 0, 0, 0, 0, 0, 0, 523, 505, 492, 487, 516, 535, 505, 497, 0, 0, 0, 0, 0, 0, 0, 0, 528, 487, 495, 496, 515, 495, 512, 530, 15, 6, 0, 0, 0, 0, 0, 0, 526, 516, 524, 514, 516, 540, 535, 523, 0, 0, 0, 0, 0, 0, 0, 0, 541, 505, 520, 495, 517, 528, 538, 521, 0, 0, 0, 0, 0, 0, 0, 0, 522, 512, 527, 539, 511, 488, 500, 489, 7, 0, 0, 0, 0, 0, 0, 0, 477, 505, 505, 529, 515, 511, 488, 520, 0, 0, 0, 0, 0, 0, 0, 0, 506, 501, 558, 476, 487, 495, 530, 525, 3, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4041, 4060, 4079, 4194, 4165, 4095, 4050, 4084] avg=4096.0000 max=4194 max_rank=3 min=4041 min_rank=0 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16394 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[480, 511, 514, 513, 488, 503, 523, 524, 33, 0, 0, 0, 0, 0, 0, 0, 489, 511, 518, 527, 525, 518, 538, 525, 0, 0, 0, 0, 0, 0, 0, 0, 524, 510, 490, 481, 571, 494, 555, 546, 17, 6, 0, 0, 0, 0, 0, 0, 491, 538, 513, 521, 527, 528, 487, 486, 0, 0, 0, 0, 0, 0, 0, 0, 516, 487, 499, 549, 525, 449, 515, 505, 0, 0, 0, 0, 0, 0, 0, 0, 493, 492, 518, 496, 504, 525, 505, 450, 8, 0, 0, 0, 0, 0, 0, 0, 532, 528, 522, 517, 516, 486, 533, 531, 0, 0, 0, 0, 0, 0, 0, 0, 479, 497, 513, 527, 506, 499, 533, 482, 3, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4151, 4194, 4091, 4045, 3991, 4165, 4042] avg=4096.0000 max=4194 max_rank=2 min=3991 min_rank=5 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16243 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[464, 501, 506, 497, 480, 506, 499, 501, 30, 0, 0, 0, 0, 0, 0, 0, 556, 487, 538, 551, 496, 498, 489, 489, 0, 0, 0, 0, 0, 0, 0, 0, 543, 488, 510, 505, 508, 555, 510, 482, 15, 5, 0, 0, 0, 0, 0, 0, 490, 542, 529, 499, 507, 491, 528, 514, 0, 0, 0, 0, 0, 0, 0, 0, 481, 453, 529, 517, 517, 542, 547, 513, 0, 0, 0, 0, 0, 0, 0, 0, 532, 503, 487, 478, 506, 516, 498, 524, 8, 0, 0, 0, 0, 0, 0, 0, 539, 527, 549, 508, 523, 529, 480, 508, 0, 0, 0, 0, 0, 0, 0, 0, 513, 512, 503, 533, 516, 495, 542, 525, 3, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3984, 4104, 4121, 4100, 4099, 4052, 4163, 4145] avg=4096.0000 max=4163 max_rank=6 min=3984 min_rank=0 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16459 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[511, 518, 516, 542, 525, 528, 512, 511, 35, 0, 0, 0, 0, 0, 0, 0, 466, 537, 539, 506, 534, 519, 507, 472, 0, 0, 0, 0, 0, 0, 0, 0, 496, 487, 503, 512, 466, 505, 515, 496, 17, 6, 0, 0, 0, 0, 0, 0, 512, 522, 545, 510, 522, 471, 478, 523, 0, 0, 0, 0, 0, 0, 0, 0, 507, 520, 484, 512, 518, 522, 507, 495, 0, 0, 0, 0, 0, 0, 0, 0, 519, 500, 540, 559, 493, 490, 534, 546, 8, 0, 0, 0, 0, 0, 0, 0, 524, 503, 496, 533, 513, 495, 521, 504, 0, 0, 0, 0, 0, 0, 0, 0, 494, 505, 507, 492, 484, 497, 544, 532, 3, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4198, 4080, 4003, 4083, 4065, 4189, 4089, 4061] avg=4096.0000 max=4198 max_rank=0 min=4003 min_rank=2 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16404 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 3968, 4224, 4224, 4096, 4096, 4224, 4224, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4352, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True + [2026-09-21 08:26:50] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 265.8 | throughput per GPU (TFLOP/s/GPU): 121.2 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.686510E+00 | loss scale: 1.0 | grad norm: 3.337 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[531, 551, 533, 522, 501, 470, 502, 486, 513, 506, 482, 510, 519, 527, 484, 502, 549, 506, 511, 515, 500, 501, 516, 510, 534, 459, 518, 506, 536, 530, 500, 525, 489, 509, 546, 496, 548, 518, 562, 516, 523, 501, 522, 522, 495, 488, 499, 493, 531, 491, 512, 496, 535, 526, 473, 515, 517, 506, 516, 524, 514, 516, 526, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[520, 526, 513, 531, 480, 492, 513, 501, 462, 505, 521, 525, 525, 527, 548, 481, 456, 522, 490, 508, 549, 528, 502, 486, 515, 517, 523, 505, 467, 534, 528, 490, 521, 521, 482, 527, 515, 550, 555, 524, 527, 550, 505, 514, 508, 493, 490, 480, 451, 522, 541, 527, 525, 510, 508, 505, 483, 493, 516, 513, 571, 518, 528, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[495, 470, 507, 536, 539, 496, 500, 546, 503, 526, 457, 502, 499, 524, 522, 457, 508, 520, 543, 535, 504, 493, 512, 566, 528, 507, 522, 513, 484, 472, 506, 504, 456, 498, 503, 482, 525, 523, 554, 521, 497, 529, 507, 523, 484, 496, 541, 500, 493, 511, 529, 536, 490, 519, 540, 506, 534, 514, 534, 530, 547, 484, 541, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[498, 530, 496, 489, 515, 524, 488, 515, 481, 480, 512, 523, 486, 527, 547, 530, 516, 524, 490, 507, 508, 499, 481, 510, 481, 505, 513, 536, 500, 512, 519, 509, 505, 532, 480, 553, 518, 526, 510, 471, 506, 479, 514, 551, 555, 532, 507, 512, 534, 502, 547, 521, 528, 508, 500, 481, 508, 512, 509, 517, 561, 502, 492, 544] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[537, 520, 530, 491, 505, 503, 484, 533, 498, 519, 458, 527, 528, 520, 508, 534, 519, 526, 534, 508, 489, 503, 506, 497, 540, 503, 497, 496, 517, 458, 539, 565, 551, 514, 526, 520, 516, 526, 509, 507, 493, 535, 501, 532, 524, 499, 492, 493, 550, 500, 502, 543, 504, 494, 500, 497, 517, 514, 483, 504, 526, 511, 495, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[559, 499, 508, 527, 491, 510, 516, 477, 474, 469, 502, 505, 502, 518, 504, 511, 511, 502, 520, 546, 532, 550, 532, 521, 494, 520, 504, 487, 505, 538, 494, 535, 514, 481, 505, 485, 493, 504, 516, 519, 505, 515, 507, 501, 515, 507, 505, 535, 498, 529, 520, 541, 520, 502, 518, 525, 506, 496, 518, 503, 521, 583, 508, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[493, 527, 569, 536, 522, 529, 524, 521, 516, 540, 506, 491, 476, 513, 524, 485, 536, 524, 507, 495, 536, 523, 538, 502, 509, 475, 525, 482, 494, 527, 537, 538, 523, 480, 531, 538, 463, 516, 467, 497, 493, 520, 524, 492, 535, 516, 519, 495, 496, 507, 478, 554, 471, 517, 529, 509, 534, 516, 489, 494, 540, 511, 495, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[503, 482, 499, 514, 520, 487, 484, 479, 497, 513, 548, 496, 541, 488, 506, 549, 551, 542, 505, 504, 484, 491, 522, 520, 527, 526, 526, 506, 489, 495, 564, 525, 503, 521, 486, 508, 547, 518, 485, 531, 504, 527, 478, 475, 489, 561, 546, 520, 507, 519, 511, 502, 523, 539, 526, 484, 498, 536, 475, 496, 512, 537, 504, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[531, 551, 533, 522, 501, 470, 502, 486, 5, 2, 0, 0, 0, 0, 0, 0, 513, 506, 482, 510, 519, 527, 484, 502, 21, 13, 0, 0, 0, 0, 0, 0, 549, 485, 511, 515, 500, 501, 516, 510, 0, 0, 0, 0, 0, 0, 0, 0, 534, 459, 518, 506, 536, 530, 500, 525, 3, 0, 0, 0, 0, 0, 0, 0, 489, 509, 546, 496, 548, 511, 562, 516, 0, 0, 0, 0, 0, 0, 0, 0, 523, 501, 522, 522, 495, 488, 499, 493, 5, 3, 0, 0, 0, 0, 0, 0, 531, 491, 512, 490, 535, 526, 473, 515, 0, 0, 0, 0, 0, 0, 0, 0, 517, 506, 516, 524, 496, 516, 526, 488, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4077, 4087, 4111, 4177, 4051, 4073, 4089] avg=4096.0000 max=4177 max_rank=4 min=4051 min_rank=5 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[498, 530, 496, 489, 515, 524, 488, 515, 6, 2, 0, 0, 0, 0, 0, 0, 481, 480, 512, 523, 486, 527, 547, 530, 21, 15, 0, 0, 0, 0, 0, 0, 516, 503, 490, 507, 508, 499, 481, 510, 0, 0, 0, 0, 0, 0, 0, 0, 481, 505, 513, 536, 500, 512, 519, 509, 4, 0, 0, 0, 0, 0, 0, 0, 505, 532, 480, 553, 518, 519, 510, 471, 0, 0, 0, 0, 0, 0, 0, 0, 506, 479, 514, 551, 555, 532, 507, 512, 5, 3, 0, 0, 0, 0, 0, 0, 534, 502, 547, 514, 528, 508, 500, 481, 0, 0, 0, 0, 0, 0, 0, 0, 508, 512, 509, 517, 540, 502, 492, 544, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4122, 4014, 4079, 4088, 4164, 4114, 4124] avg=4096.0000 max=4164 max_rank=5 min=4014 min_rank=2 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16490 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[495, 470, 507, 536, 539, 496, 500, 546, 6, 2, 0, 0, 0, 0, 0, 0, 503, 526, 457, 502, 499, 524, 522, 457, 21, 14, 0, 0, 0, 0, 0, 0, 508, 499, 543, 535, 504, 493, 512, 566, 0, 0, 0, 0, 0, 0, 0, 0, 528, 507, 522, 513, 484, 472, 506, 504, 4, 0, 0, 0, 0, 0, 0, 0, 456, 498, 503, 482, 525, 516, 554, 521, 0, 0, 0, 0, 0, 0, 0, 0, 497, 529, 507, 523, 484, 496, 541, 500, 5, 3, 0, 0, 0, 0, 0, 0, 493, 511, 529, 529, 490, 519, 540, 506, 0, 0, 0, 0, 0, 0, 0, 0, 534, 514, 534, 530, 527, 484, 541, 525, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4025, 4160, 4040, 4055, 4085, 4117, 4189] avg=4096.0000 max=4189 max_rank=7 min=4025 min_rank=1 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16446 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[520, 526, 513, 531, 480, 492, 513, 501, 6, 3, 0, 0, 0, 0, 0, 0, 462, 505, 521, 525, 525, 527, 548, 481, 21, 15, 0, 0, 0, 0, 0, 0, 456, 501, 490, 508, 549, 528, 502, 486, 0, 0, 0, 0, 0, 0, 0, 0, 515, 517, 523, 505, 467, 534, 528, 490, 4, 0, 0, 0, 0, 0, 0, 0, 521, 521, 482, 527, 515, 542, 555, 524, 0, 0, 0, 0, 0, 0, 0, 0, 527, 550, 505, 514, 508, 493, 490, 480, 5, 3, 0, 0, 0, 0, 0, 0, 451, 522, 541, 520, 525, 510, 508, 505, 0, 0, 0, 0, 0, 0, 0, 0, 483, 493, 516, 513, 550, 518, 528, 505, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4130, 4020, 4083, 4187, 4075, 4082, 4106] avg=4096.0000 max=4187 max_rank=4 min=4020 min_rank=2 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16450 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[537, 520, 530, 491, 505, 503, 484, 533, 10, 8, 0, 0, 0, 0, 0, 0, 498, 519, 458, 527, 528, 520, 508, 534, 26, 18, 0, 0, 0, 0, 0, 0, 519, 500, 534, 508, 489, 503, 506, 497, 0, 0, 0, 0, 0, 0, 0, 0, 540, 503, 497, 496, 517, 458, 539, 565, 7, 0, 0, 0, 0, 0, 0, 0, 551, 514, 526, 520, 516, 508, 509, 507, 0, 0, 0, 0, 0, 0, 0, 0, 493, 535, 501, 532, 524, 499, 492, 493, 10, 6, 0, 0, 0, 0, 0, 0, 550, 500, 502, 530, 504, 494, 500, 497, 0, 0, 0, 0, 0, 0, 0, 0, 517, 514, 483, 504, 498, 511, 495, 498, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4136, 4056, 4122, 4151, 4085, 4077, 4020] avg=4096.0000 max=4151 max_rank=4 min=4020 min_rank=7 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16333 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[559, 499, 508, 527, 491, 510, 516, 477, 6, 2, 0, 0, 0, 0, 0, 0, 474, 469, 502, 505, 502, 518, 504, 511, 20, 14, 0, 0, 0, 0, 0, 0, 511, 482, 520, 546, 532, 550, 532, 521, 0, 0, 0, 0, 0, 0, 0, 0, 494, 520, 504, 487, 505, 538, 494, 535, 4, 0, 0, 0, 0, 0, 0, 0, 514, 481, 505, 485, 493, 497, 516, 519, 0, 0, 0, 0, 0, 0, 0, 0, 505, 515, 507, 501, 515, 507, 505, 535, 5, 3, 0, 0, 0, 0, 0, 0, 498, 529, 520, 534, 520, 502, 518, 525, 0, 0, 0, 0, 0, 0, 0, 0, 506, 496, 518, 503, 501, 583, 508, 510, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4019, 4194, 4081, 4010, 4098, 4146, 4125] avg=4096.0000 max=4194 max_rank=2 min=4010 min_rank=4 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[493, 527, 569, 536, 522, 529, 524, 521, 6, 2, 0, 0, 0, 0, 0, 0, 516, 540, 506, 491, 476, 513, 524, 485, 21, 14, 0, 0, 0, 0, 0, 0, 536, 503, 507, 495, 536, 523, 538, 502, 0, 0, 0, 0, 0, 0, 0, 0, 509, 475, 525, 482, 494, 527, 537, 538, 4, 0, 0, 0, 0, 0, 0, 0, 523, 480, 531, 538, 463, 509, 467, 497, 0, 0, 0, 0, 0, 0, 0, 0, 493, 520, 524, 492, 535, 516, 519, 495, 5, 3, 0, 0, 0, 0, 0, 0, 496, 507, 478, 547, 471, 517, 529, 509, 0, 0, 0, 0, 0, 0, 0, 0, 534, 516, 489, 494, 520, 511, 495, 499, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4229, 4086, 4140, 4091, 4008, 4102, 4054, 4058] avg=4096.0000 max=4229 max_rank=0 min=4008 min_rank=4 imbalance_ratio=1.032471 max_load_reduction=28539 max_load_reduction_pct=0.870941 moved_to_echo=16222 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[503, 482, 499, 514, 520, 487, 484, 479, 5, 2, 0, 0, 0, 0, 0, 0, 497, 513, 548, 496, 541, 488, 506, 549, 22, 13, 0, 0, 0, 0, 0, 0, 551, 520, 505, 504, 484, 491, 522, 520, 0, 0, 0, 0, 0, 0, 0, 0, 527, 526, 526, 506, 489, 495, 564, 525, 3, 0, 0, 0, 0, 0, 0, 0, 503, 521, 486, 508, 547, 511, 485, 531, 0, 0, 0, 0, 0, 0, 0, 0, 504, 527, 478, 475, 489, 561, 546, 520, 5, 3, 0, 0, 0, 0, 0, 0, 507, 519, 511, 496, 523, 539, 526, 484, 0, 0, 0, 0, 0, 0, 0, 0, 498, 536, 475, 496, 494, 537, 504, 517, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3975, 4173, 4097, 4161, 4092, 4108, 4105, 4057] avg=4096.0000 max=4173 max_rank=1 min=3975 min_rank=0 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16362 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3968, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[519, 517, 490, 512, 522, 526, 518, 491, 491, 523, 491, 562, 516, 520, 495, 530, 507, 523, 515, 507, 496, 512, 488, 482, 518, 538, 532, 481, 515, 488, 560, 539, 502, 509, 503, 518, 488, 494, 521, 534, 553, 527, 521, 492, 515, 531, 522, 517, 511, 508, 465, 468, 507, 502, 510, 500, 521, 528, 554, 485, 498, 490, 545, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[502, 486, 529, 516, 534, 499, 560, 503, 486, 538, 507, 492, 473, 507, 502, 515, 551, 495, 505, 499, 536, 480, 469, 529, 491, 540, 490, 527, 529, 530, 500, 512, 516, 513, 513, 536, 522, 530, 490, 483, 528, 478, 501, 480, 530, 522, 506, 527, 504, 520, 519, 495, 509, 518, 528, 527, 513, 505, 512, 498, 514, 512, 531, 556] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[516, 495, 556, 518, 490, 490, 530, 513, 496, 526, 531, 502, 490, 477, 484, 504, 509, 511, 504, 495, 516, 518, 544, 538, 502, 516, 561, 488, 502, 497, 516, 534, 519, 487, 552, 553, 488, 514, 490, 499, 454, 510, 538, 521, 540, 528, 495, 514, 501, 504, 516, 535, 533, 497, 510, 494, 491, 566, 492, 526, 497, 497, 534, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[570, 520, 531, 539, 511, 492, 520, 481, 522, 491, 487, 531, 509, 503, 510, 471, 512, 572, 501, 513, 506, 528, 530, 518, 492, 496, 537, 508, 501, 528, 506, 491, 503, 519, 495, 483, 480, 500, 505, 500, 486, 498, 508, 541, 465, 511, 494, 544, 516, 492, 526, 493, 499, 556, 561, 531, 526, 545, 548, 499, 512, 503, 525, 477] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[527, 507, 517, 530, 534, 534, 520, 507, 541, 519, 501, 496, 481, 545, 540, 539, 531, 545, 476, 531, 489, 512, 494, 512, 529, 512, 534, 511, 542, 524, 506, 487, 558, 522, 498, 482, 491, 526, 502, 526, 524, 489, 467, 486, 527, 525, 503, 482, 466, 510, 546, 494, 447, 563, 521, 510, 485, 463, 527, 501, 530, 524, 512, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[471, 505, 474, 513, 527, 539, 488, 523, 480, 531, 501, 519, 496, 486, 497, 517, 521, 586, 541, 513, 494, 514, 478, 519, 498, 516, 483, 512, 513, 509, 525, 532, 495, 521, 497, 481, 511, 496, 524, 537, 529, 543, 492, 495, 498, 547, 504, 552, 542, 497, 448, 522, 518, 532, 503, 504, 529, 538, 491, 494, 544, 559, 506, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[494, 484, 502, 527, 537, 490, 537, 529, 507, 543, 489, 502, 494, 548, 501, 538, 519, 511, 522, 521, 537, 488, 504, 485, 530, 556, 537, 472, 483, 521, 511, 475, 488, 525, 560, 534, 499, 527, 496, 510, 488, 509, 489, 538, 543, 473, 497, 551, 520, 522, 526, 515, 492, 480, 520, 502, 514, 510, 496, 519, 530, 521, 487, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[522, 489, 502, 517, 541, 503, 519, 508, 504, 506, 507, 510, 546, 516, 511, 529, 514, 496, 529, 482, 498, 524, 488, 534, 557, 484, 532, 525, 502, 483, 483, 543, 504, 531, 481, 501, 536, 522, 504, 531, 552, 510, 522, 517, 505, 521, 492, 518, 484, 500, 512, 556, 473, 508, 513, 483, 470, 542, 505, 520, 510, 500, 532, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[494, 484, 502, 527, 519, 490, 537, 529, 0, 0, 0, 0, 0, 0, 0, 0, 507, 543, 489, 502, 494, 548, 501, 538, 10, 7, 0, 0, 0, 0, 0, 0, 519, 506, 522, 521, 537, 488, 504, 485, 0, 0, 0, 0, 0, 0, 0, 0, 530, 556, 523, 472, 483, 521, 511, 475, 0, 0, 0, 0, 0, 0, 0, 0, 488, 525, 560, 534, 499, 527, 496, 510, 5, 3, 2, 0, 0, 0, 0, 0, 488, 509, 489, 538, 543, 473, 497, 549, 0, 0, 0, 0, 0, 0, 0, 0, 520, 522, 526, 515, 492, 480, 520, 502, 18, 4, 0, 0, 0, 0, 0, 0, 514, 500, 496, 519, 530, 521, 487, 493, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4139, 4082, 4071, 4149, 4086, 4099, 4060] avg=4096.0000 max=4149 max_rank=4 min=4060 min_rank=7 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16394 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[570, 520, 531, 539, 494, 492, 520, 481, 0, 0, 0, 0, 0, 0, 0, 0, 522, 491, 487, 531, 509, 503, 510, 471, 10, 7, 0, 0, 0, 0, 0, 0, 512, 566, 501, 513, 506, 528, 530, 518, 0, 0, 0, 0, 0, 0, 0, 0, 492, 496, 523, 508, 501, 528, 506, 491, 0, 0, 0, 0, 0, 0, 0, 0, 503, 519, 495, 483, 480, 500, 505, 500, 6, 3, 2, 0, 0, 0, 0, 0, 486, 498, 508, 541, 465, 511, 494, 542, 0, 0, 0, 0, 0, 0, 0, 0, 516, 492, 526, 493, 499, 556, 561, 531, 17, 4, 0, 0, 0, 0, 0, 0, 526, 535, 548, 499, 512, 503, 525, 477, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4147, 4041, 4174, 4045, 3996, 4045, 4195, 4125] avg=4096.0000 max=4195 max_rank=6 min=3996 min_rank=4 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16361 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[471, 505, 474, 513, 509, 539, 488, 523, 0, 0, 0, 0, 0, 0, 0, 0, 480, 531, 501, 519, 496, 486, 497, 517, 9, 7, 0, 0, 0, 0, 0, 0, 521, 580, 541, 513, 494, 514, 478, 519, 0, 0, 0, 0, 0, 0, 0, 0, 498, 516, 470, 512, 513, 509, 525, 532, 0, 0, 0, 0, 0, 0, 0, 0, 495, 521, 497, 481, 511, 496, 524, 537, 6, 3, 2, 0, 0, 0, 0, 0, 529, 543, 492, 495, 498, 547, 504, 550, 0, 0, 0, 0, 0, 0, 0, 0, 542, 497, 448, 522, 518, 532, 503, 504, 18, 4, 0, 0, 0, 0, 0, 0, 529, 528, 491, 494, 544, 559, 506, 498, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4022, 4043, 4160, 4075, 4073, 4158, 4088, 4149] avg=4096.0000 max=4160 max_rank=2 min=4022 min_rank=0 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16468 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[519, 517, 490, 512, 504, 526, 518, 491, 0, 0, 0, 0, 0, 0, 0, 0, 491, 523, 491, 562, 516, 520, 495, 530, 10, 7, 0, 0, 0, 0, 0, 0, 507, 517, 515, 507, 496, 512, 488, 482, 0, 0, 0, 0, 0, 0, 0, 0, 518, 538, 518, 481, 515, 488, 560, 539, 0, 0, 0, 0, 0, 0, 0, 0, 502, 509, 503, 518, 488, 494, 521, 534, 6, 3, 2, 0, 0, 0, 0, 0, 553, 527, 521, 492, 515, 531, 522, 515, 0, 0, 0, 0, 0, 0, 0, 0, 511, 508, 465, 468, 507, 502, 510, 500, 18, 4, 0, 0, 0, 0, 0, 0, 521, 518, 554, 485, 498, 490, 545, 505, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4145, 4024, 4157, 4080, 4176, 3993, 4116] avg=4096.0000 max=4176 max_rank=5 min=3993 min_rank=6 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16365 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[527, 507, 517, 530, 516, 534, 520, 507, 0, 0, 0, 0, 0, 0, 0, 0, 541, 519, 501, 496, 481, 545, 540, 539, 10, 6, 0, 0, 0, 0, 0, 0, 531, 539, 476, 531, 489, 512, 494, 512, 0, 0, 0, 0, 0, 0, 0, 0, 529, 512, 520, 511, 542, 524, 506, 487, 0, 0, 0, 0, 0, 0, 0, 0, 558, 522, 498, 482, 491, 526, 502, 526, 6, 2, 1, 0, 0, 0, 0, 0, 524, 489, 467, 486, 527, 525, 503, 481, 0, 0, 0, 0, 0, 0, 0, 0, 466, 510, 546, 494, 447, 563, 521, 510, 18, 4, 0, 0, 0, 0, 0, 0, 485, 455, 527, 501, 530, 524, 512, 488, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4158, 4178, 4084, 4131, 4114, 4002, 4079, 4022] avg=4096.0000 max=4178 max_rank=1 min=4002 min_rank=5 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16217 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[522, 489, 502, 517, 523, 503, 519, 508, 0, 0, 0, 0, 0, 0, 0, 0, 504, 506, 507, 510, 546, 516, 511, 529, 10, 7, 0, 0, 0, 0, 0, 0, 514, 491, 529, 482, 498, 524, 488, 534, 0, 0, 0, 0, 0, 0, 0, 0, 557, 484, 518, 525, 502, 483, 483, 543, 0, 0, 0, 0, 0, 0, 0, 0, 504, 531, 481, 501, 536, 522, 504, 531, 5, 3, 2, 0, 0, 0, 0, 0, 552, 510, 522, 517, 505, 521, 492, 516, 0, 0, 0, 0, 0, 0, 0, 0, 484, 500, 512, 556, 473, 508, 513, 483, 18, 4, 0, 0, 0, 0, 0, 0, 470, 532, 505, 520, 510, 500, 532, 509, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4146, 4060, 4095, 4120, 4135, 4051, 4078] avg=4096.0000 max=4146 max_rank=1 min=4051 min_rank=6 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[502, 486, 529, 516, 512, 499, 560, 503, 0, 0, 0, 0, 0, 0, 0, 0, 486, 538, 507, 492, 473, 507, 502, 515, 15, 12, 0, 0, 0, 0, 0, 0, 551, 485, 505, 499, 536, 480, 469, 529, 0, 0, 0, 0, 0, 0, 0, 0, 491, 540, 465, 527, 529, 530, 500, 512, 0, 0, 0, 0, 0, 0, 0, 0, 516, 513, 513, 536, 522, 530, 490, 483, 10, 7, 4, 0, 0, 0, 0, 0, 528, 478, 501, 480, 530, 522, 506, 523, 0, 0, 0, 0, 0, 0, 0, 0, 504, 520, 519, 495, 509, 518, 528, 527, 22, 10, 0, 0, 0, 0, 0, 0, 513, 486, 512, 498, 514, 512, 531, 556, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4047, 4054, 4094, 4124, 4068, 4152, 4122] avg=4096.0000 max=4152 max_rank=6 min=4047 min_rank=1 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16466 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[516, 495, 556, 518, 474, 490, 530, 513, 0, 0, 0, 0, 0, 0, 0, 0, 496, 526, 531, 502, 490, 477, 484, 504, 11, 8, 0, 0, 0, 0, 0, 0, 509, 506, 504, 495, 516, 518, 544, 538, 0, 0, 0, 0, 0, 0, 0, 0, 502, 516, 545, 488, 502, 497, 516, 534, 0, 0, 0, 0, 0, 0, 0, 0, 519, 487, 552, 553, 488, 514, 490, 499, 5, 3, 2, 0, 0, 0, 0, 0, 454, 510, 538, 521, 540, 528, 495, 512, 0, 0, 0, 0, 0, 0, 0, 0, 501, 504, 516, 535, 533, 497, 510, 494, 16, 5, 0, 0, 0, 0, 0, 0, 491, 555, 492, 526, 497, 497, 534, 504, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4029, 4130, 4100, 4112, 4098, 4111, 4096] avg=4096.0000 max=4130 max_rank=2 min=4029 min_rank=1 imbalance_ratio=1.008301 max_load_reduction=28638 max_load_reduction_pct=0.873962 moved_to_echo=16417 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[500, 483, 505, 544, 506, 497, 522, 502, 491, 519, 523, 507, 502, 499, 529, 559, 525, 477, 539, 508, 529, 514, 481, 517, 463, 482, 496, 510, 493, 519, 530, 491, 501, 528, 558, 557, 497, 500, 523, 476, 511, 513, 503, 494, 508, 494, 509, 559, 532, 520, 509, 527, 513, 505, 537, 503, 534, 497, 486, 525, 568, 484, 506, 529] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[534, 502, 522, 526, 505, 514, 521, 494, 505, 514, 547, 508, 561, 483, 484, 511, 469, 502, 496, 507, 494, 489, 488, 464, 512, 477, 560, 517, 527, 533, 527, 504, 520, 516, 488, 499, 544, 499, 528, 508, 549, 525, 486, 517, 476, 520, 561, 511, 487, 527, 542, 517, 501, 480, 509, 497, 502, 543, 538, 504, 538, 535, 529, 475] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[527, 457, 531, 481, 528, 522, 515, 507, 551, 510, 502, 487, 514, 492, 525, 487, 539, 537, 532, 524, 550, 526, 515, 492, 532, 512, 513, 492, 513, 536, 523, 507, 508, 474, 515, 514, 482, 531, 519, 512, 537, 505, 502, 522, 456, 572, 530, 490, 511, 476, 505, 505, 497, 497, 550, 526, 508, 517, 494, 503, 529, 486, 536, 480] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[507, 495, 540, 482, 475, 511, 524, 490, 511, 545, 516, 488, 494, 502, 484, 519, 497, 573, 490, 508, 488, 476, 516, 489, 496, 486, 526, 551, 543, 475, 483, 471, 500, 481, 527, 500, 500, 491, 518, 471, 512, 535, 522, 509, 533, 551, 508, 547, 475, 554, 559, 519, 506, 513, 556, 535, 533, 542, 515, 527, 521, 522, 532, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[514, 510, 546, 498, 514, 506, 496, 547, 553, 533, 525, 496, 495, 521, 527, 490, 560, 496, 478, 519, 523, 503, 468, 511, 519, 536, 495, 510, 524, 510, 502, 500, 505, 562, 526, 497, 496, 534, 498, 526, 552, 486, 537, 497, 476, 515, 518, 477, 482, 470, 523, 528, 544, 520, 529, 516, 531, 504, 484, 485, 496, 503, 521, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[510, 543, 466, 534, 555, 541, 513, 488, 509, 512, 525, 508, 513, 507, 513, 518, 477, 539, 503, 484, 492, 500, 459, 506, 512, 507, 518, 529, 465, 465, 504, 489, 515, 544, 528, 545, 517, 520, 526, 532, 542, 472, 504, 511, 493, 470, 487, 530, 452, 561, 524, 525, 531, 516, 524, 553, 550, 508, 550, 498, 485, 556, 511, 484] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[506, 533, 540, 551, 532, 529, 527, 489, 488, 488, 464, 526, 515, 530, 467, 503, 508, 503, 505, 546, 528, 496, 516, 531, 508, 511, 534, 487, 507, 508, 519, 524, 486, 519, 511, 527, 509, 483, 503, 467, 522, 503, 504, 535, 512, 485, 551, 532, 491, 525, 496, 534, 514, 472, 508, 508, 505, 542, 550, 522, 492, 521, 502, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[492, 530, 505, 537, 514, 486, 494, 480, 518, 503, 512, 504, 513, 530, 497, 535, 526, 508, 458, 511, 534, 475, 513, 513, 500, 506, 542, 533, 529, 530, 491, 527, 534, 533, 525, 505, 480, 514, 517, 515, 477, 530, 537, 504, 528, 505, 526, 501, 499, 519, 536, 513, 470, 502, 531, 505, 534, 497, 520, 513, 561, 485, 486, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[527, 457, 528, 481, 528, 522, 515, 507, 0, 0, 0, 0, 0, 0, 0, 0, 551, 510, 502, 487, 514, 492, 525, 487, 3, 2, 1, 0, 0, 0, 0, 0, 539, 537, 532, 524, 550, 526, 515, 492, 25, 15, 0, 0, 0, 0, 0, 0, 532, 512, 513, 492, 513, 536, 523, 507, 12, 3, 0, 0, 0, 0, 0, 0, 508, 474, 514, 514, 482, 531, 519, 512, 0, 0, 0, 0, 0, 0, 0, 0, 523, 505, 502, 522, 456, 572, 530, 490, 0, 0, 0, 0, 0, 0, 0, 0, 511, 476, 505, 505, 497, 497, 532, 526, 0, 0, 0, 0, 0, 0, 0, 0, 483, 517, 494, 503, 529, 486, 536, 480, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4065, 4074, 4255, 4143, 4054, 4100, 4049, 4028] avg=4096.0000 max=4255 max_rank=2 min=4028 min_rank=7 imbalance_ratio=1.038818 max_load_reduction=28513 max_load_reduction_pct=0.870148 moved_to_echo=16231 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[500, 483, 502, 544, 506, 497, 522, 502, 0, 0, 0, 0, 0, 0, 0, 0, 491, 519, 523, 507, 502, 499, 529, 559, 3, 2, 2, 0, 0, 0, 0, 0, 525, 477, 539, 508, 529, 514, 481, 517, 26, 14, 0, 0, 0, 0, 0, 0, 463, 482, 496, 510, 493, 519, 530, 491, 12, 3, 0, 0, 0, 0, 0, 0, 501, 528, 556, 557, 497, 500, 523, 476, 0, 0, 0, 0, 0, 0, 0, 0, 497, 513, 503, 494, 508, 494, 509, 559, 0, 0, 0, 0, 0, 0, 0, 0, 532, 520, 509, 527, 513, 505, 520, 503, 0, 0, 0, 0, 0, 0, 0, 0, 508, 497, 486, 525, 568, 484, 506, 529, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4056, 4136, 4130, 3999, 4138, 4077, 4129, 4103] avg=4096.0000 max=4138 max_rank=4 min=3999 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16447 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[507, 495, 537, 482, 475, 511, 524, 490, 0, 0, 0, 0, 0, 0, 0, 0, 511, 545, 516, 488, 494, 502, 484, 519, 3, 2, 2, 0, 0, 0, 0, 0, 497, 573, 490, 508, 488, 476, 516, 489, 26, 15, 0, 0, 0, 0, 0, 0, 496, 486, 526, 551, 543, 475, 483, 471, 12, 3, 0, 0, 0, 0, 0, 0, 500, 481, 525, 500, 500, 491, 518, 471, 0, 0, 0, 0, 0, 0, 0, 0, 498, 535, 522, 509, 533, 551, 508, 547, 0, 0, 0, 0, 0, 0, 0, 0, 475, 554, 559, 519, 506, 513, 538, 535, 0, 0, 0, 0, 0, 0, 0, 0, 507, 542, 515, 527, 521, 522, 532, 503, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4021, 4066, 4078, 4046, 3986, 4203, 4199, 4169] avg=4096.0000 max=4203 max_rank=5 min=3986 min_rank=4 imbalance_ratio=1.026123 max_load_reduction=28565 max_load_reduction_pct=0.871735 moved_to_echo=16557 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[534, 502, 519, 526, 505, 514, 521, 494, 0, 0, 0, 0, 0, 0, 0, 0, 505, 514, 547, 508, 561, 483, 484, 511, 3, 2, 1, 0, 0, 0, 0, 0, 469, 502, 496, 507, 494, 489, 488, 464, 25, 13, 0, 0, 0, 0, 0, 0, 512, 477, 560, 517, 527, 533, 527, 504, 13, 3, 0, 0, 0, 0, 0, 0, 520, 516, 487, 499, 544, 499, 528, 508, 0, 0, 0, 0, 0, 0, 0, 0, 534, 525, 486, 517, 476, 520, 561, 511, 0, 0, 0, 0, 0, 0, 0, 0, 487, 527, 542, 517, 501, 480, 493, 497, 0, 0, 0, 0, 0, 0, 0, 0, 477, 543, 538, 504, 538, 535, 529, 475, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4119, 3947, 4173, 4101, 4130, 4044, 4139] avg=4096.0000 max=4173 max_rank=3 min=3947 min_rank=2 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16414 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[510, 543, 460, 534, 555, 541, 513, 488, 0, 0, 0, 0, 0, 0, 0, 0, 509, 512, 525, 508, 513, 507, 513, 518, 6, 4, 5, 0, 0, 0, 0, 0, 477, 539, 503, 484, 492, 500, 459, 506, 33, 18, 0, 0, 0, 0, 0, 0, 512, 507, 518, 529, 465, 465, 504, 489, 15, 6, 0, 0, 0, 0, 0, 0, 515, 544, 523, 545, 517, 520, 526, 532, 0, 0, 0, 0, 0, 0, 0, 0, 523, 472, 504, 511, 493, 470, 487, 530, 0, 0, 0, 0, 0, 0, 0, 0, 452, 561, 524, 525, 531, 516, 500, 553, 0, 0, 0, 0, 0, 0, 0, 0, 517, 508, 550, 498, 485, 556, 511, 484, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4144, 4120, 4011, 4010, 4222, 3990, 4162, 4109] avg=4096.0000 max=4222 max_rank=4 min=3990 min_rank=5 imbalance_ratio=1.030762 max_load_reduction=28546 max_load_reduction_pct=0.871155 moved_to_echo=16483 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[506, 533, 537, 551, 532, 529, 527, 489, 0, 0, 0, 0, 0, 0, 0, 0, 488, 488, 464, 526, 515, 530, 467, 503, 3, 2, 1, 0, 0, 0, 0, 0, 508, 503, 505, 546, 528, 496, 516, 531, 25, 13, 0, 0, 0, 0, 0, 0, 508, 511, 534, 487, 507, 508, 519, 524, 12, 3, 0, 0, 0, 0, 0, 0, 486, 519, 510, 527, 509, 483, 503, 467, 0, 0, 0, 0, 0, 0, 0, 0, 508, 503, 504, 535, 512, 485, 551, 532, 0, 0, 0, 0, 0, 0, 0, 0, 491, 525, 496, 534, 514, 472, 492, 508, 0, 0, 0, 0, 0, 0, 0, 0, 480, 542, 550, 522, 492, 521, 502, 518, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4204, 3987, 4171, 4113, 4004, 4130, 4032, 4127] avg=4096.0000 max=4204 max_rank=0 min=3987 min_rank=1 imbalance_ratio=1.026367 max_load_reduction=28564 max_load_reduction_pct=0.871704 moved_to_echo=16293 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[492, 530, 502, 537, 514, 486, 494, 480, 0, 0, 0, 0, 0, 0, 0, 0, 518, 503, 512, 504, 513, 530, 497, 535, 3, 2, 2, 0, 0, 0, 0, 0, 526, 508, 458, 511, 534, 475, 513, 513, 26, 14, 0, 0, 0, 0, 0, 0, 500, 506, 542, 533, 529, 530, 491, 527, 11, 3, 0, 0, 0, 0, 0, 0, 534, 533, 523, 505, 480, 514, 517, 515, 0, 0, 0, 0, 0, 0, 0, 0, 464, 530, 537, 504, 528, 505, 526, 501, 0, 0, 0, 0, 0, 0, 0, 0, 499, 519, 536, 513, 470, 502, 514, 505, 0, 0, 0, 0, 0, 0, 0, 0, 508, 497, 520, 513, 561, 485, 486, 520, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4035, 4119, 4078, 4172, 4121, 4095, 4058, 4090] avg=4096.0000 max=4172 max_rank=3 min=4035 min_rank=0 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16364 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[514, 510, 543, 498, 514, 506, 496, 547, 0, 0, 0, 0, 0, 0, 0, 0, 553, 533, 525, 496, 495, 521, 527, 490, 3, 2, 2, 0, 0, 0, 0, 0, 560, 496, 478, 519, 523, 503, 468, 511, 26, 14, 0, 0, 0, 0, 0, 0, 519, 536, 495, 510, 524, 510, 502, 500, 13, 3, 0, 0, 0, 0, 0, 0, 505, 562, 524, 497, 496, 534, 498, 526, 0, 0, 0, 0, 0, 0, 0, 0, 537, 486, 537, 497, 476, 515, 518, 477, 0, 0, 0, 0, 0, 0, 0, 0, 482, 470, 523, 528, 544, 520, 512, 516, 0, 0, 0, 0, 0, 0, 0, 0, 505, 504, 484, 485, 496, 503, 521, 505, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4147, 4098, 4112, 4142, 4043, 4095, 4003] avg=4096.0000 max=4147 max_rank=1 min=4003 min_rank=7 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16283 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3968, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 3968, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33792 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[520, 497, 504, 497, 552, 531, 475, 526, 532, 512, 510, 555, 494, 493, 515, 511, 524, 512, 529, 504, 523, 493, 499, 519, 514, 482, 502, 547, 507, 512, 513, 511, 502, 526, 542, 494, 509, 490, 501, 520, 517, 515, 537, 489, 516, 510, 503, 543, 506, 501, 526, 487, 471, 500, 520, 494, 530, 514, 544, 495, 498, 533, 511, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[501, 493, 511, 500, 517, 546, 497, 494, 534, 559, 516, 522, 530, 499, 481, 509, 520, 529, 506, 472, 530, 497, 500, 497, 489, 510, 525, 492, 517, 475, 505, 512, 509, 495, 536, 514, 498, 526, 520, 515, 494, 503, 501, 527, 498, 511, 495, 521, 522, 503, 520, 521, 516, 491, 541, 510, 547, 484, 547, 542, 515, 547, 531, 483] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[516, 510, 556, 498, 492, 501, 502, 540, 478, 490, 519, 507, 504, 529, 526, 487, 523, 470, 508, 485, 543, 520, 511, 497, 519, 455, 514, 530, 525, 498, 536, 537, 519, 508, 533, 491, 477, 518, 510, 538, 500, 487, 499, 503, 525, 513, 488, 541, 507, 505, 495, 559, 495, 516, 554, 505, 513, 510, 505, 542, 532, 498, 504, 552] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[535, 501, 511, 513, 498, 550, 514, 512, 487, 530, 507, 529, 477, 490, 485, 562, 518, 485, 500, 487, 515, 508, 511, 512, 497, 530, 477, 538, 521, 527, 513, 469, 514, 498, 493, 538, 521, 516, 509, 519, 545, 494, 466, 541, 533, 523, 475, 511, 511, 499, 533, 532, 498, 475, 521, 525, 525, 518, 512, 527, 473, 527, 542, 545] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[478, 527, 507, 524, 518, 484, 538, 524, 532, 526, 516, 503, 525, 499, 525, 517, 481, 521, 516, 546, 528, 500, 512, 489, 501, 508, 531, 501, 515, 495, 503, 530, 496, 536, 511, 520, 486, 530, 460, 529, 512, 491, 522, 482, 506, 479, 500, 494, 530, 543, 504, 498, 504, 492, 495, 517, 542, 513, 516, 539, 545, 534, 530, 492] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[513, 511, 489, 532, 507, 518, 533, 477, 532, 505, 505, 499, 518, 519, 500, 500, 509, 512, 526, 533, 562, 508, 509, 545, 530, 494, 519, 523, 516, 518, 492, 531, 500, 511, 511, 533, 494, 528, 489, 503, 524, 496, 494, 485, 541, 484, 491, 536, 558, 509, 525, 515, 503, 514, 504, 521, 504, 514, 463, 523, 502, 524, 501, 483] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[538, 516, 497, 525, 506, 555, 495, 527, 499, 508, 512, 538, 498, 497, 512, 495, 506, 512, 496, 470, 549, 487, 505, 519, 511, 508, 517, 522, 521, 493, 519, 511, 485, 509, 487, 519, 478, 496, 491, 489, 549, 491, 488, 497, 492, 526, 506, 498, 515, 543, 552, 528, 516, 539, 504, 519, 527, 513, 541, 508, 495, 488, 556, 559] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[528, 471, 521, 551, 541, 533, 523, 507, 553, 561, 516, 503, 520, 472, 478, 483, 521, 512, 531, 516, 480, 503, 478, 505, 504, 480, 489, 508, 542, 493, 527, 518, 487, 550, 529, 494, 516, 495, 535, 512, 522, 484, 518, 501, 522, 494, 534, 540, 521, 533, 513, 521, 487, 555, 500, 506, 493, 533, 510, 525, 495, 494, 494, 487] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[501, 493, 511, 500, 517, 528, 497, 494, 0, 0, 0, 0, 0, 0, 0, 0, 534, 559, 516, 522, 530, 499, 481, 509, 2, 0, 0, 0, 0, 0, 0, 0, 520, 529, 506, 472, 530, 497, 500, 497, 12, 4, 0, 0, 0, 0, 0, 0, 489, 510, 525, 492, 517, 475, 505, 512, 12, 0, 0, 0, 0, 0, 0, 0, 509, 495, 536, 514, 498, 526, 520, 515, 10, 9, 0, 0, 0, 0, 0, 0, 494, 503, 501, 527, 498, 511, 495, 521, 31, 0, 0, 0, 0, 0, 0, 0, 501, 503, 520, 521, 516, 491, 541, 510, 0, 0, 0, 0, 0, 0, 0, 0, 547, 484, 547, 501, 515, 547, 531, 483, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4041, 4152, 4067, 4037, 4132, 4081, 4103, 4155] avg=4096.0000 max=4155 max_rank=7 min=4037 min_rank=3 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16471 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[516, 510, 556, 498, 492, 484, 502, 540, 0, 0, 0, 0, 0, 0, 0, 0, 478, 490, 519, 507, 504, 529, 526, 487, 2, 0, 0, 0, 0, 0, 0, 0, 523, 470, 508, 485, 543, 520, 511, 497, 12, 4, 0, 0, 0, 0, 0, 0, 519, 455, 514, 530, 525, 498, 536, 537, 11, 0, 0, 0, 0, 0, 0, 0, 519, 508, 533, 491, 477, 518, 510, 538, 10, 9, 0, 0, 0, 0, 0, 0, 500, 487, 499, 503, 525, 513, 488, 541, 31, 0, 0, 0, 0, 0, 0, 0, 486, 505, 495, 559, 495, 516, 554, 505, 0, 0, 0, 0, 0, 0, 0, 0, 513, 510, 505, 501, 532, 498, 504, 552, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4042, 4073, 4125, 4113, 4087, 4115, 4115] avg=4096.0000 max=4125 max_rank=3 min=4042 min_rank=1 imbalance_ratio=1.007080 max_load_reduction=28643 max_load_reduction_pct=0.874115 moved_to_echo=16430 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[535, 501, 511, 513, 498, 532, 514, 512, 0, 0, 0, 0, 0, 0, 0, 0, 487, 530, 507, 529, 477, 490, 485, 562, 2, 0, 0, 0, 0, 0, 0, 0, 518, 485, 500, 487, 515, 508, 511, 512, 12, 4, 0, 0, 0, 0, 0, 0, 497, 530, 477, 538, 521, 527, 513, 469, 12, 0, 0, 0, 0, 0, 0, 0, 514, 498, 493, 538, 521, 516, 509, 519, 10, 9, 0, 0, 0, 0, 0, 0, 545, 494, 466, 541, 533, 523, 475, 511, 30, 0, 0, 0, 0, 0, 0, 0, 490, 499, 533, 532, 498, 475, 521, 525, 0, 0, 0, 0, 0, 0, 0, 0, 525, 518, 512, 487, 473, 527, 542, 545, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4069, 4052, 4084, 4127, 4118, 4073, 4129] avg=4096.0000 max=4129 max_rank=7 min=4052 min_rank=2 imbalance_ratio=1.008057 max_load_reduction=28639 max_load_reduction_pct=0.873993 moved_to_echo=16447 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[520, 497, 504, 497, 552, 513, 475, 526, 0, 0, 0, 0, 0, 0, 0, 0, 532, 512, 510, 555, 494, 493, 515, 511, 2, 0, 0, 0, 0, 0, 0, 0, 524, 512, 529, 504, 523, 493, 499, 519, 12, 4, 0, 0, 0, 0, 0, 0, 514, 482, 502, 547, 507, 512, 513, 511, 12, 0, 0, 0, 0, 0, 0, 0, 502, 526, 542, 494, 509, 490, 501, 520, 10, 9, 0, 0, 0, 0, 0, 0, 517, 515, 537, 489, 516, 510, 503, 543, 28, 0, 0, 0, 0, 0, 0, 0, 485, 501, 526, 487, 471, 500, 520, 494, 0, 0, 0, 0, 0, 0, 0, 0, 530, 514, 544, 457, 498, 533, 511, 509, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4124, 4119, 4100, 4103, 4158, 3984, 4096] avg=4096.0000 max=4158 max_rank=5 min=3984 min_rank=6 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16341 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[538, 516, 497, 525, 506, 522, 495, 527, 0, 0, 0, 0, 0, 0, 0, 0, 499, 508, 512, 538, 498, 497, 512, 495, 9, 0, 0, 0, 0, 0, 0, 0, 506, 512, 496, 470, 549, 487, 505, 519, 15, 7, 0, 0, 0, 0, 0, 0, 511, 508, 517, 522, 521, 493, 519, 511, 17, 0, 0, 0, 0, 0, 0, 0, 485, 509, 487, 519, 478, 496, 491, 489, 15, 11, 0, 0, 0, 0, 0, 0, 549, 491, 488, 497, 492, 526, 506, 498, 34, 0, 0, 0, 0, 0, 0, 0, 489, 543, 552, 528, 516, 539, 504, 519, 0, 0, 0, 0, 0, 0, 0, 0, 527, 513, 541, 459, 495, 488, 556, 559, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4068, 4066, 4119, 3980, 4081, 4190, 4138] avg=4096.0000 max=4190 max_rank=6 min=3980 min_rank=4 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[478, 527, 507, 524, 518, 468, 538, 524, 0, 0, 0, 0, 0, 0, 0, 0, 532, 526, 516, 503, 525, 499, 525, 517, 2, 0, 0, 0, 0, 0, 0, 0, 481, 521, 516, 546, 528, 500, 512, 489, 12, 3, 0, 0, 0, 0, 0, 0, 501, 508, 531, 501, 515, 495, 503, 530, 11, 0, 0, 0, 0, 0, 0, 0, 496, 536, 511, 520, 486, 530, 460, 529, 10, 9, 0, 0, 0, 0, 0, 0, 512, 491, 522, 482, 506, 479, 500, 494, 31, 0, 0, 0, 0, 0, 0, 0, 509, 543, 504, 498, 504, 492, 495, 517, 0, 0, 0, 0, 0, 0, 0, 0, 542, 513, 516, 498, 545, 534, 530, 492, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4145, 4108, 4095, 4087, 4017, 4062, 4170] avg=4096.0000 max=4170 max_rank=7 min=4017 min_rank=5 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16336 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[513, 511, 489, 532, 507, 500, 533, 477, 0, 0, 0, 0, 0, 0, 0, 0, 532, 505, 505, 499, 518, 519, 500, 500, 2, 0, 0, 0, 0, 0, 0, 0, 509, 512, 526, 533, 562, 508, 509, 545, 13, 4, 0, 0, 0, 0, 0, 0, 530, 494, 519, 523, 516, 518, 492, 531, 12, 0, 0, 0, 0, 0, 0, 0, 500, 511, 511, 533, 494, 528, 489, 503, 10, 10, 0, 0, 0, 0, 0, 0, 524, 496, 494, 485, 541, 484, 491, 536, 30, 0, 0, 0, 0, 0, 0, 0, 535, 509, 525, 515, 503, 514, 504, 521, 0, 0, 0, 0, 0, 0, 0, 0, 504, 514, 463, 483, 502, 524, 501, 483, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4062, 4080, 4221, 4135, 4089, 4081, 4126, 3974] avg=4096.0000 max=4221 max_rank=2 min=3974 min_rank=7 imbalance_ratio=1.030518 max_load_reduction=28547 max_load_reduction_pct=0.871185 moved_to_echo=16270 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[528, 471, 521, 551, 541, 515, 523, 507, 0, 0, 0, 0, 0, 0, 0, 0, 553, 561, 516, 503, 520, 472, 478, 483, 2, 0, 0, 0, 0, 0, 0, 0, 521, 512, 531, 516, 480, 503, 478, 505, 12, 4, 0, 0, 0, 0, 0, 0, 504, 480, 489, 508, 542, 493, 527, 518, 12, 0, 0, 0, 0, 0, 0, 0, 487, 550, 529, 494, 516, 495, 535, 512, 10, 9, 0, 0, 0, 0, 0, 0, 522, 484, 518, 501, 522, 494, 534, 540, 30, 0, 0, 0, 0, 0, 0, 0, 500, 533, 513, 521, 487, 555, 500, 506, 0, 0, 0, 0, 0, 0, 0, 0, 493, 533, 510, 485, 495, 494, 494, 487, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4157, 4088, 4062, 4073, 4137, 4145, 4115, 3991] avg=4096.0000 max=4157 max_rank=0 min=3991 min_rank=7 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4096, 4352, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 3968, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 3968, 4096, 4096, 4224, 4096, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 3968, 4096, 4224, 4224, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[530, 462, 501, 529, 536, 518, 523, 490, 482, 476, 494, 514, 497, 534, 482, 515, 530, 493, 526, 536, 515, 502, 538, 507, 487, 522, 497, 543, 474, 533, 493, 505, 533, 507, 527, 509, 514, 512, 545, 513, 512, 506, 503, 486, 511, 500, 523, 549, 500, 522, 517, 490, 510, 552, 543, 535, 524, 502, 503, 499, 533, 493, 516, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[506, 525, 488, 508, 513, 496, 515, 518, 488, 542, 490, 498, 524, 541, 498, 541, 523, 491, 508, 545, 501, 505, 512, 506, 518, 550, 510, 515, 526, 546, 490, 505, 595, 465, 520, 506, 490, 516, 489, 490, 550, 493, 517, 541, 490, 512, 519, 497, 481, 515, 501, 517, 491, 504, 496, 502, 517, 539, 523, 517, 524, 496, 474, 539] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[530, 527, 467, 528, 526, 503, 512, 539, 492, 517, 497, 526, 475, 549, 535, 536, 534, 502, 521, 499, 499, 529, 495, 504, 502, 490, 545, 501, 501, 469, 515, 490, 467, 538, 519, 510, 511, 524, 536, 508, 525, 506, 540, 512, 491, 500, 504, 525, 511, 514, 512, 491, 502, 521, 518, 478, 517, 510, 536, 544, 535, 491, 496, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[539, 541, 473, 526, 510, 497, 521, 491, 501, 482, 494, 498, 512, 472, 537, 490, 545, 519, 554, 497, 533, 589, 520, 536, 487, 509, 494, 540, 535, 472, 515, 496, 527, 529, 525, 506, 500, 520, 476, 489, 518, 493, 510, 504, 507, 518, 529, 510, 518, 533, 543, 504, 553, 472, 515, 487, 500, 500, 529, 492, 510, 519, 501, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[516, 492, 473, 504, 514, 478, 515, 507, 497, 532, 502, 517, 522, 538, 507, 550, 484, 546, 527, 539, 512, 486, 474, 498, 526, 521, 509, 541, 482, 548, 515, 553, 523, 473, 511, 507, 499, 469, 541, 542, 539, 528, 474, 535, 517, 522, 488, 494, 519, 535, 517, 502, 547, 508, 486, 470, 510, 539, 507, 485, 515, 517, 480, 544] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[513, 506, 530, 539, 523, 486, 526, 489, 490, 566, 522, 515, 516, 511, 487, 494, 499, 517, 477, 521, 498, 499, 536, 481, 502, 470, 519, 539, 519, 506, 530, 522, 495, 474, 538, 506, 529, 530, 499, 528, 542, 532, 515, 520, 551, 511, 510, 539, 504, 481, 499, 492, 533, 516, 515, 506, 507, 515, 460, 520, 501, 527, 548, 477] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[472, 509, 510, 552, 534, 496, 518, 518, 521, 519, 472, 532, 506, 501, 530, 528, 479, 548, 557, 511, 495, 526, 496, 489, 516, 487, 516, 507, 493, 531, 468, 511, 502, 486, 519, 490, 487, 501, 521, 521, 520, 497, 518, 523, 465, 553, 527, 522, 525, 519, 497, 521, 503, 552, 546, 500, 496, 527, 501, 530, 527, 535, 489, 500] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[523, 522, 506, 521, 482, 494, 514, 522, 506, 524, 527, 446, 543, 508, 538, 548, 518, 493, 519, 522, 518, 545, 553, 548, 531, 491, 526, 515, 518, 503, 480, 470, 502, 535, 485, 525, 512, 508, 461, 481, 504, 531, 521, 492, 470, 514, 492, 500, 553, 495, 509, 527, 538, 467, 520, 536, 488, 543, 501, 511, 518, 493, 515, 547] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[506, 525, 488, 508, 513, 496, 515, 518, 9, 0, 0, 0, 0, 0, 0, 0, 488, 542, 490, 498, 524, 541, 498, 541, 2, 0, 0, 0, 0, 0, 0, 0, 523, 491, 478, 545, 501, 505, 512, 506, 0, 0, 0, 0, 0, 0, 0, 0, 518, 550, 510, 515, 526, 546, 490, 505, 12, 3, 0, 0, 0, 0, 0, 0, 595, 465, 520, 506, 490, 516, 489, 490, 18, 0, 0, 0, 0, 0, 0, 0, 538, 493, 517, 541, 490, 512, 519, 497, 0, 0, 0, 0, 0, 0, 0, 0, 481, 515, 501, 517, 489, 504, 496, 502, 0, 0, 0, 0, 0, 0, 0, 0, 517, 539, 523, 517, 524, 496, 474, 539, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4124, 4061, 4175, 4089, 4107, 4005, 4129] avg=4096.0000 max=4175 max_rank=3 min=4005 min_rank=6 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16330 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[530, 462, 501, 529, 536, 518, 523, 490, 9, 0, 0, 0, 0, 0, 0, 0, 482, 476, 494, 514, 497, 534, 482, 515, 2, 0, 0, 0, 0, 0, 0, 0, 530, 493, 494, 536, 515, 502, 538, 507, 0, 0, 0, 0, 0, 0, 0, 0, 487, 522, 497, 543, 474, 533, 493, 505, 13, 2, 0, 0, 0, 0, 0, 0, 533, 507, 527, 509, 514, 512, 545, 513, 19, 0, 0, 0, 0, 0, 0, 0, 501, 506, 503, 486, 511, 500, 523, 549, 0, 0, 0, 0, 0, 0, 0, 0, 500, 522, 517, 490, 508, 552, 543, 535, 0, 0, 0, 0, 0, 0, 0, 0, 524, 502, 503, 499, 533, 493, 516, 495, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 3996, 4115, 4069, 4179, 4079, 4167, 4065] avg=4096.0000 max=4179 max_rank=4 min=3996 min_rank=1 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16490 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[539, 541, 473, 526, 510, 497, 521, 491, 9, 0, 0, 0, 0, 0, 0, 0, 501, 482, 494, 498, 512, 472, 537, 490, 2, 0, 0, 0, 0, 0, 0, 0, 545, 519, 521, 497, 533, 589, 520, 536, 0, 0, 0, 0, 0, 0, 0, 0, 487, 509, 494, 540, 535, 472, 515, 496, 13, 2, 0, 0, 0, 0, 0, 0, 527, 529, 525, 506, 500, 520, 476, 489, 20, 0, 0, 0, 0, 0, 0, 0, 507, 493, 510, 504, 507, 518, 529, 510, 0, 0, 0, 0, 0, 0, 0, 0, 518, 533, 543, 504, 551, 472, 515, 487, 0, 0, 0, 0, 0, 0, 0, 0, 500, 500, 529, 492, 510, 519, 501, 506, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 3988, 4260, 4063, 4092, 4078, 4123, 4057] avg=4096.0000 max=4260 max_rank=2 min=3988 min_rank=1 imbalance_ratio=1.040039 max_load_reduction=28508 max_load_reduction_pct=0.869995 moved_to_echo=16350 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[530, 527, 467, 528, 526, 503, 512, 539, 9, 0, 0, 0, 0, 0, 0, 0, 492, 517, 497, 526, 475, 549, 535, 536, 2, 0, 0, 0, 0, 0, 0, 0, 534, 502, 490, 499, 499, 529, 495, 504, 0, 0, 0, 0, 0, 0, 0, 0, 502, 490, 545, 501, 501, 469, 515, 490, 13, 2, 0, 0, 0, 0, 0, 0, 467, 538, 519, 510, 511, 524, 536, 508, 18, 0, 0, 0, 0, 0, 0, 0, 514, 506, 540, 512, 491, 500, 504, 525, 0, 0, 0, 0, 0, 0, 0, 0, 511, 514, 512, 491, 500, 521, 518, 478, 0, 0, 0, 0, 0, 0, 0, 0, 517, 510, 536, 544, 535, 491, 496, 521, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4141, 4129, 4052, 4028, 4131, 4092, 4045, 4150] avg=4096.0000 max=4150 max_rank=7 min=4028 min_rank=3 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16418 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[516, 492, 473, 504, 514, 478, 515, 507, 13, 0, 0, 0, 0, 0, 0, 0, 497, 532, 502, 517, 522, 538, 507, 550, 4, 6, 0, 0, 0, 0, 0, 0, 484, 546, 488, 539, 512, 486, 474, 498, 0, 0, 0, 0, 0, 0, 0, 0, 526, 521, 509, 541, 482, 548, 515, 553, 17, 8, 0, 0, 0, 0, 0, 0, 523, 473, 511, 507, 499, 469, 541, 542, 22, 0, 0, 0, 0, 0, 0, 0, 518, 528, 474, 535, 517, 522, 488, 494, 0, 0, 0, 0, 0, 0, 0, 0, 519, 535, 517, 502, 543, 508, 486, 470, 0, 0, 0, 0, 0, 0, 0, 0, 510, 533, 507, 485, 515, 517, 480, 544, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4012, 4175, 4027, 4220, 4087, 4076, 4080, 4091] avg=4096.0000 max=4220 max_rank=3 min=4012 min_rank=0 imbalance_ratio=1.030273 max_load_reduction=28548 max_load_reduction_pct=0.871216 moved_to_echo=16334 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[513, 506, 530, 539, 523, 486, 526, 489, 9, 0, 0, 0, 0, 0, 0, 0, 490, 566, 522, 515, 516, 511, 487, 494, 2, 0, 0, 0, 0, 0, 0, 0, 499, 517, 449, 521, 498, 499, 536, 481, 0, 0, 0, 0, 0, 0, 0, 0, 502, 470, 519, 539, 519, 506, 530, 522, 11, 2, 0, 0, 0, 0, 0, 0, 495, 474, 538, 506, 529, 530, 499, 528, 17, 0, 0, 0, 0, 0, 0, 0, 531, 532, 515, 520, 551, 511, 510, 539, 0, 0, 0, 0, 0, 0, 0, 0, 504, 481, 499, 492, 531, 516, 515, 506, 0, 0, 0, 0, 0, 0, 0, 0, 507, 515, 460, 520, 501, 527, 548, 477, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4103, 4000, 4120, 4116, 4209, 4044, 4055] avg=4096.0000 max=4209 max_rank=5 min=4000 min_rank=2 imbalance_ratio=1.027588 max_load_reduction=28559 max_load_reduction_pct=0.871552 moved_to_echo=16424 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[523, 522, 506, 521, 482, 494, 514, 522, 9, 0, 0, 0, 0, 0, 0, 0, 506, 524, 527, 446, 543, 508, 538, 548, 2, 0, 0, 0, 0, 0, 0, 0, 518, 493, 488, 522, 518, 545, 553, 548, 0, 0, 0, 0, 0, 0, 0, 0, 531, 491, 526, 515, 518, 503, 480, 470, 13, 2, 0, 0, 0, 0, 0, 0, 502, 535, 485, 525, 512, 508, 461, 481, 18, 0, 0, 0, 0, 0, 0, 0, 493, 531, 521, 492, 470, 514, 492, 500, 0, 0, 0, 0, 0, 0, 0, 0, 553, 495, 509, 527, 536, 467, 520, 536, 0, 0, 0, 0, 0, 0, 0, 0, 488, 543, 501, 511, 518, 493, 515, 547, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4142, 4185, 4049, 4027, 4013, 4143, 4116] avg=4096.0000 max=4185 max_rank=2 min=4013 min_rank=5 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16299 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[472, 509, 510, 552, 534, 496, 518, 518, 9, 0, 0, 0, 0, 0, 0, 0, 521, 519, 472, 532, 506, 501, 530, 528, 2, 0, 0, 0, 0, 0, 0, 0, 479, 548, 524, 511, 495, 526, 496, 489, 0, 0, 0, 0, 0, 0, 0, 0, 516, 487, 516, 507, 493, 531, 468, 511, 13, 2, 0, 0, 0, 0, 0, 0, 502, 486, 519, 490, 487, 501, 521, 521, 20, 0, 0, 0, 0, 0, 0, 0, 509, 497, 518, 523, 465, 553, 527, 522, 0, 0, 0, 0, 0, 0, 0, 0, 525, 519, 497, 521, 501, 552, 546, 500, 0, 0, 0, 0, 0, 0, 0, 0, 496, 527, 501, 530, 527, 535, 489, 500, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4111, 4068, 4044, 4047, 4114, 4161, 4105] avg=4096.0000 max=4161 max_rank=6 min=4044 min_rank=3 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16427 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 3968, 4224, 4224, 3968, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 3968, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[513, 505, 510, 539, 509, 521, 525, 462, 513, 522, 539, 514, 525, 504, 532, 497, 533, 486, 473, 460, 502, 512, 499, 520, 532, 516, 505, 485, 503, 500, 537, 482, 513, 524, 514, 551, 526, 487, 470, 537, 545, 540, 493, 532, 487, 478, 512, 529, 491, 519, 525, 543, 549, 540, 520, 508, 480, 507, 508, 517, 506, 523, 531, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[528, 553, 511, 491, 506, 559, 485, 529, 535, 501, 515, 530, 496, 538, 504, 496, 508, 490, 471, 486, 517, 513, 479, 491, 495, 540, 527, 509, 513, 525, 489, 542, 486, 490, 517, 514, 520, 508, 515, 522, 500, 489, 565, 539, 506, 564, 511, 499, 524, 481, 482, 474, 537, 531, 507, 515, 502, 510, 549, 508, 536, 525, 489, 481] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[491, 518, 491, 512, 487, 551, 527, 509, 512, 544, 556, 503, 471, 543, 476, 533, 506, 503, 535, 554, 529, 492, 511, 500, 514, 498, 529, 490, 521, 506, 498, 504, 480, 498, 535, 523, 589, 505, 497, 489, 501, 498, 516, 516, 540, 517, 500, 487, 505, 520, 495, 494, 544, 526, 514, 484, 500, 533, 496, 479, 491, 545, 548, 489] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[519, 503, 533, 515, 522, 499, 503, 523, 507, 481, 480, 501, 546, 525, 484, 502, 552, 555, 492, 514, 501, 539, 544, 482, 463, 530, 516, 499, 502, 524, 486, 526, 486, 543, 516, 508, 517, 503, 506, 541, 494, 504, 544, 491, 505, 497, 513, 516, 484, 474, 509, 541, 509, 516, 511, 520, 522, 541, 502, 498, 519, 538, 514, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[519, 551, 514, 483, 506, 524, 505, 539, 510, 507, 533, 550, 545, 523, 500, 509, 511, 509, 528, 502, 490, 483, 521, 500, 524, 512, 512, 521, 513, 474, 490, 562, 499, 506, 465, 553, 530, 529, 473, 513, 501, 512, 480, 537, 457, 527, 538, 490, 508, 525, 518, 527, 499, 497, 507, 499, 499, 529, 508, 492, 536, 503, 521, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[523, 526, 479, 541, 497, 494, 508, 483, 497, 505, 528, 505, 528, 526, 520, 485, 477, 536, 482, 501, 503, 548, 532, 506, 515, 525, 511, 522, 518, 517, 512, 504, 509, 552, 505, 498, 495, 537, 480, 544, 526, 515, 527, 506, 530, 508, 521, 495, 471, 492, 518, 511, 517, 505, 518, 537, 556, 512, 519, 510, 525, 485, 477, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[514, 488, 503, 497, 506, 542, 507, 485, 510, 548, 517, 484, 488, 509, 513, 502, 496, 505, 541, 487, 523, 520, 497, 515, 509, 502, 510, 529, 498, 505, 549, 541, 534, 494, 483, 528, 507, 507, 527, 514, 489, 490, 535, 497, 496, 494, 490, 465, 528, 535, 497, 561, 527, 529, 487, 506, 543, 524, 540, 506, 522, 525, 510, 538] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[524, 495, 523, 482, 550, 506, 523, 525, 482, 497, 517, 503, 475, 542, 460, 508, 505, 502, 533, 564, 533, 479, 483, 496, 493, 524, 517, 521, 546, 492, 505, 520, 491, 506, 492, 547, 485, 526, 541, 515, 513, 534, 550, 522, 537, 532, 487, 533, 547, 494, 501, 500, 528, 540, 508, 485, 491, 499, 469, 514, 482, 533, 502, 539] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[513, 505, 510, 539, 509, 516, 525, 462, 0, 0, 0, 0, 0, 0, 0, 0, 513, 522, 539, 514, 525, 504, 532, 497, 1, 0, 0, 0, 0, 0, 0, 0, 533, 486, 473, 460, 502, 512, 499, 520, 8, 6, 5, 5, 0, 0, 0, 0, 532, 516, 505, 485, 503, 500, 537, 481, 0, 0, 0, 0, 0, 0, 0, 0, 513, 524, 514, 545, 526, 487, 470, 537, 0, 0, 0, 0, 0, 0, 0, 0, 545, 540, 493, 532, 487, 478, 512, 529, 0, 0, 0, 0, 0, 0, 0, 0, 491, 519, 525, 543, 544, 540, 520, 508, 0, 0, 0, 0, 0, 0, 0, 0, 480, 507, 508, 517, 506, 515, 531, 488, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4147, 4009, 4059, 4116, 4116, 4190, 4052] avg=4096.0000 max=4190 max_rank=6 min=4009 min_rank=2 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16474 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[528, 553, 511, 491, 506, 554, 485, 529, 0, 0, 0, 0, 0, 0, 0, 0, 535, 501, 515, 530, 496, 538, 504, 496, 2, 0, 0, 0, 0, 0, 0, 0, 508, 490, 471, 486, 517, 513, 479, 491, 8, 5, 5, 5, 1, 0, 0, 0, 495, 540, 527, 509, 513, 525, 489, 539, 0, 0, 0, 0, 0, 0, 0, 0, 486, 490, 517, 509, 520, 508, 515, 522, 0, 0, 0, 0, 0, 0, 0, 0, 500, 489, 565, 539, 506, 564, 511, 499, 0, 0, 0, 0, 0, 0, 0, 0, 524, 481, 482, 474, 532, 531, 507, 515, 0, 0, 0, 0, 0, 0, 0, 0, 502, 510, 549, 508, 536, 517, 489, 481, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4157, 4117, 3979, 4137, 4067, 4173, 4046, 4092] avg=4096.0000 max=4173 max_rank=5 min=3979 min_rank=2 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[519, 503, 533, 515, 522, 491, 503, 523, 0, 0, 0, 0, 0, 0, 0, 0, 507, 481, 480, 501, 546, 525, 484, 502, 4, 0, 0, 0, 0, 0, 0, 0, 552, 555, 492, 514, 501, 539, 544, 482, 12, 9, 11, 8, 5, 0, 0, 0, 463, 530, 516, 499, 502, 524, 486, 511, 0, 0, 0, 0, 0, 0, 0, 0, 486, 543, 516, 499, 517, 503, 506, 541, 0, 0, 0, 0, 0, 0, 0, 0, 494, 504, 544, 491, 505, 497, 513, 516, 6, 0, 0, 0, 0, 0, 0, 0, 484, 474, 509, 541, 498, 516, 511, 520, 0, 0, 0, 0, 0, 0, 0, 0, 522, 541, 502, 498, 519, 526, 514, 518, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4030, 4224, 4031, 4111, 4070, 4053, 4140] avg=4096.0000 max=4224 max_rank=2 min=4030 min_rank=1 imbalance_ratio=1.031250 max_load_reduction=28544 max_load_reduction_pct=0.871094 moved_to_echo=16374 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[514, 488, 503, 497, 506, 537, 507, 485, 0, 0, 0, 0, 0, 0, 0, 0, 510, 548, 517, 484, 488, 509, 513, 502, 2, 0, 0, 0, 0, 0, 0, 0, 496, 505, 541, 487, 523, 520, 497, 515, 8, 5, 5, 5, 1, 0, 0, 0, 509, 502, 510, 529, 498, 505, 549, 538, 0, 0, 0, 0, 0, 0, 0, 0, 534, 494, 483, 523, 507, 507, 527, 514, 0, 0, 0, 0, 0, 0, 0, 0, 489, 490, 535, 497, 496, 494, 490, 465, 0, 0, 0, 0, 0, 0, 0, 0, 528, 535, 497, 561, 522, 529, 487, 506, 0, 0, 0, 0, 0, 0, 0, 0, 543, 524, 540, 506, 522, 517, 510, 538, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4037, 4073, 4108, 4140, 4089, 3956, 4165, 4200] avg=4096.0000 max=4200 max_rank=7 min=3956 min_rank=5 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16410 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[491, 518, 491, 512, 487, 546, 527, 509, 0, 0, 0, 0, 0, 0, 0, 0, 512, 544, 556, 503, 471, 543, 476, 533, 2, 0, 0, 0, 0, 0, 0, 0, 506, 503, 535, 554, 529, 492, 511, 500, 8, 5, 5, 5, 0, 0, 0, 0, 514, 498, 529, 490, 521, 506, 498, 502, 0, 0, 0, 0, 0, 0, 0, 0, 480, 498, 535, 518, 589, 505, 497, 489, 0, 0, 0, 0, 0, 0, 0, 0, 501, 498, 516, 516, 540, 517, 500, 487, 0, 0, 0, 0, 0, 0, 0, 0, 505, 520, 495, 494, 539, 526, 514, 484, 0, 0, 0, 0, 0, 0, 0, 0, 500, 533, 496, 479, 491, 537, 548, 489, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4140, 4153, 4058, 4111, 4075, 4077, 4073] avg=4096.0000 max=4153 max_rank=2 min=4058 min_rank=3 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16336 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[519, 551, 514, 483, 506, 519, 505, 539, 0, 0, 0, 0, 0, 0, 0, 0, 510, 507, 533, 550, 545, 523, 500, 509, 2, 0, 0, 0, 0, 0, 0, 0, 511, 509, 528, 502, 490, 483, 521, 500, 8, 6, 5, 5, 1, 0, 0, 0, 524, 512, 512, 521, 513, 474, 490, 559, 0, 0, 0, 0, 0, 0, 0, 0, 499, 506, 465, 547, 530, 529, 473, 513, 0, 0, 0, 0, 0, 0, 0, 0, 501, 512, 480, 537, 457, 527, 538, 490, 0, 0, 0, 0, 0, 0, 0, 0, 508, 525, 518, 527, 494, 497, 507, 499, 0, 0, 0, 0, 0, 0, 0, 0, 499, 529, 508, 492, 536, 495, 521, 520, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4136, 4179, 4069, 4105, 4062, 4042, 4075, 4100] avg=4096.0000 max=4179 max_rank=1 min=4042 min_rank=5 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16279 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[523, 526, 479, 541, 497, 489, 508, 483, 0, 0, 0, 0, 0, 0, 0, 0, 497, 505, 528, 505, 528, 526, 520, 485, 2, 0, 0, 0, 0, 0, 0, 0, 477, 536, 482, 501, 503, 548, 532, 506, 7, 5, 5, 5, 0, 0, 0, 0, 515, 525, 511, 522, 518, 517, 512, 502, 0, 0, 0, 0, 0, 0, 0, 0, 509, 552, 505, 493, 495, 537, 480, 544, 0, 0, 0, 0, 0, 0, 0, 0, 526, 515, 527, 506, 530, 508, 521, 495, 0, 0, 0, 0, 0, 0, 0, 0, 471, 492, 518, 511, 512, 505, 518, 537, 0, 0, 0, 0, 0, 0, 0, 0, 556, 512, 519, 510, 525, 478, 477, 513, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4046, 4096, 4107, 4122, 4115, 4128, 4064, 4090] avg=4096.0000 max=4128 max_rank=5 min=4046 min_rank=0 imbalance_ratio=1.007812 max_load_reduction=28640 max_load_reduction_pct=0.874023 moved_to_echo=16397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[524, 495, 523, 482, 550, 501, 523, 525, 0, 0, 0, 0, 0, 0, 0, 0, 482, 497, 517, 503, 475, 542, 460, 508, 2, 0, 0, 0, 0, 0, 0, 0, 505, 502, 533, 564, 533, 479, 483, 496, 8, 6, 5, 5, 0, 0, 0, 0, 493, 524, 517, 521, 546, 492, 505, 518, 0, 0, 0, 0, 0, 0, 0, 0, 491, 506, 492, 541, 485, 526, 541, 515, 0, 0, 0, 0, 0, 0, 0, 0, 513, 534, 550, 522, 537, 532, 487, 533, 0, 0, 0, 0, 0, 0, 0, 0, 547, 494, 501, 500, 523, 540, 508, 485, 0, 0, 0, 0, 0, 0, 0, 0, 491, 499, 469, 514, 482, 525, 502, 539, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4123, 3986, 4119, 4116, 4097, 4208, 4098, 4021] avg=4096.0000 max=4208 max_rank=5 min=3986 min_rank=1 imbalance_ratio=1.027344 max_load_reduction=28560 max_load_reduction_pct=0.871582 moved_to_echo=16424 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 128, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[519, 556, 523, 498, 490, 530, 524, 484, 493, 509, 504, 508, 488, 532, 508, 522, 530, 537, 495, 526, 505, 554, 525, 500, 476, 481, 478, 515, 518, 490, 535, 509, 524, 528, 521, 524, 531, 518, 510, 490, 561, 539, 526, 484, 485, 513, 460, 521, 523, 500, 511, 570, 474, 480, 504, 548, 517, 484, 515, 508, 512, 497, 507, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[511, 522, 537, 510, 502, 501, 487, 512, 516, 509, 526, 543, 493, 501, 502, 505, 498, 526, 537, 497, 534, 556, 528, 504, 489, 491, 523, 512, 480, 514, 513, 484, 540, 507, 517, 530, 497, 521, 494, 509, 503, 560, 508, 559, 474, 522, 498, 472, 533, 504, 523, 528, 517, 514, 479, 489, 511, 540, 521, 495, 511, 521, 497, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[498, 509, 532, 508, 518, 488, 525, 511, 503, 499, 474, 470, 519, 539, 511, 544, 521, 500, 547, 545, 513, 482, 497, 515, 525, 510, 535, 491, 487, 513, 502, 499, 503, 501, 568, 483, 515, 528, 515, 501, 510, 519, 466, 503, 499, 513, 467, 518, 507, 549, 524, 495, 483, 557, 510, 524, 530, 548, 496, 536, 518, 519, 498, 535] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[538, 523, 523, 503, 514, 505, 523, 504, 530, 535, 579, 531, 529, 537, 501, 488, 518, 520, 530, 518, 538, 480, 503, 510, 522, 512, 471, 490, 495, 488, 510, 480, 477, 523, 493, 490, 525, 506, 484, 534, 534, 544, 500, 533, 520, 481, 508, 503, 479, 494, 523, 483, 519, 478, 555, 511, 504, 507, 558, 519, 497, 520, 522, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[514, 523, 504, 501, 537, 513, 536, 518, 515, 473, 507, 543, 457, 545, 566, 515, 505, 502, 519, 501, 510, 519, 529, 488, 542, 531, 496, 486, 547, 495, 476, 466, 492, 484, 508, 518, 499, 460, 538, 538, 530, 519, 501, 511, 527, 507, 531, 509, 519, 495, 533, 488, 504, 496, 513, 513, 529, 521, 507, 561, 517, 514, 491, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[538, 513, 524, 488, 490, 530, 521, 512, 503, 499, 497, 525, 510, 528, 505, 561, 524, 486, 500, 517, 496, 508, 494, 551, 535, 510, 511, 544, 499, 509, 525, 449, 523, 485, 545, 490, 512, 514, 505, 462, 550, 504, 543, 486, 524, 500, 513, 477, 531, 526, 508, 525, 509, 508, 484, 526, 545, 509, 516, 490, 547, 500, 493, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[514, 508, 510, 519, 498, 487, 484, 530, 481, 505, 495, 511, 528, 486, 524, 501, 484, 519, 553, 508, 543, 495, 522, 499, 509, 485, 471, 571, 515, 517, 501, 519, 516, 514, 546, 526, 497, 512, 484, 487, 531, 522, 497, 488, 522, 525, 568, 522, 492, 521, 510, 486, 523, 522, 527, 512, 527, 493, 515, 534, 521, 493, 493, 550] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[558, 503, 507, 498, 501, 491, 533, 479, 524, 492, 511, 477, 524, 532, 521, 521, 495, 548, 486, 540, 529, 524, 487, 504, 464, 475, 568, 498, 528, 500, 518, 534, 537, 508, 525, 542, 511, 506, 486, 518, 525, 515, 514, 549, 510, 512, 502, 501, 526, 524, 505, 493, 504, 513, 495, 544, 478, 512, 498, 509, 514, 518, 489, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[515, 556, 523, 498, 490, 530, 524, 484, 0, 0, 0, 0, 0, 0, 0, 0, 493, 509, 504, 508, 488, 525, 508, 522, 0, 0, 0, 0, 0, 0, 0, 0, 530, 537, 495, 526, 481, 554, 525, 500, 0, 0, 0, 0, 0, 0, 0, 0, 476, 481, 478, 515, 518, 490, 535, 509, 24, 16, 9, 3, 0, 0, 0, 0, 524, 528, 521, 524, 531, 518, 510, 490, 4, 4, 0, 0, 0, 0, 0, 0, 552, 539, 526, 484, 485, 513, 460, 521, 0, 0, 0, 0, 0, 0, 0, 0, 523, 500, 511, 570, 474, 480, 504, 548, 0, 0, 0, 0, 0, 0, 0, 0, 517, 484, 515, 508, 512, 497, 507, 505, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4057, 4148, 4054, 4154, 4080, 4110, 4045] avg=4096.0000 max=4154 max_rank=4 min=4045 min_rank=7 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[507, 522, 537, 510, 502, 501, 487, 512, 0, 0, 0, 0, 0, 0, 0, 0, 516, 509, 526, 543, 493, 494, 502, 505, 0, 0, 0, 0, 0, 0, 0, 0, 498, 526, 537, 497, 508, 556, 528, 504, 0, 0, 0, 0, 0, 0, 0, 0, 489, 491, 523, 512, 480, 514, 513, 484, 26, 16, 8, 3, 0, 0, 0, 0, 540, 507, 517, 530, 497, 521, 494, 509, 4, 4, 0, 0, 0, 0, 0, 0, 495, 560, 508, 559, 474, 522, 498, 472, 0, 0, 0, 0, 0, 0, 0, 0, 533, 504, 523, 528, 517, 514, 479, 489, 0, 0, 0, 0, 0, 0, 0, 0, 511, 540, 521, 495, 511, 521, 497, 495, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4088, 4154, 4059, 4123, 4088, 4087, 4091] avg=4096.0000 max=4154 max_rank=2 min=4059 min_rank=3 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[494, 509, 532, 508, 518, 488, 525, 511, 0, 0, 0, 0, 0, 0, 0, 0, 503, 499, 474, 470, 519, 532, 511, 544, 0, 0, 0, 0, 0, 0, 0, 0, 521, 500, 547, 545, 488, 482, 497, 515, 0, 0, 0, 0, 0, 0, 0, 0, 525, 510, 535, 491, 487, 513, 502, 499, 25, 17, 8, 3, 0, 0, 0, 0, 503, 501, 568, 483, 515, 528, 515, 501, 4, 4, 0, 0, 0, 0, 0, 0, 502, 519, 466, 503, 499, 513, 467, 518, 0, 0, 0, 0, 0, 0, 0, 0, 507, 549, 524, 495, 483, 557, 510, 524, 0, 0, 0, 0, 0, 0, 0, 0, 530, 548, 496, 536, 518, 519, 498, 518, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4052, 4095, 4115, 4122, 3987, 4149, 4163] avg=4096.0000 max=4163 max_rank=7 min=3987 min_rank=5 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16421 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[524, 523, 523, 503, 514, 505, 523, 504, 0, 0, 0, 0, 0, 0, 0, 0, 530, 535, 579, 531, 529, 524, 501, 488, 0, 0, 0, 0, 0, 0, 0, 0, 518, 520, 530, 518, 508, 480, 503, 510, 0, 0, 0, 0, 0, 0, 0, 0, 522, 512, 471, 490, 495, 488, 510, 480, 30, 20, 12, 5, 0, 0, 0, 0, 477, 523, 493, 490, 525, 506, 484, 534, 9, 8, 0, 0, 0, 0, 0, 0, 522, 544, 500, 533, 520, 481, 508, 503, 0, 0, 0, 0, 0, 0, 0, 0, 479, 494, 523, 483, 519, 478, 555, 511, 5, 0, 0, 0, 0, 0, 0, 0, 504, 507, 558, 519, 497, 520, 522, 476, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4217, 4087, 4035, 4049, 4111, 4047, 4103] avg=4096.0000 max=4217 max_rank=1 min=4035 min_rank=3 imbalance_ratio=1.029541 max_load_reduction=28551 max_load_reduction_pct=0.871307 moved_to_echo=16310 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[510, 523, 504, 501, 537, 513, 536, 518, 0, 0, 0, 0, 0, 0, 0, 0, 515, 473, 507, 543, 457, 538, 566, 515, 0, 0, 0, 0, 0, 0, 0, 0, 505, 502, 519, 501, 485, 519, 529, 488, 0, 0, 0, 0, 0, 0, 0, 0, 542, 531, 496, 486, 547, 495, 476, 466, 25, 16, 8, 3, 0, 0, 0, 0, 492, 484, 508, 518, 499, 460, 538, 538, 4, 4, 0, 0, 0, 0, 0, 0, 522, 519, 501, 511, 527, 507, 531, 509, 0, 0, 0, 0, 0, 0, 0, 0, 519, 495, 533, 488, 504, 496, 513, 513, 0, 0, 0, 0, 0, 0, 0, 0, 529, 521, 507, 561, 517, 514, 491, 500, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4142, 4114, 4048, 4091, 4045, 4127, 4061, 4140] avg=4096.0000 max=4142 max_rank=0 min=4045 min_rank=4 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[534, 513, 524, 488, 490, 530, 521, 512, 0, 0, 0, 0, 0, 0, 0, 0, 503, 499, 497, 525, 510, 521, 505, 561, 0, 0, 0, 0, 0, 0, 0, 0, 524, 486, 500, 517, 472, 508, 494, 551, 0, 0, 0, 0, 0, 0, 0, 0, 535, 510, 511, 544, 499, 509, 525, 449, 24, 16, 9, 3, 0, 0, 0, 0, 523, 485, 545, 490, 512, 514, 505, 462, 4, 4, 0, 0, 0, 0, 0, 0, 541, 504, 543, 486, 524, 500, 513, 477, 0, 0, 0, 0, 0, 0, 0, 0, 531, 526, 508, 525, 509, 508, 484, 526, 0, 0, 0, 0, 0, 0, 0, 0, 545, 509, 516, 490, 547, 500, 493, 500, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4121, 4052, 4134, 4044, 4088, 4117, 4100] avg=4096.0000 max=4134 max_rank=3 min=4044 min_rank=4 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16349 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[510, 508, 510, 519, 498, 487, 484, 530, 0, 0, 0, 0, 0, 0, 0, 0, 481, 505, 495, 511, 528, 479, 524, 501, 0, 0, 0, 0, 0, 0, 0, 0, 484, 519, 553, 508, 517, 495, 522, 499, 0, 0, 0, 0, 0, 0, 0, 0, 509, 485, 471, 571, 515, 517, 501, 519, 26, 17, 8, 3, 0, 0, 0, 0, 516, 514, 546, 526, 497, 512, 484, 487, 4, 4, 0, 0, 0, 0, 0, 0, 523, 522, 497, 488, 522, 525, 568, 522, 0, 0, 0, 0, 0, 0, 0, 0, 492, 521, 510, 486, 523, 522, 527, 512, 0, 0, 0, 0, 0, 0, 0, 0, 527, 493, 515, 534, 521, 493, 493, 533, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4046, 4024, 4097, 4142, 4090, 4167, 4093, 4109] avg=4096.0000 max=4167 max_rank=5 min=4024 min_rank=1 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16459 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[554, 503, 507, 498, 501, 491, 533, 479, 0, 0, 0, 0, 0, 0, 0, 0, 524, 492, 511, 477, 524, 525, 521, 521, 0, 0, 0, 0, 0, 0, 0, 0, 495, 548, 486, 540, 503, 524, 487, 504, 0, 0, 0, 0, 0, 0, 0, 0, 464, 475, 568, 498, 528, 500, 518, 534, 26, 16, 8, 3, 0, 0, 0, 0, 537, 508, 525, 542, 511, 506, 486, 518, 4, 4, 0, 0, 0, 0, 0, 0, 517, 515, 514, 549, 510, 512, 502, 501, 0, 0, 0, 0, 0, 0, 0, 0, 526, 524, 505, 493, 504, 513, 495, 544, 0, 0, 0, 0, 0, 0, 0, 0, 478, 512, 498, 509, 514, 518, 489, 499, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 4095, 4087, 4138, 4141, 4120, 4104, 4017] avg=4096.0000 max=4141 max_rank=4 min=4017 min_rank=7 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4224, 4224, 3968, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 3968, 256, 256, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[537, 494, 514, 531, 521, 512, 545, 477, 505, 496, 502, 529, 534, 513, 521, 499, 551, 491, 482, 523, 498, 522, 511, 503, 484, 495, 502, 522, 499, 489, 518, 480, 537, 495, 522, 513, 506, 513, 488, 528, 493, 524, 545, 477, 529, 510, 506, 510, 507, 486, 496, 536, 529, 509, 548, 543, 511, 511, 531, 513, 509, 492, 503, 548] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[497, 489, 522, 494, 524, 549, 521, 511, 540, 524, 502, 526, 475, 497, 520, 485, 508, 531, 504, 475, 517, 539, 535, 530, 476, 521, 508, 495, 533, 532, 509, 494, 536, 527, 518, 547, 482, 558, 507, 512, 483, 501, 503, 514, 499, 509, 490, 542, 509, 524, 488, 509, 531, 489, 482, 531, 517, 542, 532, 485, 520, 498, 516, 484] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[458, 504, 525, 528, 478, 550, 465, 511, 479, 504, 521, 484, 535, 477, 506, 539, 521, 492, 513, 528, 505, 524, 562, 483, 487, 485, 490, 541, 549, 527, 564, 514, 483, 491, 548, 547, 527, 504, 515, 536, 522, 465, 520, 534, 508, 498, 533, 511, 525, 497, 499, 470, 483, 512, 481, 502, 530, 508, 510, 505, 549, 560, 510, 536] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[514, 509, 502, 533, 517, 519, 531, 520, 513, 499, 520, 531, 516, 506, 523, 480, 484, 529, 511, 503, 528, 502, 527, 506, 507, 504, 518, 532, 524, 489, 522, 535, 535, 471, 475, 511, 488, 491, 508, 492, 503, 501, 529, 518, 506, 509, 510, 525, 554, 534, 502, 498, 508, 543, 497, 532, 514, 528, 479, 519, 508, 502, 481, 543] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[508, 505, 478, 520, 540, 523, 521, 526, 498, 512, 511, 527, 517, 548, 528, 498, 495, 485, 482, 477, 535, 525, 520, 523, 504, 535, 517, 534, 520, 521, 514, 479, 509, 511, 512, 504, 495, 500, 504, 508, 515, 474, 500, 544, 483, 534, 541, 535, 493, 519, 488, 513, 509, 520, 513, 528, 524, 489, 492, 516, 459, 562, 519, 529] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[531, 504, 522, 488, 500, 549, 499, 520, 498, 492, 507, 503, 542, 499, 514, 508, 521, 546, 561, 512, 481, 521, 479, 526, 475, 494, 491, 456, 482, 495, 531, 510, 510, 530, 493, 518, 481, 496, 476, 548, 548, 537, 523, 522, 517, 518, 526, 527, 547, 516, 534, 512, 519, 503, 488, 546, 491, 538, 521, 502, 503, 520, 494, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[508, 526, 500, 503, 509, 506, 551, 498, 491, 508, 522, 496, 475, 504, 551, 508, 498, 471, 523, 488, 512, 543, 544, 520, 514, 492, 542, 504, 527, 513, 469, 493, 528, 508, 498, 511, 531, 525, 510, 524, 516, 511, 522, 506, 541, 477, 500, 502, 506, 493, 473, 541, 540, 528, 544, 495, 519, 509, 525, 496, 502, 516, 521, 541] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[551, 541, 521, 541, 495, 495, 522, 507, 537, 527, 510, 476, 499, 525, 504, 499, 553, 517, 496, 531, 475, 516, 489, 533, 489, 508, 499, 539, 509, 510, 515, 492, 508, 501, 515, 499, 506, 515, 521, 519, 552, 503, 519, 534, 495, 510, 513, 498, 513, 466, 474, 545, 514, 503, 515, 507, 512, 520, 511, 511, 528, 516, 507, 497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[537, 494, 514, 531, 521, 495, 545, 477, 0, 0, 0, 0, 0, 0, 0, 0, 505, 496, 502, 529, 534, 513, 521, 499, 13, 1, 1, 0, 0, 0, 0, 0, 551, 491, 482, 523, 498, 515, 511, 503, 0, 0, 0, 0, 0, 0, 0, 0, 484, 495, 502, 522, 499, 489, 518, 480, 17, 14, 0, 0, 0, 0, 0, 0, 537, 495, 522, 513, 506, 513, 488, 528, 5, 0, 0, 0, 0, 0, 0, 0, 493, 524, 532, 477, 529, 510, 506, 510, 0, 0, 0, 0, 0, 0, 0, 0, 507, 486, 496, 536, 529, 509, 548, 543, 1, 0, 0, 0, 0, 0, 0, 0, 511, 511, 531, 513, 509, 492, 503, 533, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4114, 4074, 4020, 4107, 4081, 4155, 4103] avg=4096.0000 max=4155 max_rank=6 min=4020 min_rank=3 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16446 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[458, 504, 525, 528, 478, 532, 465, 511, 0, 0, 0, 0, 0, 0, 0, 0, 479, 504, 521, 484, 535, 477, 506, 539, 12, 2, 1, 0, 0, 0, 0, 0, 521, 492, 513, 528, 505, 516, 562, 483, 0, 0, 0, 0, 0, 0, 0, 0, 487, 485, 490, 541, 549, 527, 564, 514, 18, 13, 0, 0, 0, 0, 0, 0, 483, 491, 548, 547, 527, 504, 515, 536, 5, 0, 0, 0, 0, 0, 0, 0, 522, 465, 508, 534, 508, 498, 533, 511, 0, 0, 0, 0, 0, 0, 0, 0, 525, 497, 499, 470, 483, 512, 481, 502, 1, 0, 0, 0, 0, 0, 0, 0, 530, 508, 510, 505, 549, 560, 510, 522, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4001, 4060, 4120, 4188, 4156, 4079, 3970, 4194] avg=4096.0000 max=4194 max_rank=7 min=3970 min_rank=6 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[497, 489, 522, 494, 524, 531, 521, 511, 0, 0, 0, 0, 0, 0, 0, 0, 540, 524, 502, 526, 475, 497, 520, 485, 12, 2, 1, 0, 0, 0, 0, 0, 508, 531, 504, 475, 517, 531, 535, 530, 0, 0, 0, 0, 0, 0, 0, 0, 476, 521, 508, 495, 533, 532, 509, 494, 18, 12, 0, 0, 0, 0, 0, 0, 536, 527, 518, 547, 482, 558, 507, 512, 5, 0, 0, 0, 0, 0, 0, 0, 483, 501, 491, 514, 499, 509, 490, 542, 0, 0, 0, 0, 0, 0, 0, 0, 509, 524, 488, 509, 531, 489, 482, 531, 1, 0, 0, 0, 0, 0, 0, 0, 517, 542, 532, 485, 520, 498, 516, 471, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4084, 4131, 4098, 4192, 4029, 4064, 4081] avg=4096.0000 max=4192 max_rank=4 min=4029 min_rank=5 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[531, 504, 522, 488, 500, 531, 499, 520, 0, 0, 0, 0, 0, 0, 0, 0, 498, 492, 507, 503, 542, 499, 514, 508, 12, 1, 1, 0, 0, 0, 0, 0, 521, 546, 561, 512, 481, 514, 479, 526, 0, 0, 0, 0, 0, 0, 0, 0, 475, 494, 491, 456, 482, 495, 531, 510, 18, 13, 0, 0, 0, 0, 0, 0, 510, 530, 493, 518, 481, 496, 476, 548, 5, 0, 0, 0, 0, 0, 0, 0, 548, 537, 511, 522, 517, 518, 526, 527, 0, 0, 0, 0, 0, 0, 0, 0, 547, 516, 534, 512, 519, 503, 488, 546, 1, 0, 0, 0, 0, 0, 0, 0, 491, 538, 521, 502, 503, 520, 494, 493, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4077, 4140, 3965, 4057, 4206, 4166, 4062] avg=4096.0000 max=4206 max_rank=5 min=3965 min_rank=3 imbalance_ratio=1.026855 max_load_reduction=28562 max_load_reduction_pct=0.871643 moved_to_echo=16491 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[514, 509, 502, 533, 517, 502, 531, 520, 0, 0, 0, 0, 0, 0, 0, 0, 513, 499, 520, 531, 516, 506, 523, 480, 12, 1, 1, 0, 0, 0, 0, 0, 484, 529, 511, 503, 528, 495, 527, 506, 0, 0, 0, 0, 0, 0, 0, 0, 507, 504, 518, 532, 524, 489, 522, 535, 17, 14, 0, 0, 0, 0, 0, 0, 535, 471, 475, 511, 488, 491, 508, 492, 5, 0, 0, 0, 0, 0, 0, 0, 503, 501, 517, 518, 506, 509, 510, 525, 0, 0, 0, 0, 0, 0, 0, 0, 554, 534, 502, 498, 508, 543, 497, 532, 1, 0, 0, 0, 0, 0, 0, 0, 514, 528, 479, 519, 508, 502, 481, 528, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4102, 4083, 4162, 3976, 4089, 4169, 4059] avg=4096.0000 max=4169 max_rank=6 min=3976 min_rank=4 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16293 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[508, 505, 478, 520, 540, 506, 521, 526, 0, 0, 0, 0, 0, 0, 0, 0, 498, 512, 511, 527, 517, 548, 528, 498, 12, 2, 1, 0, 0, 0, 0, 0, 495, 485, 482, 477, 535, 517, 520, 523, 0, 0, 0, 0, 0, 0, 0, 0, 504, 535, 517, 534, 520, 521, 514, 479, 17, 13, 0, 0, 0, 0, 0, 0, 509, 511, 512, 504, 495, 500, 504, 508, 5, 0, 0, 0, 0, 0, 0, 0, 515, 474, 488, 544, 483, 534, 541, 535, 0, 0, 0, 0, 0, 0, 0, 0, 493, 519, 488, 513, 509, 520, 513, 528, 1, 0, 0, 0, 0, 0, 0, 0, 524, 489, 492, 516, 459, 562, 519, 515, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4154, 4034, 4154, 4048, 4114, 4084, 4076] avg=4096.0000 max=4154 max_rank=1 min=4034 min_rank=2 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16322 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[508, 526, 500, 503, 509, 485, 551, 498, 0, 0, 0, 0, 0, 0, 0, 0, 491, 508, 522, 496, 475, 504, 551, 508, 17, 6, 8, 0, 0, 0, 0, 0, 498, 471, 523, 488, 512, 527, 544, 520, 0, 0, 0, 0, 0, 0, 0, 0, 514, 492, 542, 504, 527, 513, 469, 493, 21, 17, 0, 0, 0, 0, 0, 0, 528, 508, 498, 511, 531, 525, 510, 524, 8, 0, 0, 0, 0, 0, 0, 0, 516, 511, 505, 506, 541, 477, 500, 502, 0, 0, 0, 0, 0, 0, 0, 0, 506, 493, 473, 541, 540, 528, 544, 495, 2, 0, 0, 0, 0, 0, 0, 0, 519, 509, 525, 496, 502, 516, 521, 516, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4086, 4083, 4092, 4143, 4058, 4122, 4104] avg=4096.0000 max=4143 max_rank=4 min=4058 min_rank=5 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16427 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[551, 541, 521, 541, 495, 479, 522, 507, 0, 0, 0, 0, 0, 0, 0, 0, 537, 527, 510, 476, 499, 525, 504, 499, 12, 1, 1, 0, 0, 0, 0, 0, 553, 517, 496, 531, 475, 509, 489, 533, 0, 0, 0, 0, 0, 0, 0, 0, 489, 508, 499, 539, 509, 510, 515, 492, 16, 12, 0, 0, 0, 0, 0, 0, 508, 501, 515, 499, 506, 515, 521, 519, 5, 0, 0, 0, 0, 0, 0, 0, 552, 503, 507, 534, 495, 510, 513, 498, 0, 0, 0, 0, 0, 0, 0, 0, 513, 466, 474, 545, 514, 503, 515, 507, 1, 0, 0, 0, 0, 0, 0, 0, 512, 520, 511, 511, 528, 516, 507, 484, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4157, 4091, 4103, 4089, 4089, 4112, 4038, 4089] avg=4096.0000 max=4157 max_rank=0 min=4038 min_rank=6 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16328 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3968, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 3968, 4224, 4224, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[453, 514, 522, 524, 524, 502, 509, 496, 534, 494, 504, 478, 510, 548, 521, 520, 539, 481, 480, 496, 526, 459, 494, 488, 518, 538, 521, 546, 531, 547, 505, 527, 487, 491, 483, 512, 514, 493, 513, 495, 500, 522, 528, 546, 524, 499, 533, 516, 517, 526, 502, 491, 545, 520, 530, 506, 509, 556, 500, 511, 535, 481, 491, 543] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[524, 497, 531, 499, 519, 533, 511, 511, 519, 495, 465, 494, 519, 519, 473, 489, 506, 496, 491, 540, 520, 499, 499, 486, 516, 540, 541, 561, 541, 491, 540, 520, 510, 538, 524, 498, 532, 510, 513, 481, 493, 514, 502, 489, 471, 533, 519, 535, 488, 561, 525, 492, 518, 503, 545, 505, 543, 498, 499, 505, 520, 527, 512, 480] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[521, 523, 522, 500, 514, 509, 501, 522, 534, 525, 500, 510, 542, 508, 469, 501, 512, 538, 487, 461, 549, 481, 532, 516, 525, 509, 495, 547, 519, 497, 548, 493, 516, 495, 515, 506, 524, 475, 494, 519, 495, 542, 494, 520, 502, 501, 487, 516, 521, 513, 512, 506, 481, 518, 542, 531, 494, 495, 531, 495, 508, 533, 517, 560] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[501, 567, 499, 519, 516, 482, 546, 512, 493, 519, 489, 515, 482, 504, 525, 501, 531, 518, 510, 513, 509, 553, 507, 512, 497, 515, 481, 506, 558, 501, 521, 494, 536, 495, 479, 515, 539, 503, 493, 506, 512, 497, 517, 514, 494, 480, 509, 510, 516, 554, 516, 518, 510, 542, 518, 513, 486, 503, 515, 564, 513, 494, 508, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[508, 518, 496, 519, 511, 501, 484, 525, 499, 512, 483, 549, 526, 582, 536, 480, 518, 504, 534, 504, 546, 532, 511, 520, 494, 505, 494, 493, 515, 503, 509, 508, 464, 544, 539, 516, 510, 544, 475, 536, 469, 574, 524, 508, 481, 479, 530, 503, 505, 495, 468, 516, 517, 508, 557, 495, 484, 497, 492, 506, 532, 571, 514, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[477, 513, 505, 557, 500, 504, 508, 523, 525, 505, 495, 507, 500, 539, 475, 499, 523, 508, 551, 486, 524, 541, 529, 511, 528, 537, 485, 497, 490, 521, 490, 509, 521, 496, 540, 507, 521, 510, 525, 489, 500, 556, 498, 508, 509, 550, 484, 485, 508, 534, 517, 493, 489, 511, 487, 525, 503, 572, 488, 519, 507, 527, 519, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[546, 500, 512, 551, 536, 510, 540, 563, 521, 491, 528, 510, 525, 460, 521, 527, 528, 524, 511, 508, 521, 510, 513, 496, 525, 507, 485, 534, 504, 485, 501, 507, 509, 511, 522, 543, 506, 509, 500, 499, 494, 530, 516, 544, 518, 484, 468, 533, 531, 521, 518, 491, 545, 478, 504, 477, 501, 506, 513, 469, 515, 518, 517, 478] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[503, 518, 456, 487, 518, 516, 509, 519, 572, 525, 528, 532, 478, 502, 512, 498, 489, 489, 476, 501, 528, 506, 530, 537, 523, 511, 532, 501, 542, 538, 497, 478, 521, 520, 501, 484, 511, 542, 552, 533, 516, 539, 484, 513, 510, 533, 519, 478, 489, 510, 509, 549, 522, 519, 477, 501, 508, 570, 528, 460, 500, 506, 507, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[521, 523, 522, 500, 514, 509, 501, 512, 0, 0, 0, 0, 0, 0, 0, 0, 534, 525, 500, 510, 542, 508, 469, 501, 15, 0, 0, 0, 0, 0, 0, 0, 512, 538, 487, 461, 549, 481, 532, 516, 3, 0, 0, 0, 0, 0, 0, 0, 525, 509, 495, 547, 499, 497, 548, 493, 0, 0, 0, 0, 0, 0, 0, 0, 516, 495, 515, 506, 524, 475, 494, 519, 6, 1, 0, 0, 0, 0, 0, 0, 495, 542, 494, 520, 502, 501, 487, 516, 9, 5, 0, 0, 0, 0, 0, 0, 521, 504, 512, 506, 481, 518, 542, 531, 0, 0, 0, 0, 0, 0, 0, 0, 494, 495, 531, 495, 508, 533, 517, 560, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4104, 4079, 4113, 4051, 4071, 4115, 4133] avg=4096.0000 max=4133 max_rank=7 min=4051 min_rank=4 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16370 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[508, 518, 496, 519, 511, 501, 484, 515, 0, 0, 0, 0, 0, 0, 0, 0, 499, 512, 483, 549, 526, 582, 536, 480, 15, 0, 0, 0, 0, 0, 0, 0, 518, 504, 534, 504, 546, 532, 511, 520, 3, 0, 0, 0, 0, 0, 0, 0, 494, 505, 494, 493, 495, 503, 509, 508, 0, 0, 0, 0, 0, 0, 0, 0, 464, 544, 539, 516, 510, 544, 475, 536, 5, 1, 0, 0, 0, 0, 0, 0, 469, 574, 524, 508, 481, 479, 530, 503, 9, 5, 0, 0, 0, 0, 0, 0, 505, 487, 468, 516, 517, 508, 557, 495, 0, 0, 0, 0, 0, 0, 0, 0, 484, 497, 492, 506, 532, 571, 514, 496, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4052, 4182, 4172, 4001, 4134, 4082, 4053, 4092] avg=4096.0000 max=4182 max_rank=1 min=4001 min_rank=3 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16361 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[453, 514, 522, 524, 524, 502, 509, 487, 0, 0, 0, 0, 0, 0, 0, 0, 534, 494, 504, 478, 510, 548, 521, 520, 16, 0, 0, 0, 0, 0, 0, 0, 539, 481, 480, 496, 526, 459, 494, 488, 3, 0, 0, 0, 0, 0, 0, 0, 518, 538, 521, 546, 510, 547, 505, 527, 0, 0, 0, 0, 0, 0, 0, 0, 487, 491, 483, 512, 514, 493, 513, 495, 6, 1, 0, 0, 0, 0, 0, 0, 500, 522, 528, 546, 524, 499, 533, 516, 8, 5, 0, 0, 0, 0, 0, 0, 517, 517, 502, 491, 545, 520, 530, 506, 0, 0, 0, 0, 0, 0, 0, 0, 509, 556, 500, 511, 535, 481, 491, 543, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4035, 4125, 3966, 4212, 3995, 4181, 4128, 4126] avg=4096.0000 max=4212 max_rank=3 min=3966 min_rank=2 imbalance_ratio=1.028320 max_load_reduction=28556 max_load_reduction_pct=0.871460 moved_to_echo=16430 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[501, 567, 499, 519, 516, 482, 546, 502, 0, 0, 0, 0, 0, 0, 0, 0, 493, 519, 489, 515, 482, 504, 525, 501, 16, 0, 0, 0, 0, 0, 0, 0, 531, 518, 510, 513, 509, 553, 507, 512, 3, 0, 0, 0, 0, 0, 0, 0, 497, 515, 481, 506, 537, 501, 521, 494, 0, 0, 0, 0, 0, 0, 0, 0, 536, 495, 479, 515, 539, 503, 493, 506, 6, 1, 0, 0, 0, 0, 0, 0, 512, 497, 517, 514, 494, 480, 509, 510, 9, 5, 0, 0, 0, 0, 0, 0, 516, 545, 516, 518, 510, 542, 518, 513, 0, 0, 0, 0, 0, 0, 0, 0, 486, 503, 515, 564, 513, 494, 508, 503, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4132, 4044, 4156, 4052, 4073, 4047, 4178, 4086] avg=4096.0000 max=4178 max_rank=6 min=4044 min_rank=1 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[524, 497, 531, 499, 519, 533, 511, 493, 0, 0, 0, 0, 0, 0, 0, 0, 519, 495, 465, 494, 519, 519, 473, 489, 20, 0, 0, 0, 0, 0, 0, 0, 506, 496, 491, 540, 520, 499, 499, 486, 9, 0, 0, 0, 0, 0, 0, 0, 516, 540, 541, 561, 513, 491, 540, 520, 0, 0, 0, 0, 0, 0, 0, 0, 510, 538, 524, 498, 532, 510, 513, 481, 10, 6, 0, 0, 0, 0, 0, 0, 493, 514, 502, 489, 471, 533, 519, 535, 12, 8, 0, 0, 0, 0, 0, 0, 488, 540, 525, 492, 518, 503, 545, 505, 0, 0, 0, 0, 0, 0, 0, 0, 543, 498, 499, 505, 520, 527, 512, 480, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 3993, 4046, 4222, 4122, 4076, 4116, 4086] avg=4096.0000 max=4222 max_rank=3 min=3993 min_rank=1 imbalance_ratio=1.030762 max_load_reduction=28546 max_load_reduction_pct=0.871155 moved_to_echo=16400 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[477, 513, 505, 557, 500, 504, 508, 513, 0, 0, 0, 0, 0, 0, 0, 0, 525, 505, 495, 507, 500, 539, 475, 499, 14, 0, 0, 0, 0, 0, 0, 0, 523, 508, 551, 486, 524, 541, 529, 511, 3, 0, 0, 0, 0, 0, 0, 0, 528, 537, 485, 497, 472, 521, 490, 509, 0, 0, 0, 0, 0, 0, 0, 0, 521, 496, 540, 507, 521, 510, 525, 489, 6, 1, 0, 0, 0, 0, 0, 0, 500, 556, 498, 508, 509, 550, 484, 485, 9, 4, 0, 0, 0, 0, 0, 0, 508, 525, 517, 493, 489, 511, 487, 525, 0, 0, 0, 0, 0, 0, 0, 0, 503, 572, 488, 519, 507, 527, 519, 508, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4059, 4176, 4039, 4116, 4103, 4055, 4143] avg=4096.0000 max=4176 max_rank=2 min=4039 min_rank=3 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16417 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[546, 500, 512, 551, 536, 510, 540, 552, 0, 0, 0, 0, 0, 0, 0, 0, 521, 491, 528, 510, 525, 460, 521, 527, 15, 0, 0, 0, 0, 0, 0, 0, 528, 524, 511, 508, 521, 510, 513, 496, 3, 0, 0, 0, 0, 0, 0, 0, 525, 507, 485, 534, 484, 485, 501, 507, 0, 0, 0, 0, 0, 0, 0, 0, 509, 511, 522, 543, 506, 509, 500, 499, 6, 1, 0, 0, 0, 0, 0, 0, 494, 530, 516, 544, 518, 484, 468, 533, 10, 5, 0, 0, 0, 0, 0, 0, 531, 512, 518, 491, 545, 478, 504, 477, 0, 0, 0, 0, 0, 0, 0, 0, 501, 506, 513, 469, 515, 518, 517, 478, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4247, 4098, 4114, 4028, 4106, 4102, 4056, 4017] avg=4096.0000 max=4247 max_rank=0 min=4017 min_rank=7 imbalance_ratio=1.036865 max_load_reduction=28521 max_load_reduction_pct=0.870392 moved_to_echo=16281 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[503, 518, 456, 487, 518, 516, 509, 509, 0, 0, 0, 0, 0, 0, 0, 0, 572, 525, 528, 532, 478, 502, 512, 498, 16, 0, 0, 0, 0, 0, 0, 0, 489, 489, 476, 501, 528, 506, 530, 537, 3, 0, 0, 0, 0, 0, 0, 0, 523, 511, 532, 501, 521, 538, 497, 478, 0, 0, 0, 0, 0, 0, 0, 0, 521, 520, 501, 484, 511, 542, 552, 533, 6, 1, 0, 0, 0, 0, 0, 0, 516, 539, 484, 513, 510, 533, 519, 478, 9, 5, 0, 0, 0, 0, 0, 0, 489, 501, 509, 549, 522, 519, 477, 501, 0, 0, 0, 0, 0, 0, 0, 0, 508, 570, 528, 460, 500, 506, 507, 506, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4016, 4163, 4059, 4101, 4171, 4106, 4067, 4085] avg=4096.0000 max=4171 max_rank=4 min=4016 min_rank=0 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4352, 4096, 4224, 4096, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[518, 505, 509, 508, 502, 513, 505, 519, 523, 545, 528, 481, 516, 497, 515, 507, 526, 503, 466, 515, 536, 502, 514, 535, 483, 518, 495, 538, 508, 491, 471, 499, 492, 488, 495, 528, 532, 529, 509, 477, 515, 498, 534, 529, 544, 509, 546, 532, 521, 506, 521, 510, 533, 529, 483, 507, 531, 526, 540, 501, 475, 500, 502, 535] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[515, 517, 513, 523, 540, 504, 528, 517, 489, 505, 512, 534, 510, 505, 512, 520, 520, 497, 542, 531, 514, 532, 493, 542, 492, 538, 505, 491, 535, 487, 476, 538, 518, 534, 480, 546, 522, 522, 480, 493, 551, 579, 499, 498, 494, 499, 475, 539, 523, 478, 502, 470, 480, 530, 539, 489, 513, 483, 523, 539, 504, 493, 482, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[501, 534, 532, 526, 528, 534, 532, 508, 515, 495, 536, 496, 517, 535, 510, 521, 489, 468, 508, 508, 488, 532, 512, 481, 507, 543, 520, 461, 514, 552, 533, 518, 508, 538, 523, 542, 501, 525, 494, 504, 496, 530, 518, 507, 496, 477, 471, 496, 506, 512, 522, 507, 494, 552, 513, 535, 544, 499, 520, 513, 481, 510, 464, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[519, 497, 490, 502, 530, 522, 491, 480, 506, 508, 553, 514, 552, 506, 502, 472, 520, 511, 520, 513, 509, 522, 508, 489, 522, 517, 503, 498, 491, 513, 541, 535, 530, 518, 542, 510, 544, 472, 486, 508, 529, 494, 526, 501, 503, 527, 523, 529, 510, 488, 505, 514, 491, 514, 485, 515, 508, 493, 505, 535, 534, 537, 499, 537] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[488, 527, 506, 550, 508, 500, 507, 484, 492, 563, 489, 519, 526, 506, 520, 533, 498, 513, 517, 486, 520, 519, 499, 518, 526, 491, 518, 505, 514, 502, 514, 519, 560, 521, 490, 504, 486, 522, 486, 517, 526, 503, 481, 460, 493, 486, 510, 545, 511, 487, 535, 550, 521, 517, 515, 519, 555, 496, 546, 495, 513, 537, 519, 485] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[498, 528, 514, 532, 510, 505, 505, 504, 523, 485, 511, 529, 504, 506, 519, 521, 487, 537, 501, 521, 515, 464, 520, 503, 514, 503, 518, 500, 547, 469, 513, 496, 499, 528, 529, 501, 455, 504, 524, 532, 493, 518, 546, 509, 526, 501, 531, 524, 496, 489, 506, 504, 557, 545, 522, 543, 511, 504, 495, 521, 531, 527, 498, 497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[545, 509, 481, 539, 507, 517, 505, 522, 506, 523, 542, 563, 546, 496, 526, 509, 517, 538, 504, 524, 508, 542, 537, 498, 487, 489, 483, 501, 516, 498, 500, 486, 485, 523, 501, 518, 482, 482, 523, 497, 491, 487, 511, 499, 506, 530, 509, 503, 511, 513, 536, 538, 537, 504, 530, 503, 470, 529, 513, 520, 534, 537, 482, 500] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[507, 528, 518, 498, 512, 480, 518, 527, 518, 533, 532, 499, 527, 507, 511, 513, 494, 480, 516, 549, 537, 501, 539, 502, 497, 489, 500, 531, 484, 520, 512, 540, 532, 536, 528, 517, 522, 480, 547, 519, 535, 509, 499, 503, 504, 501, 475, 466, 542, 523, 489, 500, 530, 497, 497, 533, 499, 534, 522, 492, 518, 493, 505, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[501, 534, 532, 518, 528, 534, 532, 508, 0, 0, 0, 0, 0, 0, 0, 0, 515, 495, 503, 496, 517, 535, 510, 521, 0, 0, 0, 0, 0, 0, 0, 0, 489, 468, 508, 508, 488, 532, 512, 481, 2, 0, 0, 0, 0, 0, 0, 0, 507, 543, 520, 461, 514, 552, 533, 518, 32, 0, 0, 0, 0, 0, 0, 0, 508, 538, 523, 542, 501, 525, 494, 504, 3, 0, 0, 0, 0, 0, 0, 0, 496, 530, 518, 507, 496, 477, 471, 496, 14, 1, 0, 0, 0, 0, 0, 0, 506, 512, 522, 507, 494, 538, 513, 535, 0, 0, 0, 0, 0, 0, 0, 0, 544, 499, 520, 513, 481, 510, 464, 516, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4187, 4092, 3988, 4180, 4138, 4006, 4127, 4050] avg=4096.0000 max=4187 max_rank=0 min=3988 min_rank=2 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16321 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[518, 505, 509, 500, 502, 513, 505, 519, 0, 0, 0, 0, 0, 0, 0, 0, 523, 545, 496, 481, 516, 497, 515, 507, 0, 0, 0, 0, 0, 0, 0, 0, 526, 503, 466, 515, 536, 502, 514, 535, 2, 0, 0, 0, 0, 0, 0, 0, 483, 518, 495, 538, 508, 491, 471, 499, 31, 0, 0, 0, 0, 0, 0, 0, 492, 488, 495, 528, 532, 529, 509, 477, 3, 0, 0, 0, 0, 0, 0, 0, 515, 498, 534, 529, 544, 509, 546, 532, 14, 1, 0, 0, 0, 0, 0, 0, 521, 506, 521, 510, 533, 515, 483, 507, 0, 0, 0, 0, 0, 0, 0, 0, 531, 526, 540, 501, 475, 500, 502, 535, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4071, 4080, 4099, 4034, 4053, 4222, 4096, 4113] avg=4096.0000 max=4222 max_rank=5 min=4034 min_rank=3 imbalance_ratio=1.030762 max_load_reduction=28546 max_load_reduction_pct=0.871155 moved_to_echo=16484 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[515, 517, 513, 515, 540, 504, 528, 517, 0, 0, 0, 0, 0, 0, 0, 0, 489, 505, 481, 534, 510, 505, 512, 520, 0, 0, 0, 0, 0, 0, 0, 0, 520, 497, 542, 531, 514, 532, 493, 542, 2, 0, 0, 0, 0, 0, 0, 0, 492, 538, 505, 491, 535, 487, 476, 538, 30, 0, 0, 0, 0, 0, 0, 0, 518, 534, 480, 546, 522, 522, 480, 493, 3, 0, 0, 0, 0, 0, 0, 0, 551, 579, 499, 498, 494, 499, 475, 539, 14, 1, 0, 0, 0, 0, 0, 0, 523, 478, 502, 470, 480, 516, 539, 489, 0, 0, 0, 0, 0, 0, 0, 0, 513, 483, 523, 539, 504, 493, 482, 514, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4149, 4056, 4173, 4092, 4098, 4149, 3997, 4054] avg=4096.0000 max=4173 max_rank=2 min=3997 min_rank=6 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16298 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[519, 497, 490, 485, 530, 522, 491, 480, 0, 0, 0, 0, 0, 0, 0, 0, 506, 508, 510, 514, 552, 506, 502, 472, 0, 0, 0, 0, 0, 0, 0, 0, 520, 511, 520, 513, 509, 522, 508, 489, 4, 0, 0, 0, 0, 0, 0, 0, 522, 517, 503, 498, 491, 513, 541, 535, 37, 0, 0, 0, 0, 0, 0, 0, 530, 518, 542, 510, 544, 472, 486, 508, 7, 0, 0, 0, 0, 0, 0, 0, 529, 494, 526, 501, 503, 527, 523, 529, 16, 6, 0, 0, 0, 0, 0, 0, 510, 488, 505, 514, 491, 493, 485, 515, 0, 0, 0, 0, 0, 0, 0, 0, 508, 493, 505, 535, 534, 537, 499, 537, 6, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4014, 4070, 4096, 4157, 4117, 4154, 4001, 4159] avg=4096.0000 max=4159 max_rank=7 min=4001 min_rank=6 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[488, 527, 506, 542, 508, 500, 507, 484, 0, 0, 0, 0, 0, 0, 0, 0, 492, 563, 459, 519, 526, 506, 520, 533, 0, 0, 0, 0, 0, 0, 0, 0, 498, 513, 517, 486, 520, 519, 499, 518, 2, 0, 0, 0, 0, 0, 0, 0, 526, 491, 518, 505, 514, 502, 514, 519, 29, 0, 0, 0, 0, 0, 0, 0, 560, 521, 490, 504, 486, 522, 486, 517, 3, 0, 0, 0, 0, 0, 0, 0, 526, 503, 481, 460, 493, 486, 510, 545, 13, 1, 0, 0, 0, 0, 0, 0, 511, 487, 535, 550, 521, 504, 515, 519, 0, 0, 0, 0, 0, 0, 0, 0, 555, 496, 546, 495, 513, 537, 519, 485, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4062, 4118, 4072, 4118, 4089, 4018, 4142, 4149] avg=4096.0000 max=4149 max_rank=7 min=4018 min_rank=5 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[545, 509, 481, 531, 507, 517, 505, 522, 0, 0, 0, 0, 0, 0, 0, 0, 506, 523, 509, 563, 546, 496, 526, 509, 0, 0, 0, 0, 0, 0, 0, 0, 517, 538, 504, 524, 508, 542, 537, 498, 2, 0, 0, 0, 0, 0, 0, 0, 487, 489, 483, 501, 516, 498, 500, 486, 32, 0, 0, 0, 0, 0, 0, 0, 485, 523, 501, 518, 482, 482, 523, 497, 3, 0, 0, 0, 0, 0, 0, 0, 491, 487, 511, 499, 506, 530, 509, 503, 13, 1, 0, 0, 0, 0, 0, 0, 511, 513, 536, 538, 537, 491, 530, 503, 0, 0, 0, 0, 0, 0, 0, 0, 470, 529, 513, 520, 534, 537, 482, 500, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4117, 4178, 4170, 3992, 4014, 4050, 4159, 4088] avg=4096.0000 max=4178 max_rank=1 min=3992 min_rank=3 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16311 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[498, 528, 514, 524, 510, 505, 505, 504, 0, 0, 0, 0, 0, 0, 0, 0, 523, 485, 480, 529, 504, 506, 519, 521, 0, 0, 0, 0, 0, 0, 0, 0, 487, 537, 501, 521, 515, 464, 520, 503, 2, 0, 0, 0, 0, 0, 0, 0, 514, 503, 518, 500, 547, 469, 513, 496, 30, 0, 0, 0, 0, 0, 0, 0, 499, 528, 529, 501, 455, 504, 524, 532, 3, 0, 0, 0, 0, 0, 0, 0, 493, 518, 546, 509, 526, 501, 531, 524, 14, 1, 0, 0, 0, 0, 0, 0, 496, 489, 506, 504, 557, 531, 522, 543, 0, 0, 0, 0, 0, 0, 0, 0, 511, 504, 495, 521, 531, 527, 498, 497, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4067, 4050, 4090, 4075, 4163, 4148, 4087] avg=4096.0000 max=4163 max_rank=5 min=4050 min_rank=2 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16473 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[507, 528, 518, 490, 512, 480, 518, 527, 0, 0, 0, 0, 0, 0, 0, 0, 518, 533, 499, 499, 527, 507, 511, 513, 0, 0, 0, 0, 0, 0, 0, 0, 494, 480, 516, 549, 537, 501, 539, 502, 2, 0, 0, 0, 0, 0, 0, 0, 497, 489, 500, 531, 484, 520, 512, 540, 32, 0, 0, 0, 0, 0, 0, 0, 532, 536, 528, 517, 522, 480, 547, 519, 3, 0, 0, 0, 0, 0, 0, 0, 535, 509, 499, 503, 504, 501, 475, 466, 13, 1, 0, 0, 0, 0, 0, 0, 542, 523, 489, 500, 530, 484, 497, 533, 0, 0, 0, 0, 0, 0, 0, 0, 499, 534, 522, 492, 518, 493, 505, 502, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4107, 4120, 4105, 4184, 4006, 4098, 4068] avg=4096.0000 max=4184 max_rank=4 min=4006 min_rank=5 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16356 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 3968, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 3968, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[511, 487, 560, 514, 533, 532, 526, 498, 549, 510, 530, 514, 553, 514, 468, 496, 533, 475, 525, 519, 517, 555, 475, 503, 506, 536, 493, 506, 520, 497, 546, 518, 499, 503, 508, 481, 503, 520, 532, 530, 531, 519, 523, 524, 518, 467, 531, 519, 476, 476, 474, 508, 461, 504, 531, 501, 557, 467, 511, 521, 496, 520, 516, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[527, 492, 539, 570, 486, 535, 514, 520, 500, 525, 537, 508, 505, 515, 542, 483, 549, 507, 505, 511, 517, 552, 521, 514, 514, 504, 521, 526, 487, 530, 523, 523, 482, 529, 527, 517, 520, 495, 520, 526, 518, 500, 504, 476, 511, 517, 532, 452, 537, 530, 490, 503, 522, 468, 471, 496, 492, 501, 507, 514, 490, 511, 517, 491] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[491, 563, 475, 515, 528, 498, 514, 472, 547, 511, 526, 513, 491, 527, 504, 512, 506, 525, 500, 506, 518, 513, 532, 571, 479, 505, 526, 497, 508, 486, 515, 535, 533, 504, 510, 493, 522, 514, 546, 499, 525, 502, 517, 514, 523, 494, 522, 500, 506, 518, 508, 489, 505, 491, 507, 531, 516, 539, 476, 504, 477, 539, 511, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[505, 547, 489, 501, 538, 510, 503, 508, 478, 527, 493, 535, 524, 496, 515, 547, 526, 534, 501, 530, 527, 486, 522, 507, 527, 480, 517, 487, 539, 520, 510, 511, 507, 539, 499, 501, 530, 512, 503, 533, 521, 543, 460, 542, 540, 511, 541, 491, 484, 504, 463, 488, 484, 522, 519, 521, 505, 511, 512, 511, 501, 495, 513, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[483, 550, 519, 483, 530, 485, 509, 529, 482, 530, 539, 525, 460, 533, 554, 501, 469, 496, 530, 533, 546, 490, 514, 530, 512, 510, 490, 532, 541, 488, 509, 506, 484, 527, 560, 497, 472, 536, 523, 476, 488, 484, 525, 507, 498, 530, 523, 524, 521, 522, 539, 497, 516, 471, 525, 532, 487, 505, 458, 519, 512, 511, 554, 537] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[491, 459, 494, 474, 480, 514, 542, 502, 489, 505, 498, 497, 546, 505, 500, 511, 504, 516, 525, 481, 517, 503, 544, 544, 539, 496, 505, 537, 528, 500, 535, 546, 526, 501, 494, 497, 518, 519, 553, 512, 515, 495, 476, 543, 526, 496, 524, 516, 531, 538, 525, 509, 503, 514, 538, 490, 472, 460, 561, 518, 493, 503, 543, 532] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[497, 530, 539, 506, 524, 491, 522, 515, 489, 498, 538, 495, 485, 513, 505, 505, 516, 495, 478, 501, 491, 506, 485, 500, 526, 503, 571, 502, 516, 570, 514, 532, 492, 507, 514, 461, 536, 533, 515, 505, 549, 494, 512, 524, 552, 484, 499, 504, 527, 501, 473, 512, 509, 502, 539, 516, 522, 508, 496, 549, 503, 553, 495, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[515, 534, 509, 500, 547, 501, 509, 502, 518, 521, 496, 510, 538, 504, 542, 502, 557, 489, 537, 474, 480, 519, 521, 523, 479, 492, 519, 488, 527, 481, 498, 530, 495, 509, 488, 515, 535, 514, 508, 527, 500, 497, 524, 549, 505, 503, 518, 500, 531, 521, 545, 529, 485, 530, 496, 497, 539, 507, 531, 523, 511, 489, 503, 482] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[491, 563, 475, 515, 526, 498, 514, 472, 0, 0, 0, 0, 0, 0, 0, 0, 547, 511, 520, 513, 491, 527, 504, 512, 0, 0, 0, 0, 0, 0, 0, 0, 506, 525, 500, 506, 518, 513, 532, 554, 0, 0, 0, 0, 0, 0, 0, 0, 479, 505, 526, 497, 508, 486, 515, 517, 0, 0, 0, 0, 0, 0, 0, 0, 533, 504, 510, 493, 522, 514, 544, 499, 0, 0, 0, 0, 0, 0, 0, 0, 525, 502, 517, 514, 523, 494, 522, 500, 0, 0, 0, 0, 0, 0, 0, 0, 506, 518, 508, 489, 505, 491, 507, 531, 18, 17, 2, 0, 0, 0, 0, 0, 516, 539, 476, 504, 477, 539, 511, 524, 4, 2, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4054, 4125, 4154, 4033, 4119, 4097, 4092, 4094] avg=4096.0000 max=4154 max_rank=2 min=4033 min_rank=3 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16402 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[483, 550, 519, 483, 528, 485, 509, 529, 0, 0, 0, 0, 0, 0, 0, 0, 482, 530, 532, 525, 460, 533, 554, 501, 0, 0, 0, 0, 0, 0, 0, 0, 469, 496, 530, 533, 546, 490, 514, 514, 0, 0, 0, 0, 0, 0, 0, 0, 512, 510, 490, 532, 541, 488, 509, 489, 0, 0, 0, 0, 0, 0, 0, 0, 484, 527, 560, 497, 472, 536, 521, 476, 0, 0, 0, 0, 0, 0, 0, 0, 488, 484, 525, 507, 498, 530, 523, 524, 0, 0, 0, 0, 0, 0, 0, 0, 521, 522, 539, 497, 516, 471, 525, 532, 17, 16, 2, 0, 0, 0, 0, 0, 487, 505, 458, 519, 512, 511, 554, 537, 5, 2, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4117, 4092, 4071, 4073, 4079, 4158, 4092] avg=4096.0000 max=4158 max_rank=6 min=4071 min_rank=3 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16402 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[511, 487, 560, 514, 531, 532, 526, 498, 0, 0, 0, 0, 0, 0, 0, 0, 549, 510, 524, 514, 553, 514, 468, 496, 0, 0, 0, 0, 0, 0, 0, 0, 533, 475, 525, 519, 517, 555, 475, 488, 0, 0, 0, 0, 0, 0, 0, 0, 506, 536, 493, 506, 520, 497, 546, 500, 0, 0, 0, 0, 0, 0, 0, 0, 499, 503, 508, 481, 503, 520, 530, 530, 0, 0, 0, 0, 0, 0, 0, 0, 531, 519, 523, 524, 518, 467, 531, 519, 0, 0, 0, 0, 0, 0, 0, 0, 476, 476, 474, 508, 461, 504, 531, 501, 18, 15, 2, 0, 0, 0, 0, 0, 557, 467, 511, 521, 496, 520, 516, 522, 4, 2, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4159, 4128, 4087, 4104, 4074, 4132, 3966, 4118] avg=4096.0000 max=4159 max_rank=0 min=3966 min_rank=6 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16290 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[491, 459, 494, 474, 478, 514, 542, 502, 0, 0, 0, 0, 0, 0, 0, 0, 489, 505, 492, 497, 546, 505, 500, 511, 0, 0, 0, 0, 0, 0, 0, 0, 504, 516, 525, 481, 517, 503, 544, 528, 0, 0, 0, 0, 0, 0, 0, 0, 539, 496, 505, 537, 528, 500, 535, 528, 0, 0, 0, 0, 0, 0, 0, 0, 526, 501, 494, 497, 518, 519, 551, 512, 0, 0, 0, 0, 0, 0, 0, 0, 515, 495, 476, 543, 526, 496, 524, 516, 0, 0, 0, 0, 0, 0, 0, 0, 531, 538, 525, 509, 503, 514, 538, 490, 18, 16, 2, 0, 0, 0, 0, 0, 472, 460, 561, 518, 493, 503, 543, 532, 4, 2, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3954, 4045, 4118, 4168, 4118, 4091, 4184, 4090] avg=4096.0000 max=4184 max_rank=6 min=3954 min_rank=0 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16483 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[527, 492, 539, 570, 484, 535, 514, 520, 0, 0, 0, 0, 0, 0, 0, 0, 500, 525, 530, 508, 505, 515, 542, 483, 0, 0, 0, 0, 0, 0, 0, 0, 549, 507, 505, 511, 517, 552, 521, 499, 0, 0, 0, 0, 0, 0, 0, 0, 514, 504, 521, 526, 487, 530, 523, 505, 0, 0, 0, 0, 0, 0, 0, 0, 482, 529, 527, 517, 520, 495, 518, 526, 0, 0, 0, 0, 0, 0, 0, 0, 518, 500, 504, 476, 511, 517, 532, 452, 0, 0, 0, 0, 0, 0, 0, 0, 537, 530, 490, 503, 522, 468, 471, 496, 18, 15, 2, 0, 0, 0, 0, 0, 492, 501, 507, 514, 490, 511, 517, 491, 5, 2, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4181, 4108, 4161, 4110, 4114, 4010, 4052, 4032] avg=4096.0000 max=4181 max_rank=0 min=4010 min_rank=5 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16208 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[505, 547, 489, 501, 534, 510, 503, 508, 0, 0, 0, 0, 0, 0, 0, 0, 478, 527, 477, 535, 524, 496, 515, 547, 0, 0, 0, 0, 0, 0, 0, 0, 526, 534, 501, 530, 527, 486, 522, 488, 0, 0, 0, 0, 0, 0, 0, 0, 527, 480, 517, 487, 539, 520, 510, 490, 0, 0, 0, 0, 0, 0, 0, 0, 507, 539, 499, 501, 530, 512, 499, 533, 0, 0, 0, 0, 0, 0, 0, 0, 521, 543, 460, 542, 540, 511, 537, 491, 0, 0, 0, 0, 0, 0, 0, 0, 484, 504, 463, 488, 484, 522, 519, 521, 21, 19, 8, 0, 0, 0, 0, 0, 505, 511, 512, 511, 501, 495, 513, 522, 8, 4, 4, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4099, 4114, 4070, 4120, 4145, 4033, 4090] avg=4096.0000 max=4145 max_rank=5 min=4033 min_rank=6 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[497, 530, 539, 506, 522, 491, 522, 515, 0, 0, 0, 0, 0, 0, 0, 0, 489, 498, 531, 495, 485, 513, 505, 505, 0, 0, 0, 0, 0, 0, 0, 0, 516, 495, 478, 501, 491, 506, 485, 485, 0, 0, 0, 0, 0, 0, 0, 0, 526, 503, 571, 502, 516, 570, 514, 514, 0, 0, 0, 0, 0, 0, 0, 0, 492, 507, 514, 461, 536, 533, 513, 505, 0, 0, 0, 0, 0, 0, 0, 0, 549, 494, 512, 524, 552, 484, 499, 504, 0, 0, 0, 0, 0, 0, 0, 0, 527, 501, 473, 512, 509, 502, 539, 516, 18, 15, 2, 0, 0, 0, 0, 0, 522, 508, 496, 549, 503, 553, 495, 524, 5, 2, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4122, 4021, 3957, 4216, 4061, 4118, 4114, 4159] avg=4096.0000 max=4216 max_rank=3 min=3957 min_rank=2 imbalance_ratio=1.029297 max_load_reduction=28552 max_load_reduction_pct=0.871338 moved_to_echo=16452 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[515, 534, 509, 500, 545, 501, 509, 502, 0, 0, 0, 0, 0, 0, 0, 0, 518, 521, 490, 510, 538, 504, 542, 502, 0, 0, 0, 0, 0, 0, 0, 0, 557, 489, 537, 474, 480, 519, 521, 508, 0, 0, 0, 0, 0, 0, 0, 0, 479, 492, 519, 488, 527, 481, 498, 512, 0, 0, 0, 0, 0, 0, 0, 0, 495, 509, 488, 515, 535, 514, 506, 527, 0, 0, 0, 0, 0, 0, 0, 0, 500, 497, 524, 549, 505, 503, 518, 500, 0, 0, 0, 0, 0, 0, 0, 0, 531, 521, 545, 529, 485, 530, 496, 497, 18, 15, 2, 0, 0, 0, 0, 0, 539, 507, 531, 523, 511, 489, 503, 482, 4, 2, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4125, 4085, 3996, 4089, 4096, 4169, 4093] avg=4096.0000 max=4169 max_rank=6 min=3996 min_rank=3 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16447 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 3968, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 128, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33792 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[504, 529, 570, 501, 526, 513, 507, 492, 506, 525, 518, 519, 490, 498, 490, 502, 486, 513, 524, 511, 534, 531, 523, 489, 494, 539, 501, 508, 486, 499, 512, 544, 467, 506, 536, 554, 483, 482, 514, 562, 492, 527, 471, 508, 485, 513, 526, 547, 524, 533, 571, 491, 501, 522, 529, 528, 486, 485, 490, 514, 528, 477, 510, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[506, 502, 513, 511, 519, 532, 500, 497, 504, 487, 517, 507, 529, 481, 494, 521, 498, 515, 510, 525, 513, 510, 490, 492, 488, 507, 518, 517, 528, 513, 508, 522, 514, 525, 571, 519, 512, 562, 477, 518, 513, 482, 539, 495, 509, 509, 532, 479, 497, 532, 543, 508, 515, 507, 521, 486, 526, 482, 538, 520, 518, 504, 521, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[518, 509, 548, 525, 510, 518, 506, 504, 517, 509, 511, 506, 530, 531, 493, 499, 517, 514, 551, 545, 539, 507, 509, 540, 513, 536, 481, 469, 510, 501, 540, 503, 529, 535, 466, 558, 513, 519, 485, 480, 507, 479, 535, 481, 547, 475, 522, 474, 509, 509, 517, 504, 506, 473, 518, 487, 536, 570, 533, 497, 520, 481, 489, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[525, 451, 507, 525, 492, 511, 478, 531, 511, 517, 483, 527, 568, 481, 486, 512, 535, 513, 527, 504, 535, 523, 522, 512, 519, 490, 507, 515, 525, 479, 520, 525, 511, 518, 524, 525, 473, 540, 506, 483, 517, 505, 515, 490, 511, 524, 520, 516, 524, 514, 535, 504, 495, 537, 512, 530, 514, 522, 515, 521, 510, 490, 486, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[529, 482, 534, 527, 483, 530, 513, 543, 484, 533, 524, 497, 526, 525, 483, 494, 534, 506, 521, 528, 498, 497, 567, 516, 472, 488, 498, 539, 522, 489, 510, 508, 516, 526, 526, 501, 479, 534, 481, 533, 532, 533, 509, 490, 503, 531, 490, 521, 500, 521, 474, 500, 513, 487, 510, 502, 504, 564, 523, 522, 530, 508, 530, 475] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[516, 495, 515, 499, 506, 517, 533, 545, 508, 553, 486, 535, 502, 479, 486, 517, 498, 499, 519, 540, 498, 490, 513, 508, 488, 523, 539, 489, 536, 503, 541, 534, 484, 519, 500, 528, 552, 514, 499, 470, 498, 502, 545, 535, 503, 519, 530, 523, 495, 494, 513, 542, 513, 527, 518, 480, 481, 494, 506, 506, 533, 531, 510, 494] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[471, 495, 496, 519, 537, 501, 493, 530, 509, 518, 534, 519, 514, 480, 513, 524, 523, 514, 550, 547, 511, 495, 521, 513, 512, 509, 490, 553, 521, 500, 493, 523, 546, 535, 522, 511, 482, 494, 522, 491, 499, 485, 506, 555, 512, 492, 510, 553, 462, 519, 490, 468, 473, 515, 539, 515, 489, 509, 533, 527, 516, 516, 532, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[545, 534, 491, 526, 545, 541, 506, 505, 524, 529, 521, 523, 486, 500, 531, 510, 534, 547, 490, 503, 529, 522, 489, 463, 499, 524, 539, 490, 533, 485, 508, 539, 531, 537, 528, 558, 489, 505, 531, 465, 542, 496, 492, 504, 473, 479, 492, 529, 495, 501, 516, 501, 529, 484, 514, 539, 523, 495, 530, 500, 516, 478, 512, 473] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[506, 502, 500, 511, 519, 532, 500, 497, 0, 0, 0, 0, 0, 0, 0, 0, 504, 487, 517, 507, 529, 481, 494, 521, 13, 2, 0, 0, 0, 0, 0, 0, 498, 515, 510, 492, 513, 510, 490, 492, 0, 0, 0, 0, 0, 0, 0, 0, 488, 507, 518, 517, 528, 513, 508, 522, 6, 0, 0, 0, 0, 0, 0, 0, 514, 525, 571, 508, 512, 562, 477, 518, 0, 0, 0, 0, 0, 0, 0, 0, 513, 482, 539, 495, 509, 509, 532, 479, 17, 0, 0, 0, 0, 0, 0, 0, 497, 532, 543, 508, 515, 507, 521, 486, 16, 0, 0, 0, 0, 0, 0, 0, 526, 482, 538, 520, 518, 504, 521, 520, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4055, 4020, 4107, 4187, 4075, 4125, 4132] avg=4096.0000 max=4187 max_rank=4 min=4020 min_rank=2 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16519 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[504, 529, 556, 501, 526, 513, 507, 492, 0, 0, 0, 0, 0, 0, 0, 0, 506, 525, 518, 519, 490, 498, 490, 502, 14, 2, 0, 0, 0, 0, 0, 0, 486, 513, 524, 478, 534, 531, 523, 489, 0, 0, 0, 0, 0, 0, 0, 0, 494, 539, 501, 508, 486, 499, 512, 544, 7, 0, 0, 0, 0, 0, 0, 0, 467, 506, 536, 541, 483, 482, 514, 562, 0, 0, 0, 0, 0, 0, 0, 0, 492, 527, 471, 508, 485, 513, 526, 547, 17, 0, 0, 0, 0, 0, 0, 0, 524, 533, 571, 491, 501, 522, 529, 528, 16, 0, 0, 0, 0, 0, 0, 0, 486, 485, 490, 514, 528, 477, 510, 522, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4064, 4078, 4090, 4091, 4086, 4215, 4016] avg=4096.0000 max=4215 max_rank=6 min=4016 min_rank=7 imbalance_ratio=1.029053 max_load_reduction=28553 max_load_reduction_pct=0.871368 moved_to_echo=16408 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[525, 451, 494, 525, 492, 511, 478, 531, 0, 0, 0, 0, 0, 0, 0, 0, 511, 517, 483, 527, 568, 481, 486, 512, 13, 2, 0, 0, 0, 0, 0, 0, 535, 513, 527, 473, 535, 523, 522, 512, 0, 0, 0, 0, 0, 0, 0, 0, 519, 490, 507, 515, 525, 479, 520, 525, 6, 0, 0, 0, 0, 0, 0, 0, 511, 518, 524, 514, 473, 540, 506, 483, 0, 0, 0, 0, 0, 0, 0, 0, 517, 505, 515, 490, 511, 524, 520, 516, 16, 0, 0, 0, 0, 0, 0, 0, 524, 514, 535, 504, 495, 537, 512, 530, 15, 0, 0, 0, 0, 0, 0, 0, 514, 522, 515, 521, 510, 490, 486, 525, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4007, 4100, 4140, 4086, 4069, 4114, 4166, 4086] avg=4096.0000 max=4166 max_rank=6 min=4007 min_rank=0 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16435 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[518, 509, 534, 525, 510, 518, 506, 504, 0, 0, 0, 0, 0, 0, 0, 0, 517, 509, 511, 506, 530, 531, 493, 499, 14, 3, 0, 0, 0, 0, 0, 0, 517, 514, 551, 510, 539, 507, 509, 540, 0, 0, 0, 0, 0, 0, 0, 0, 513, 536, 481, 469, 510, 501, 540, 503, 7, 0, 0, 0, 0, 0, 0, 0, 529, 535, 466, 544, 513, 519, 485, 480, 0, 0, 0, 0, 0, 0, 0, 0, 507, 479, 535, 481, 547, 475, 522, 474, 18, 0, 0, 0, 0, 0, 0, 0, 509, 509, 517, 504, 506, 473, 518, 487, 17, 0, 0, 0, 0, 0, 0, 0, 536, 570, 533, 497, 520, 481, 489, 505, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4124, 4113, 4187, 4060, 4071, 4038, 4040, 4135] avg=4096.0000 max=4187 max_rank=2 min=4038 min_rank=5 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16284 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[516, 495, 502, 499, 506, 517, 533, 545, 0, 0, 0, 0, 0, 0, 0, 0, 508, 553, 486, 535, 502, 479, 486, 517, 13, 2, 0, 0, 0, 0, 0, 0, 498, 499, 519, 507, 498, 490, 513, 508, 0, 0, 0, 0, 0, 0, 0, 0, 488, 523, 539, 489, 536, 503, 541, 534, 6, 0, 0, 0, 0, 0, 0, 0, 484, 519, 500, 517, 552, 514, 499, 470, 0, 0, 0, 0, 0, 0, 0, 0, 498, 502, 545, 535, 503, 519, 530, 523, 17, 0, 0, 0, 0, 0, 0, 0, 495, 494, 513, 542, 513, 527, 518, 480, 16, 0, 0, 0, 0, 0, 0, 0, 481, 494, 506, 506, 533, 531, 510, 494, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4081, 4032, 4159, 4055, 4172, 4098, 4058] avg=4096.0000 max=4172 max_rank=5 min=4032 min_rank=2 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[471, 495, 474, 519, 537, 501, 493, 530, 0, 0, 0, 0, 0, 0, 0, 0, 509, 518, 534, 519, 514, 480, 513, 524, 17, 7, 0, 0, 0, 0, 0, 0, 523, 514, 550, 504, 511, 495, 521, 513, 0, 0, 0, 0, 0, 0, 0, 0, 512, 509, 490, 553, 521, 500, 493, 523, 9, 0, 0, 0, 0, 0, 0, 0, 546, 535, 522, 488, 482, 494, 522, 491, 0, 0, 0, 0, 0, 0, 0, 0, 499, 485, 506, 555, 512, 492, 510, 553, 22, 0, 0, 0, 0, 0, 0, 0, 462, 519, 490, 468, 473, 515, 539, 515, 21, 5, 0, 0, 0, 0, 0, 0, 489, 509, 533, 527, 516, 516, 532, 522, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4020, 4135, 4131, 4110, 4080, 4134, 4007, 4151] avg=4096.0000 max=4151 max_rank=7 min=4007 min_rank=6 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[529, 482, 521, 527, 483, 530, 513, 543, 0, 0, 0, 0, 0, 0, 0, 0, 484, 533, 524, 497, 526, 525, 483, 494, 13, 2, 0, 0, 0, 0, 0, 0, 534, 506, 521, 495, 498, 497, 567, 516, 0, 0, 0, 0, 0, 0, 0, 0, 472, 488, 498, 539, 522, 489, 510, 508, 6, 0, 0, 0, 0, 0, 0, 0, 516, 526, 526, 490, 479, 534, 481, 533, 0, 0, 0, 0, 0, 0, 0, 0, 532, 533, 509, 490, 503, 531, 490, 521, 17, 0, 0, 0, 0, 0, 0, 0, 500, 521, 474, 500, 513, 487, 510, 502, 16, 0, 0, 0, 0, 0, 0, 0, 504, 564, 523, 522, 530, 508, 530, 475, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4081, 4134, 4032, 4085, 4126, 4023, 4159] avg=4096.0000 max=4159 max_rank=7 min=4023 min_rank=6 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[545, 534, 479, 526, 545, 541, 506, 505, 0, 0, 0, 0, 0, 0, 0, 0, 524, 529, 521, 523, 486, 500, 531, 510, 12, 3, 0, 0, 0, 0, 0, 0, 534, 547, 490, 472, 529, 522, 489, 463, 0, 0, 0, 0, 0, 0, 0, 0, 499, 524, 539, 490, 533, 485, 508, 539, 7, 0, 0, 0, 0, 0, 0, 0, 531, 537, 528, 544, 489, 505, 531, 465, 0, 0, 0, 0, 0, 0, 0, 0, 542, 496, 492, 504, 473, 479, 492, 529, 16, 0, 0, 0, 0, 0, 0, 0, 495, 501, 516, 501, 529, 484, 514, 539, 15, 0, 0, 0, 0, 0, 0, 0, 523, 495, 530, 500, 516, 478, 512, 473, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4181, 4139, 4046, 4124, 4130, 4023, 4094, 4031] avg=4096.0000 max=4181 max_rank=0 min=4023 min_rank=5 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16278 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 3968, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[517, 517, 514, 505, 536, 500, 470, 469, 509, 527, 495, 505, 516, 499, 521, 533, 545, 518, 495, 544, 541, 498, 530, 549, 495, 501, 499, 514, 489, 532, 501, 562, 522, 503, 528, 511, 527, 512, 530, 475, 509, 521, 471, 516, 516, 521, 488, 492, 513, 492, 516, 491, 500, 524, 491, 534, 516, 505, 540, 459, 509, 540, 524, 526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[536, 497, 472, 507, 491, 525, 537, 524, 524, 504, 516, 506, 512, 511, 499, 496, 520, 500, 512, 531, 531, 497, 541, 520, 516, 518, 487, 441, 538, 506, 512, 508, 506, 509, 519, 550, 522, 508, 517, 476, 545, 541, 533, 519, 532, 510, 513, 511, 506, 501, 486, 487, 517, 493, 482, 547, 537, 502, 565, 501, 497, 481, 510, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[493, 489, 509, 573, 506, 545, 486, 499, 495, 487, 498, 454, 533, 505, 518, 511, 508, 523, 548, 502, 530, 506, 507, 500, 512, 496, 506, 507, 493, 519, 486, 508, 511, 491, 525, 520, 464, 510, 521, 517, 500, 514, 499, 513, 532, 548, 528, 527, 524, 532, 508, 493, 521, 533, 535, 528, 515, 512, 523, 510, 546, 488, 521, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[524, 496, 542, 498, 495, 497, 519, 480, 502, 507, 514, 503, 507, 515, 531, 512, 547, 514, 528, 499, 502, 486, 505, 490, 563, 525, 488, 491, 479, 499, 496, 515, 527, 530, 469, 549, 523, 501, 535, 539, 478, 502, 513, 506, 507, 506, 517, 526, 533, 518, 484, 519, 482, 492, 499, 522, 531, 514, 535, 550, 553, 517, 489, 533] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[543, 477, 552, 490, 470, 515, 466, 517, 499, 527, 537, 501, 505, 521, 495, 522, 490, 535, 495, 528, 503, 525, 526, 471, 506, 540, 507, 568, 502, 489, 499, 519, 507, 524, 500, 487, 498, 510, 486, 531, 514, 479, 507, 468, 550, 527, 556, 528, 509, 502, 532, 516, 508, 519, 491, 551, 508, 522, 501, 526, 514, 483, 549, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[493, 529, 505, 495, 522, 528, 482, 501, 544, 488, 516, 525, 474, 522, 493, 518, 527, 530, 510, 486, 565, 547, 498, 510, 506, 519, 516, 510, 495, 530, 520, 486, 525, 557, 489, 481, 507, 526, 478, 467, 502, 511, 515, 517, 535, 546, 500, 512, 502, 499, 524, 536, 511, 508, 561, 530, 549, 538, 504, 484, 492, 488, 488, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[530, 513, 498, 507, 531, 515, 476, 538, 514, 526, 513, 515, 546, 534, 514, 494, 532, 517, 495, 532, 490, 494, 511, 528, 558, 473, 511, 499, 517, 492, 480, 534, 515, 486, 498, 528, 477, 509, 531, 503, 512, 471, 476, 504, 509, 524, 508, 548, 528, 502, 494, 514, 547, 486, 511, 532, 493, 514, 514, 513, 513, 522, 532, 527] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[518, 505, 519, 475, 548, 494, 543, 551, 525, 528, 463, 505, 523, 519, 540, 536, 498, 535, 509, 511, 505, 513, 554, 499, 509, 520, 507, 478, 497, 491, 482, 512, 507, 505, 505, 545, 483, 502, 517, 542, 484, 506, 501, 541, 491, 535, 532, 499, 521, 477, 503, 529, 512, 488, 496, 526, 510, 511, 496, 501, 503, 527, 528, 533] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[517, 517, 514, 505, 536, 500, 470, 469, 19, 0, 0, 0, 0, 0, 0, 0, 509, 527, 495, 505, 516, 499, 521, 533, 2, 0, 0, 0, 0, 0, 0, 0, 545, 518, 495, 544, 541, 498, 496, 549, 0, 0, 0, 0, 0, 0, 0, 0, 495, 501, 499, 514, 489, 532, 501, 562, 34, 3, 0, 0, 0, 0, 0, 0, 522, 503, 528, 511, 527, 512, 530, 475, 9, 2, 0, 0, 0, 0, 0, 0, 509, 521, 471, 516, 516, 510, 488, 492, 0, 0, 0, 0, 0, 0, 0, 0, 513, 492, 516, 491, 500, 524, 491, 534, 0, 0, 0, 0, 0, 0, 0, 0, 516, 505, 516, 459, 509, 540, 524, 526, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4047, 4107, 4186, 4130, 4119, 4023, 4061, 4095] avg=4096.0000 max=4186 max_rank=2 min=4023 min_rank=5 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16298 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[493, 489, 509, 573, 506, 545, 486, 499, 19, 0, 0, 0, 0, 0, 0, 0, 495, 487, 498, 454, 533, 505, 518, 511, 2, 0, 0, 0, 0, 0, 0, 0, 508, 523, 548, 502, 530, 506, 475, 500, 0, 0, 0, 0, 0, 0, 0, 0, 512, 496, 506, 507, 493, 519, 486, 508, 32, 3, 0, 0, 0, 0, 0, 0, 511, 491, 525, 520, 464, 510, 521, 517, 9, 2, 0, 0, 0, 0, 0, 0, 500, 514, 499, 513, 532, 537, 528, 527, 0, 0, 0, 0, 0, 0, 0, 0, 524, 532, 508, 493, 521, 533, 535, 528, 0, 0, 0, 0, 0, 0, 0, 0, 515, 512, 499, 510, 546, 488, 521, 507, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4003, 4092, 4062, 4070, 4150, 4174, 4098] avg=4096.0000 max=4174 max_rank=6 min=4003 min_rank=1 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16492 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[536, 497, 472, 507, 491, 525, 537, 524, 20, 0, 0, 0, 0, 0, 0, 0, 524, 504, 516, 506, 512, 511, 499, 496, 2, 0, 0, 0, 0, 0, 0, 0, 520, 500, 512, 531, 531, 497, 507, 520, 0, 0, 0, 0, 0, 0, 0, 0, 516, 518, 487, 441, 538, 506, 512, 508, 34, 3, 0, 0, 0, 0, 0, 0, 506, 509, 519, 550, 522, 508, 517, 476, 8, 2, 0, 0, 0, 0, 0, 0, 545, 541, 533, 519, 532, 500, 513, 511, 0, 0, 0, 0, 0, 0, 0, 0, 506, 501, 486, 487, 517, 493, 482, 547, 0, 0, 0, 0, 0, 0, 0, 0, 537, 502, 540, 501, 497, 481, 510, 510, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4070, 4118, 4063, 4117, 4194, 4019, 4078] avg=4096.0000 max=4194 max_rank=5 min=4019 min_rank=6 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16408 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[524, 496, 542, 498, 495, 497, 519, 480, 19, 0, 0, 0, 0, 0, 0, 0, 502, 507, 514, 503, 507, 515, 531, 512, 2, 0, 0, 0, 0, 0, 0, 0, 547, 514, 528, 499, 502, 486, 473, 490, 0, 0, 0, 0, 0, 0, 0, 0, 563, 525, 488, 491, 479, 499, 496, 515, 32, 3, 0, 0, 0, 0, 0, 0, 527, 530, 469, 549, 523, 501, 535, 539, 8, 2, 0, 0, 0, 0, 0, 0, 478, 502, 513, 506, 507, 496, 517, 526, 0, 0, 0, 0, 0, 0, 0, 0, 533, 518, 484, 519, 482, 492, 499, 522, 0, 0, 0, 0, 0, 0, 0, 0, 531, 514, 511, 550, 553, 517, 489, 533, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4070, 4093, 4039, 4091, 4183, 4045, 4049, 4198] avg=4096.0000 max=4198 max_rank=7 min=4039 min_rank=2 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16475 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[543, 477, 552, 490, 470, 515, 466, 517, 18, 0, 0, 0, 0, 0, 0, 0, 499, 527, 537, 501, 505, 521, 495, 522, 2, 0, 0, 0, 0, 0, 0, 0, 490, 535, 495, 528, 503, 525, 493, 471, 0, 0, 0, 0, 0, 0, 0, 0, 506, 540, 507, 568, 502, 489, 499, 519, 33, 3, 0, 0, 0, 0, 0, 0, 507, 524, 500, 487, 498, 510, 486, 531, 9, 2, 0, 0, 0, 0, 0, 0, 514, 479, 507, 468, 550, 516, 556, 528, 0, 0, 0, 0, 0, 0, 0, 0, 509, 502, 532, 516, 508, 519, 491, 551, 0, 0, 0, 0, 0, 0, 0, 0, 508, 522, 478, 526, 514, 483, 549, 525, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4048, 4109, 4040, 4166, 4054, 4118, 4128, 4105] avg=4096.0000 max=4166 max_rank=3 min=4040 min_rank=2 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16405 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[493, 529, 505, 495, 522, 528, 482, 501, 23, 0, 0, 0, 0, 0, 0, 0, 544, 488, 516, 525, 474, 522, 493, 518, 7, 0, 0, 0, 0, 0, 0, 0, 527, 530, 510, 486, 565, 547, 462, 510, 0, 0, 0, 0, 0, 0, 0, 0, 506, 519, 516, 510, 495, 530, 520, 486, 36, 5, 0, 0, 0, 0, 0, 0, 525, 557, 489, 481, 507, 526, 478, 467, 12, 8, 0, 0, 0, 0, 0, 0, 502, 511, 515, 517, 535, 527, 500, 512, 0, 0, 0, 0, 0, 0, 0, 0, 502, 499, 524, 536, 511, 508, 561, 530, 0, 0, 0, 0, 0, 0, 0, 0, 549, 538, 468, 484, 492, 488, 488, 496, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4087, 4137, 4123, 4050, 4119, 4171, 4003] avg=4096.0000 max=4171 max_rank=6 min=4003 min_rank=7 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16343 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[530, 513, 498, 507, 531, 515, 476, 538, 18, 0, 0, 0, 0, 0, 0, 0, 514, 526, 513, 515, 546, 534, 514, 494, 2, 0, 0, 0, 0, 0, 0, 0, 532, 517, 495, 532, 490, 494, 479, 528, 0, 0, 0, 0, 0, 0, 0, 0, 558, 473, 511, 499, 517, 492, 480, 534, 32, 3, 0, 0, 0, 0, 0, 0, 515, 486, 498, 528, 477, 509, 531, 503, 9, 2, 0, 0, 0, 0, 0, 0, 512, 471, 476, 504, 509, 513, 508, 548, 0, 0, 0, 0, 0, 0, 0, 0, 528, 502, 494, 514, 547, 486, 511, 532, 0, 0, 0, 0, 0, 0, 0, 0, 493, 514, 491, 513, 513, 522, 532, 527, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4158, 4067, 4099, 4058, 4041, 4114, 4105] avg=4096.0000 max=4158 max_rank=1 min=4041 min_rank=5 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16318 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[518, 505, 519, 475, 548, 494, 543, 551, 18, 0, 0, 0, 0, 0, 0, 0, 525, 528, 463, 505, 523, 519, 540, 536, 2, 0, 0, 0, 0, 0, 0, 0, 498, 535, 509, 511, 505, 513, 519, 499, 0, 0, 0, 0, 0, 0, 0, 0, 509, 520, 507, 478, 497, 491, 482, 512, 35, 3, 0, 0, 0, 0, 0, 0, 507, 505, 505, 545, 483, 502, 517, 542, 9, 2, 0, 0, 0, 0, 0, 0, 484, 506, 501, 541, 491, 524, 532, 499, 0, 0, 0, 0, 0, 0, 0, 0, 521, 477, 503, 529, 512, 488, 496, 526, 0, 0, 0, 0, 0, 0, 0, 0, 510, 511, 473, 501, 503, 527, 528, 533, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4171, 4141, 4089, 4034, 4117, 4078, 4052, 4086] avg=4096.0000 max=4171 max_rank=0 min=4034 min_rank=3 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16333 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 3968, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4096, 4352, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[554, 510, 501, 509, 489, 513, 526, 557, 544, 549, 494, 516, 533, 528, 522, 488, 509, 503, 503, 524, 521, 527, 547, 546, 467, 477, 492, 518, 479, 493, 498, 499, 525, 488, 537, 543, 488, 534, 554, 522, 480, 536, 517, 492, 461, 489, 483, 515, 480, 528, 509, 534, 485, 525, 511, 480, 537, 544, 511, 528, 523, 508, 482, 483] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[525, 514, 524, 549, 483, 509, 503, 466, 505, 493, 522, 513, 505, 513, 497, 480, 485, 518, 511, 511, 508, 500, 529, 547, 501, 520, 528, 508, 526, 492, 488, 522, 515, 530, 531, 480, 489, 523, 535, 535, 502, 517, 510, 517, 514, 501, 526, 494, 489, 522, 487, 513, 509, 547, 526, 481, 544, 546, 546, 520, 500, 522, 505, 497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[519, 490, 497, 509, 531, 538, 533, 512, 497, 546, 519, 527, 498, 521, 497, 520, 535, 492, 491, 501, 501, 513, 498, 507, 504, 508, 480, 518, 503, 480, 499, 511, 509, 492, 517, 507, 530, 489, 522, 520, 531, 518, 486, 528, 489, 490, 543, 528, 506, 516, 531, 496, 533, 541, 542, 516, 480, 540, 497, 505, 534, 499, 483, 555] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[518, 483, 523, 506, 464, 489, 539, 503, 503, 511, 494, 532, 509, 475, 486, 581, 512, 517, 474, 526, 506, 509, 530, 511, 507, 504, 538, 511, 509, 460, 511, 534, 517, 515, 524, 509, 520, 570, 523, 494, 551, 514, 536, 539, 511, 541, 506, 533, 519, 515, 536, 475, 493, 510, 543, 467, 471, 545, 498, 524, 461, 511, 488, 534] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[490, 498, 523, 484, 499, 486, 521, 488, 562, 480, 508, 480, 502, 503, 530, 541, 485, 485, 544, 553, 523, 486, 480, 540, 540, 478, 495, 477, 534, 527, 521, 501, 544, 500, 551, 539, 515, 515, 474, 469, 508, 532, 547, 539, 496, 585, 474, 515, 502, 484, 461, 484, 523, 505, 520, 517, 541, 523, 539, 533, 488, 531, 526, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[490, 481, 544, 519, 514, 518, 508, 505, 516, 506, 527, 496, 512, 487, 499, 495, 516, 520, 531, 542, 533, 513, 517, 484, 550, 512, 491, 515, 545, 519, 546, 522, 554, 504, 525, 488, 507, 508, 490, 514, 518, 488, 522, 479, 522, 511, 508, 502, 492, 488, 484, 514, 547, 491, 534, 479, 518, 513, 522, 509, 490, 535, 514, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[482, 497, 506, 502, 530, 546, 485, 526, 531, 521, 499, 515, 537, 518, 507, 543, 500, 509, 497, 506, 525, 482, 503, 548, 510, 549, 487, 505, 522, 533, 478, 490, 515, 499, 484, 518, 537, 468, 531, 558, 526, 516, 507, 483, 509, 468, 561, 530, 481, 528, 546, 491, 527, 476, 496, 515, 549, 514, 486, 500, 525, 472, 529, 534] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[500, 485, 533, 539, 512, 529, 509, 534, 493, 493, 516, 515, 474, 515, 549, 490, 555, 517, 522, 510, 493, 514, 511, 518, 519, 472, 481, 557, 551, 544, 503, 544, 513, 497, 475, 520, 517, 495, 476, 494, 528, 523, 537, 554, 497, 487, 517, 521, 528, 553, 491, 535, 510, 497, 514, 493, 514, 530, 488, 480, 471, 526, 507, 483] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[554, 510, 501, 509, 489, 513, 526, 557, 11, 1, 0, 0, 0, 0, 0, 0, 543, 549, 494, 516, 533, 528, 522, 488, 0, 0, 0, 0, 0, 0, 0, 0, 509, 503, 503, 524, 521, 527, 547, 533, 0, 0, 0, 0, 0, 0, 0, 0, 467, 477, 492, 518, 479, 493, 498, 499, 13, 7, 0, 0, 0, 0, 0, 0, 511, 488, 537, 543, 488, 534, 554, 522, 0, 0, 0, 0, 0, 0, 0, 0, 480, 536, 500, 492, 461, 489, 483, 515, 0, 0, 0, 0, 0, 0, 0, 0, 480, 528, 509, 534, 485, 525, 511, 480, 17, 7, 0, 0, 0, 0, 0, 0, 537, 533, 511, 528, 523, 508, 482, 483, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4171, 4173, 4167, 3943, 4177, 3956, 4076, 4105] avg=4096.0000 max=4177 max_rank=4 min=3943 min_rank=3 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16314 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[519, 490, 497, 509, 531, 538, 533, 512, 11, 1, 0, 0, 0, 0, 0, 0, 496, 546, 519, 527, 498, 521, 497, 520, 0, 0, 0, 0, 0, 0, 0, 0, 535, 492, 491, 501, 501, 513, 498, 495, 0, 0, 0, 0, 0, 0, 0, 0, 504, 508, 480, 518, 503, 480, 499, 511, 12, 6, 0, 0, 0, 0, 0, 0, 497, 492, 517, 507, 530, 489, 522, 520, 0, 0, 0, 0, 0, 0, 0, 0, 531, 518, 470, 528, 489, 490, 543, 528, 0, 0, 0, 0, 0, 0, 0, 0, 506, 516, 531, 496, 533, 541, 542, 516, 16, 6, 0, 0, 0, 0, 0, 0, 480, 529, 497, 505, 534, 499, 483, 555, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4141, 4124, 4026, 4021, 4074, 4097, 4203, 4082] avg=4096.0000 max=4203 max_rank=6 min=4021 min_rank=3 imbalance_ratio=1.026123 max_load_reduction=28565 max_load_reduction_pct=0.871735 moved_to_echo=16456 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[518, 483, 523, 506, 464, 489, 539, 503, 11, 1, 0, 0, 0, 0, 0, 0, 502, 511, 494, 532, 509, 475, 486, 581, 0, 0, 0, 0, 0, 0, 0, 0, 512, 517, 474, 526, 506, 509, 530, 499, 0, 0, 0, 0, 0, 0, 0, 0, 507, 504, 538, 511, 509, 460, 511, 534, 12, 6, 0, 0, 0, 0, 0, 0, 504, 515, 524, 509, 520, 570, 523, 494, 0, 0, 0, 0, 0, 0, 0, 0, 551, 514, 518, 539, 511, 541, 506, 533, 0, 0, 0, 0, 0, 0, 0, 0, 519, 515, 536, 475, 493, 510, 543, 467, 18, 7, 0, 0, 0, 0, 0, 0, 471, 534, 498, 524, 461, 511, 488, 534, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4037, 4090, 4073, 4092, 4159, 4213, 4083, 4021] avg=4096.0000 max=4213 max_rank=5 min=4021 min_rank=7 imbalance_ratio=1.028564 max_load_reduction=28555 max_load_reduction_pct=0.871429 moved_to_echo=16476 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[525, 514, 524, 549, 483, 509, 503, 466, 11, 1, 0, 0, 0, 0, 0, 0, 504, 493, 522, 513, 505, 513, 497, 480, 0, 0, 0, 0, 0, 0, 0, 0, 485, 518, 511, 511, 508, 500, 529, 534, 0, 0, 0, 0, 0, 0, 0, 0, 501, 520, 528, 508, 526, 492, 488, 522, 13, 6, 0, 0, 0, 0, 0, 0, 502, 530, 531, 480, 489, 523, 535, 535, 0, 0, 0, 0, 0, 0, 0, 0, 502, 517, 493, 517, 514, 501, 526, 494, 0, 0, 0, 0, 0, 0, 0, 0, 489, 522, 487, 513, 509, 547, 526, 481, 17, 7, 0, 0, 0, 0, 0, 0, 544, 535, 546, 520, 500, 522, 505, 497, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4027, 4096, 4104, 4125, 4064, 4098, 4169] avg=4096.0000 max=4169 max_rank=7 min=4027 min_rank=1 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16456 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[490, 498, 523, 484, 499, 486, 521, 488, 10, 1, 0, 0, 0, 0, 0, 0, 561, 480, 508, 480, 502, 503, 530, 541, 0, 0, 0, 0, 0, 0, 0, 0, 485, 485, 544, 553, 523, 486, 480, 527, 0, 0, 0, 0, 0, 0, 0, 0, 540, 478, 495, 477, 534, 527, 521, 501, 13, 7, 0, 0, 0, 0, 0, 0, 530, 500, 551, 539, 515, 515, 474, 469, 0, 0, 0, 0, 0, 0, 0, 0, 508, 532, 529, 539, 496, 585, 474, 515, 0, 0, 0, 0, 0, 0, 0, 0, 502, 484, 461, 484, 523, 505, 520, 517, 18, 7, 0, 0, 0, 0, 0, 0, 541, 513, 539, 533, 488, 531, 526, 524, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4000, 4105, 4083, 4093, 4093, 4178, 4021, 4195] avg=4096.0000 max=4195 max_rank=7 min=4000 min_rank=0 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16487 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[482, 497, 506, 502, 530, 546, 485, 526, 10, 1, 0, 0, 0, 0, 0, 0, 530, 521, 499, 515, 537, 518, 507, 543, 0, 0, 0, 0, 0, 0, 0, 0, 500, 509, 497, 506, 525, 482, 503, 535, 0, 0, 0, 0, 0, 0, 0, 0, 510, 549, 487, 505, 522, 533, 478, 490, 13, 6, 0, 0, 0, 0, 0, 0, 502, 499, 484, 518, 537, 468, 531, 558, 0, 0, 0, 0, 0, 0, 0, 0, 526, 516, 490, 483, 509, 468, 561, 530, 0, 0, 0, 0, 0, 0, 0, 0, 481, 528, 546, 491, 527, 476, 496, 515, 17, 7, 0, 0, 0, 0, 0, 0, 549, 504, 486, 500, 525, 472, 529, 534, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4170, 4057, 4093, 4097, 4083, 4084, 4099] avg=4096.0000 max=4170 max_rank=1 min=4057 min_rank=2 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[490, 481, 544, 519, 514, 518, 508, 505, 14, 3, 0, 0, 0, 0, 0, 0, 513, 506, 527, 496, 512, 487, 499, 495, 0, 0, 0, 0, 0, 0, 0, 0, 516, 520, 531, 542, 533, 513, 517, 467, 0, 0, 0, 0, 0, 0, 0, 0, 550, 512, 491, 515, 545, 519, 546, 522, 17, 12, 3, 0, 0, 0, 0, 0, 532, 504, 525, 488, 507, 508, 490, 514, 0, 0, 0, 0, 0, 0, 0, 0, 518, 488, 503, 479, 522, 511, 508, 502, 0, 0, 0, 0, 0, 0, 0, 0, 492, 488, 484, 514, 547, 491, 534, 479, 19, 10, 0, 0, 0, 0, 0, 0, 518, 496, 522, 509, 490, 535, 514, 525, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4035, 4139, 4232, 4068, 4031, 4058, 4109] avg=4096.0000 max=4232 max_rank=3 min=4031 min_rank=5 imbalance_ratio=1.033203 max_load_reduction=28536 max_load_reduction_pct=0.870850 moved_to_echo=16266 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[500, 485, 533, 539, 512, 529, 509, 534, 11, 1, 0, 0, 0, 0, 0, 0, 492, 493, 516, 515, 474, 515, 549, 490, 0, 0, 0, 0, 0, 0, 0, 0, 555, 517, 522, 510, 493, 514, 511, 505, 0, 0, 0, 0, 0, 0, 0, 0, 519, 472, 481, 557, 551, 544, 503, 544, 13, 6, 0, 0, 0, 0, 0, 0, 501, 497, 475, 520, 517, 495, 476, 494, 0, 0, 0, 0, 0, 0, 0, 0, 528, 523, 519, 554, 497, 487, 517, 521, 0, 0, 0, 0, 0, 0, 0, 0, 528, 553, 491, 535, 510, 497, 514, 493, 18, 6, 0, 0, 0, 0, 0, 0, 514, 519, 488, 480, 471, 526, 507, 483, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4153, 4044, 4127, 4190, 3975, 4146, 4145, 3988] avg=4096.0000 max=4190 max_rank=3 min=3975 min_rank=4 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16254 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 3968, 4224, 4224, 4096, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4224, 3968, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33664 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[510, 501, 508, 572, 511, 492, 481, 500, 492, 487, 510, 502, 503, 535, 549, 499, 515, 508, 500, 492, 494, 546, 517, 491, 499, 516, 497, 518, 535, 498, 494, 537, 512, 490, 542, 499, 513, 494, 483, 477, 499, 541, 524, 508, 544, 469, 524, 515, 529, 515, 503, 539, 540, 516, 506, 485, 526, 517, 516, 524, 545, 527, 510, 527] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[529, 481, 513, 481, 509, 525, 493, 547, 495, 489, 495, 544, 516, 484, 594, 554, 502, 493, 521, 479, 521, 525, 531, 530, 521, 479, 503, 536, 489, 483, 500, 492, 519, 503, 514, 532, 498, 528, 504, 504, 546, 530, 518, 497, 465, 539, 555, 498, 505, 482, 526, 487, 517, 517, 567, 506, 486, 518, 512, 486, 521, 551, 504, 479] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[504, 486, 469, 509, 481, 484, 532, 525, 533, 539, 490, 468, 530, 516, 481, 559, 493, 508, 541, 487, 515, 572, 532, 480, 548, 526, 526, 501, 507, 524, 502, 506, 518, 487, 484, 463, 497, 530, 526, 530, 474, 517, 533, 526, 514, 496, 510, 511, 543, 495, 534, 504, 515, 498, 527, 526, 537, 510, 521, 550, 504, 504, 503, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[542, 487, 512, 504, 500, 501, 515, 535, 490, 474, 507, 560, 545, 511, 517, 490, 533, 479, 508, 492, 548, 534, 545, 522, 536, 510, 538, 525, 519, 505, 524, 527, 498, 471, 509, 506, 555, 498, 493, 554, 500, 516, 510, 518, 516, 511, 506, 479, 503, 496, 487, 487, 552, 520, 507, 515, 487, 497, 512, 501, 501, 509, 511, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[552, 549, 496, 480, 535, 504, 521, 489, 509, 492, 499, 513, 456, 505, 520, 493, 507, 538, 494, 492, 490, 502, 501, 525, 477, 497, 516, 482, 498, 505, 569, 538, 505, 515, 543, 488, 525, 497, 474, 526, 514, 532, 490, 499, 522, 519, 477, 493, 540, 519, 551, 510, 530, 543, 528, 531, 541, 511, 529, 544, 509, 495, 525, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[540, 521, 478, 490, 483, 511, 521, 482, 518, 542, 464, 489, 524, 461, 523, 508, 499, 487, 509, 536, 494, 503, 510, 506, 510, 533, 505, 530, 531, 523, 488, 530, 486, 533, 545, 508, 536, 536, 517, 524, 494, 530, 499, 521, 498, 498, 504, 525, 520, 495, 535, 522, 487, 515, 521, 481, 515, 541, 531, 522, 513, 546, 490, 531] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[537, 534, 514, 523, 533, 505, 494, 534, 480, 457, 535, 527, 529, 499, 531, 514, 505, 494, 528, 484, 517, 508, 518, 488, 513, 506, 519, 509, 534, 521, 541, 485, 538, 507, 521, 525, 497, 531, 499, 541, 508, 501, 526, 521, 489, 509, 512, 506, 495, 507, 479, 524, 492, 530, 505, 473, 513, 549, 513, 499, 535, 483, 504, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[494, 489, 500, 525, 496, 510, 537, 479, 526, 510, 535, 516, 525, 525, 504, 534, 488, 515, 499, 528, 544, 492, 531, 541, 493, 484, 503, 526, 517, 519, 504, 500, 482, 465, 546, 489, 545, 515, 490, 517, 481, 541, 524, 490, 483, 495, 518, 510, 488, 532, 516, 510, 504, 529, 537, 510, 491, 487, 517, 564, 528, 519, 521, 535] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[510, 501, 508, 572, 511, 492, 481, 500, 21, 0, 0, 0, 0, 0, 0, 0, 492, 487, 510, 502, 503, 535, 549, 499, 3, 2, 0, 0, 0, 0, 0, 0, 515, 508, 500, 492, 494, 546, 517, 491, 5, 0, 0, 0, 0, 0, 0, 0, 499, 516, 497, 518, 528, 498, 494, 537, 0, 0, 0, 0, 0, 0, 0, 0, 512, 490, 542, 499, 513, 494, 483, 477, 8, 0, 0, 0, 0, 0, 0, 0, 499, 541, 524, 508, 544, 469, 524, 515, 11, 4, 0, 0, 0, 0, 0, 0, 529, 515, 503, 539, 540, 516, 491, 485, 0, 0, 0, 0, 0, 0, 0, 0, 526, 517, 516, 492, 545, 527, 510, 527, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4082, 4068, 4087, 4018, 4139, 4118, 4160] avg=4096.0000 max=4160 max_rank=7 min=4018 min_rank=4 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16435 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[529, 481, 513, 481, 509, 525, 493, 547, 20, 0, 0, 0, 0, 0, 0, 0, 495, 489, 495, 544, 516, 484, 594, 554, 3, 2, 0, 0, 0, 0, 0, 0, 502, 493, 521, 479, 521, 525, 531, 530, 4, 0, 0, 0, 0, 0, 0, 0, 521, 479, 503, 536, 483, 483, 500, 492, 0, 0, 0, 0, 0, 0, 0, 0, 519, 503, 514, 532, 498, 528, 504, 504, 9, 0, 0, 0, 0, 0, 0, 0, 546, 530, 518, 497, 465, 539, 555, 498, 10, 5, 0, 0, 0, 0, 0, 0, 505, 482, 526, 487, 517, 517, 550, 506, 0, 0, 0, 0, 0, 0, 0, 0, 486, 518, 512, 456, 521, 551, 504, 479, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4176, 4106, 3997, 4111, 4163, 4090, 4027] avg=4096.0000 max=4176 max_rank=1 min=3997 min_rank=3 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[540, 521, 478, 490, 483, 511, 521, 482, 21, 0, 0, 0, 0, 0, 0, 0, 518, 542, 464, 489, 524, 461, 523, 508, 3, 2, 0, 0, 0, 0, 0, 0, 499, 487, 509, 536, 494, 503, 510, 506, 5, 0, 0, 0, 0, 0, 0, 0, 510, 533, 505, 530, 524, 523, 488, 530, 0, 0, 0, 0, 0, 0, 0, 0, 486, 533, 545, 508, 536, 536, 517, 524, 8, 0, 0, 0, 0, 0, 0, 0, 494, 530, 499, 521, 498, 498, 504, 525, 11, 4, 0, 0, 0, 0, 0, 0, 520, 495, 535, 522, 487, 515, 506, 481, 0, 0, 0, 0, 0, 0, 0, 0, 515, 541, 531, 490, 513, 546, 490, 531, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4047, 4034, 4049, 4143, 4193, 4084, 4061, 4157] avg=4096.0000 max=4193 max_rank=4 min=4034 min_rank=1 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16495 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[504, 486, 469, 509, 481, 484, 532, 525, 22, 0, 0, 0, 0, 0, 0, 0, 533, 539, 490, 468, 530, 516, 481, 559, 3, 2, 0, 0, 0, 0, 0, 0, 493, 508, 541, 487, 515, 572, 532, 480, 5, 0, 0, 0, 0, 0, 0, 0, 548, 526, 526, 501, 500, 524, 502, 506, 0, 0, 0, 0, 0, 0, 0, 0, 518, 487, 484, 463, 497, 530, 526, 530, 8, 0, 0, 0, 0, 0, 0, 0, 474, 517, 533, 526, 514, 496, 510, 511, 11, 5, 0, 0, 0, 0, 0, 0, 543, 495, 534, 504, 515, 498, 511, 526, 0, 0, 0, 0, 0, 0, 0, 0, 537, 510, 521, 517, 504, 504, 503, 507, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4012, 4121, 4133, 4133, 4043, 4097, 4126, 4103] avg=4096.0000 max=4133 max_rank=2 min=4012 min_rank=0 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16369 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[552, 549, 496, 480, 535, 504, 521, 489, 22, 0, 0, 0, 0, 0, 0, 0, 509, 492, 499, 513, 456, 505, 520, 493, 3, 2, 0, 0, 0, 0, 0, 0, 507, 538, 494, 492, 490, 502, 501, 525, 4, 0, 0, 0, 0, 0, 0, 0, 477, 497, 516, 482, 492, 505, 569, 538, 0, 0, 0, 0, 0, 0, 0, 0, 505, 515, 543, 488, 525, 497, 474, 526, 8, 0, 0, 0, 0, 0, 0, 0, 514, 532, 490, 499, 522, 519, 477, 493, 11, 5, 0, 0, 0, 0, 0, 0, 540, 519, 551, 510, 530, 543, 512, 531, 0, 0, 0, 0, 0, 0, 0, 0, 541, 511, 529, 511, 509, 495, 525, 499, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4148, 3992, 4053, 4076, 4081, 4062, 4236, 4120] avg=4096.0000 max=4236 max_rank=6 min=3992 min_rank=1 imbalance_ratio=1.034180 max_load_reduction=28532 max_load_reduction_pct=0.870728 moved_to_echo=16499 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[537, 534, 514, 523, 533, 505, 494, 534, 24, 0, 0, 0, 0, 0, 0, 0, 480, 457, 535, 527, 529, 499, 531, 514, 8, 4, 0, 0, 0, 0, 0, 0, 505, 494, 528, 484, 517, 508, 518, 488, 8, 0, 0, 0, 0, 0, 0, 0, 513, 506, 519, 509, 522, 521, 541, 485, 0, 0, 0, 0, 0, 0, 0, 0, 538, 507, 521, 525, 497, 531, 499, 541, 13, 0, 0, 0, 0, 0, 0, 0, 508, 501, 526, 521, 489, 509, 512, 506, 14, 8, 0, 0, 0, 0, 0, 0, 495, 507, 479, 524, 492, 530, 476, 473, 0, 0, 0, 0, 0, 0, 0, 0, 513, 549, 513, 461, 535, 483, 504, 520, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4198, 4084, 4050, 4116, 4172, 4094, 3976, 4078] avg=4096.0000 max=4198 max_rank=0 min=3976 min_rank=6 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16320 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[542, 487, 512, 504, 500, 501, 515, 535, 20, 0, 0, 0, 0, 0, 0, 0, 490, 474, 507, 560, 545, 511, 517, 490, 3, 2, 0, 0, 0, 0, 0, 0, 533, 479, 508, 492, 548, 534, 545, 522, 5, 0, 0, 0, 0, 0, 0, 0, 536, 510, 538, 525, 512, 505, 524, 527, 0, 0, 0, 0, 0, 0, 0, 0, 498, 471, 509, 506, 555, 498, 493, 554, 8, 0, 0, 0, 0, 0, 0, 0, 500, 516, 510, 518, 516, 511, 506, 479, 10, 4, 0, 0, 0, 0, 0, 0, 503, 496, 487, 487, 552, 520, 492, 515, 0, 0, 0, 0, 0, 0, 0, 0, 487, 497, 512, 471, 501, 509, 511, 508, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4099, 4166, 4177, 4092, 4070, 4052, 3996] avg=4096.0000 max=4177 max_rank=3 min=3996 min_rank=7 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16210 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[494, 489, 500, 525, 496, 510, 537, 479, 23, 0, 0, 0, 0, 0, 0, 0, 526, 510, 535, 516, 525, 525, 504, 534, 3, 2, 0, 0, 0, 0, 0, 0, 488, 515, 499, 528, 544, 492, 531, 541, 5, 0, 0, 0, 0, 0, 0, 0, 493, 484, 503, 526, 510, 519, 504, 500, 0, 0, 0, 0, 0, 0, 0, 0, 482, 465, 546, 489, 545, 515, 490, 517, 9, 0, 0, 0, 0, 0, 0, 0, 481, 541, 524, 490, 483, 495, 518, 510, 12, 5, 0, 0, 0, 0, 0, 0, 488, 532, 516, 510, 504, 529, 520, 510, 0, 0, 0, 0, 0, 0, 0, 0, 491, 487, 517, 529, 528, 519, 521, 535, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4053, 4180, 4143, 4039, 4058, 4059, 4109, 4127] avg=4096.0000 max=4180 max_rank=1 min=4039 min_rank=3 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 3968, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33152 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[530, 515, 518, 476, 489, 494, 505, 518, 489, 516, 556, 506, 498, 512, 513, 506, 515, 505, 513, 471, 508, 505, 526, 496, 530, 507, 512, 534, 507, 494, 508, 470, 522, 489, 527, 501, 497, 528, 492, 489, 488, 486, 485, 520, 538, 514, 526, 547, 515, 506, 543, 514, 553, 534, 500, 510, 535, 508, 507, 484, 553, 535, 527, 553] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[487, 524, 525, 523, 472, 516, 533, 501, 495, 492, 520, 518, 474, 524, 499, 544, 529, 523, 474, 555, 493, 533, 517, 514, 519, 547, 502, 501, 539, 558, 512, 506, 535, 511, 496, 537, 502, 533, 525, 526, 491, 504, 511, 493, 506, 476, 511, 498, 499, 524, 522, 522, 483, 509, 515, 506, 495, 498, 496, 529, 502, 540, 512, 492] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[533, 515, 533, 488, 577, 495, 479, 539, 524, 526, 522, 543, 517, 494, 484, 492, 501, 531, 505, 531, 480, 463, 519, 506, 520, 532, 530, 575, 487, 511, 551, 524, 482, 511, 485, 498, 499, 496, 528, 520, 506, 558, 486, 502, 496, 533, 492, 509, 540, 536, 554, 537, 481, 525, 492, 504, 470, 509, 489, 512, 520, 481, 474, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[540, 521, 531, 495, 527, 511, 521, 498, 518, 458, 525, 478, 546, 549, 498, 481, 504, 546, 513, 514, 549, 488, 498, 512, 513, 526, 519, 557, 509, 522, 542, 531, 485, 530, 547, 535, 523, 489, 514, 510, 528, 504, 518, 511, 508, 504, 466, 524, 509, 523, 487, 481, 516, 480, 492, 544, 510, 499, 520, 491, 498, 521, 475, 486] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[515, 551, 482, 510, 517, 515, 512, 513, 503, 484, 553, 522, 491, 492, 510, 500, 530, 486, 494, 511, 505, 517, 500, 442, 514, 504, 530, 519, 499, 501, 503, 485, 504, 514, 530, 514, 566, 495, 528, 521, 568, 501, 524, 497, 473, 526, 510, 498, 467, 518, 513, 499, 486, 510, 571, 502, 529, 558, 547, 529, 533, 495, 523, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[517, 502, 491, 482, 519, 513, 546, 524, 508, 513, 535, 494, 510, 510, 486, 498, 505, 528, 522, 528, 522, 507, 514, 496, 539, 495, 489, 479, 482, 533, 516, 510, 518, 491, 573, 547, 496, 499, 489, 491, 536, 525, 491, 496, 546, 501, 525, 528, 509, 513, 501, 517, 540, 532, 511, 519, 509, 477, 538, 496, 501, 463, 526, 551] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[540, 465, 450, 522, 549, 519, 513, 518, 501, 528, 545, 511, 546, 487, 535, 531, 536, 503, 535, 507, 492, 494, 508, 500, 496, 488, 498, 480, 469, 547, 537, 530, 532, 531, 557, 526, 514, 482, 514, 533, 507, 511, 482, 512, 499, 515, 524, 517, 492, 492, 509, 494, 550, 471, 484, 531, 522, 522, 497, 516, 539, 515, 464, 534] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[526, 474, 529, 495, 494, 520, 498, 481, 529, 512, 532, 508, 483, 551, 494, 511, 516, 548, 501, 514, 521, 491, 525, 515, 515, 516, 543, 481, 500, 526, 526, 488, 494, 490, 497, 497, 479, 529, 522, 548, 519, 549, 505, 491, 493, 493, 498, 542, 488, 509, 519, 471, 546, 496, 552, 481, 509, 547, 517, 502, 527, 559, 485, 551] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[530, 515, 518, 476, 489, 494, 505, 518, 4, 0, 0, 0, 0, 0, 0, 0, 489, 516, 556, 506, 498, 512, 513, 506, 4, 0, 0, 0, 0, 0, 0, 0, 515, 505, 513, 471, 508, 505, 526, 496, 10, 4, 0, 0, 0, 0, 0, 0, 530, 507, 512, 534, 507, 494, 489, 470, 0, 0, 0, 0, 0, 0, 0, 0, 522, 489, 513, 501, 497, 528, 492, 489, 0, 0, 0, 0, 0, 0, 0, 0, 488, 486, 485, 520, 538, 514, 526, 547, 15, 0, 0, 0, 0, 0, 0, 0, 515, 506, 543, 514, 553, 534, 500, 510, 2, 0, 0, 0, 0, 0, 0, 0, 535, 508, 507, 484, 553, 535, 527, 547, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4049, 4100, 4053, 4043, 4031, 4119, 4177, 4196] avg=4096.0000 max=4196 max_rank=7 min=4031 min_rank=4 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16523 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[533, 515, 533, 488, 577, 495, 479, 539, 4, 0, 0, 0, 0, 0, 0, 0, 524, 526, 522, 543, 517, 494, 484, 492, 4, 0, 0, 0, 0, 0, 0, 0, 501, 531, 505, 531, 480, 463, 519, 506, 9, 4, 0, 0, 0, 0, 0, 0, 520, 532, 530, 575, 487, 511, 531, 524, 0, 0, 0, 0, 0, 0, 0, 0, 482, 511, 472, 498, 499, 496, 528, 520, 0, 0, 0, 0, 0, 0, 0, 0, 506, 558, 486, 502, 496, 533, 492, 509, 16, 0, 0, 0, 0, 0, 0, 0, 540, 536, 554, 537, 481, 525, 492, 504, 2, 0, 0, 0, 0, 0, 0, 0, 470, 509, 489, 512, 520, 481, 474, 510, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4163, 4106, 4049, 4210, 4006, 4098, 4171, 3965] avg=4096.0000 max=4210 max_rank=3 min=3965 min_rank=7 imbalance_ratio=1.027832 max_load_reduction=28558 max_load_reduction_pct=0.871521 moved_to_echo=16240 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[515, 551, 482, 510, 517, 515, 512, 513, 4, 0, 0, 0, 0, 0, 0, 0, 503, 484, 553, 522, 491, 492, 510, 500, 4, 0, 0, 0, 0, 0, 0, 0, 530, 486, 494, 511, 505, 517, 500, 442, 10, 4, 0, 0, 0, 0, 0, 0, 514, 504, 530, 519, 499, 501, 484, 485, 0, 0, 0, 0, 0, 0, 0, 0, 504, 514, 516, 514, 566, 495, 528, 521, 0, 0, 0, 0, 0, 0, 0, 0, 568, 501, 524, 497, 473, 526, 510, 498, 15, 0, 0, 0, 0, 0, 0, 0, 467, 518, 513, 499, 486, 510, 571, 502, 2, 0, 0, 0, 0, 0, 0, 0, 529, 558, 547, 529, 533, 495, 523, 503, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4059, 3999, 4036, 4158, 4112, 4068, 4217] avg=4096.0000 max=4217 max_rank=7 min=3999 min_rank=2 imbalance_ratio=1.029541 max_load_reduction=28551 max_load_reduction_pct=0.871307 moved_to_echo=16555 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[517, 502, 491, 482, 519, 513, 546, 524, 9, 0, 0, 0, 0, 0, 0, 0, 508, 513, 535, 494, 510, 510, 486, 498, 7, 3, 0, 0, 0, 0, 0, 0, 505, 528, 522, 528, 522, 507, 514, 496, 13, 10, 0, 0, 0, 0, 0, 0, 539, 495, 489, 479, 482, 533, 487, 510, 0, 0, 0, 0, 0, 0, 0, 0, 518, 491, 553, 547, 496, 499, 489, 491, 0, 0, 0, 0, 0, 0, 0, 0, 536, 525, 491, 496, 546, 501, 525, 528, 19, 0, 0, 0, 0, 0, 0, 0, 509, 513, 501, 517, 540, 532, 511, 519, 5, 0, 0, 0, 0, 0, 0, 0, 509, 477, 538, 496, 501, 463, 526, 534, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4064, 4145, 4014, 4084, 4167, 4147, 4044] avg=4096.0000 max=4167 max_rank=5 min=4014 min_rank=3 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16442 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[487, 524, 525, 523, 472, 516, 533, 501, 4, 0, 0, 0, 0, 0, 0, 0, 495, 492, 520, 518, 474, 524, 499, 544, 4, 0, 0, 0, 0, 0, 0, 0, 529, 523, 474, 555, 493, 533, 517, 514, 9, 4, 0, 0, 0, 0, 0, 0, 519, 547, 502, 501, 539, 558, 493, 506, 0, 0, 0, 0, 0, 0, 0, 0, 535, 511, 483, 537, 502, 533, 525, 526, 0, 0, 0, 0, 0, 0, 0, 0, 491, 504, 511, 493, 506, 476, 511, 498, 15, 0, 0, 0, 0, 0, 0, 0, 499, 524, 522, 522, 483, 509, 515, 506, 2, 0, 0, 0, 0, 0, 0, 0, 495, 498, 496, 529, 502, 540, 512, 486, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4070, 4151, 4165, 4152, 4005, 4082, 4058] avg=4096.0000 max=4165 max_rank=3 min=4005 min_rank=5 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16297 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[540, 521, 531, 495, 527, 511, 521, 498, 4, 0, 0, 0, 0, 0, 0, 0, 518, 458, 525, 478, 546, 549, 498, 481, 4, 0, 0, 0, 0, 0, 0, 0, 504, 546, 513, 514, 549, 488, 498, 512, 10, 4, 0, 0, 0, 0, 0, 0, 513, 526, 519, 557, 509, 522, 522, 531, 0, 0, 0, 0, 0, 0, 0, 0, 485, 530, 533, 535, 523, 489, 514, 510, 0, 0, 0, 0, 0, 0, 0, 0, 528, 504, 518, 511, 508, 504, 466, 524, 16, 0, 0, 0, 0, 0, 0, 0, 509, 523, 487, 481, 516, 480, 492, 544, 2, 0, 0, 0, 0, 0, 0, 0, 510, 499, 520, 491, 498, 521, 475, 480, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4148, 4057, 4138, 4199, 4119, 4079, 4034, 3994] avg=4096.0000 max=4199 max_rank=3 min=3994 min_rank=7 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16226 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[540, 465, 450, 522, 549, 519, 513, 518, 4, 0, 0, 0, 0, 0, 0, 0, 501, 528, 545, 511, 546, 487, 535, 531, 4, 0, 0, 0, 0, 0, 0, 0, 536, 503, 535, 507, 492, 494, 508, 500, 10, 4, 0, 0, 0, 0, 0, 0, 496, 488, 498, 480, 469, 547, 517, 530, 0, 0, 0, 0, 0, 0, 0, 0, 532, 531, 543, 526, 514, 482, 514, 533, 0, 0, 0, 0, 0, 0, 0, 0, 507, 511, 482, 512, 499, 515, 524, 517, 16, 0, 0, 0, 0, 0, 0, 0, 492, 492, 509, 494, 550, 471, 484, 531, 2, 0, 0, 0, 0, 0, 0, 0, 522, 522, 497, 516, 539, 515, 464, 528, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4188, 4089, 4025, 4175, 4083, 4025, 4103] avg=4096.0000 max=4188 max_rank=1 min=4025 min_rank=3 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[526, 474, 529, 495, 494, 520, 498, 481, 4, 0, 0, 0, 0, 0, 0, 0, 529, 512, 532, 508, 483, 551, 494, 511, 4, 0, 0, 0, 0, 0, 0, 0, 516, 548, 501, 514, 521, 491, 525, 515, 9, 4, 0, 0, 0, 0, 0, 0, 515, 516, 543, 481, 500, 526, 507, 488, 0, 0, 0, 0, 0, 0, 0, 0, 494, 490, 484, 497, 479, 529, 522, 548, 0, 0, 0, 0, 0, 0, 0, 0, 519, 549, 505, 491, 493, 493, 498, 542, 15, 0, 0, 0, 0, 0, 0, 0, 488, 509, 519, 471, 546, 496, 552, 481, 2, 0, 0, 0, 0, 0, 0, 0, 509, 547, 517, 502, 527, 559, 485, 545, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4021, 4124, 4144, 4076, 4043, 4105, 4064, 4191] avg=4096.0000 max=4191 max_rank=7 min=4021 min_rank=0 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16403 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35712 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35712 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4352, 4096, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35712 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35712 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35712 valid_num_tokens=33536 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35712 valid_num_tokens=33280 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35712 valid_num_tokens=33408 slack_slots=2304 device_initiated_grouped_gemm=True + [2026-09-21 08:26:50] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 266.2 | throughput per GPU (TFLOP/s/GPU): 121.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.637299E+00 | loss scale: 1.0 | grad norm: 3.517 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-09-21 08:26:50 +[rank0]:[W921 08:26:51.932811688 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0921 08:26:55.583000 650825 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0921 08:26:55.584000 650825 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 650924 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 650825 got signal: 15 diff --git a/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__cap2p00/rep1/mem_samples.csv b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__cap2p00/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..7d2ed2e8aa0b63b01277efe030a45c5fd157c297 --- /dev/null +++ b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__cap2p00/rep1/mem_samples.csv @@ -0,0 +1,240 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 4 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 629 +2, 629 +3, 629 +4, 629 +5, 629 +6, 629 +7, 629 +0, 907 +1, 919 +2, 929 +3, 927 +4, 925 +5, 917 +6, 923 +7, 949 +0, 1803 +1, 1803 +2, 1803 +3, 1803 +4, 1803 +5, 1803 +6, 1803 +7, 1803 +0, 2059 +1, 2059 +2, 2059 +3, 2059 +4, 2059 +5, 2059 +6, 2059 +7, 2059 +0, 13169 +1, 13263 +2, 13229 +3, 13225 +4, 13253 +5, 13285 +6, 13263 +7, 13241 +0, 13837 +1, 13847 +2, 13833 +3, 13855 +4, 13853 +5, 13853 +6, 13867 +7, 13837 +0, 14957 +1, 14959 +2, 14959 +3, 14961 +4, 14955 +5, 14959 +6, 14961 +7, 14955 +0, 15041 +1, 15043 +2, 15043 +3, 15045 +4, 15039 +5, 15043 +6, 15045 +7, 15039 +0, 15041 +1, 15043 +2, 15043 +3, 15045 +4, 15039 +5, 15043 +6, 15045 +7, 15039 +0, 15687 +1, 15689 +2, 15689 +3, 15691 +4, 15685 +5, 15689 +6, 15691 +7, 15685 +0, 15837 +1, 15839 +2, 15839 +3, 15841 +4, 15835 +5, 15839 +6, 15841 +7, 15835 +0, 16093 +1, 16095 +2, 16095 +3, 16097 +4, 16091 +5, 16095 +6, 16097 +7, 16091 +0, 16867 +1, 16869 +2, 16869 +3, 16871 +4, 16865 +5, 16869 +6, 16869 +7, 16865 +0, 24173 +1, 24175 +2, 24195 +3, 24177 +4, 24191 +5, 24195 +6, 24177 +7, 24191 +0, 32565 +1, 32567 +2, 32567 +3, 32569 +4, 32563 +5, 32567 +6, 32569 +7, 32563 +0, 32565 +1, 32567 +2, 32567 +3, 32569 +4, 32563 +5, 32567 +6, 32569 +7, 32563 +0, 32567 +1, 32569 +2, 32569 +3, 32571 +4, 32565 +5, 32569 +6, 32571 +7, 32565 +0, 32567 +1, 32569 +2, 32569 +3, 32571 +4, 32565 +5, 32569 +6, 32571 +7, 32565 +0, 32567 +1, 32569 +2, 32569 +3, 32571 +4, 32565 +5, 32569 +6, 32571 +7, 32565 +0, 33015 +1, 33009 +2, 33005 +3, 33001 +4, 33041 +5, 33041 +6, 33057 +7, 33045 +0, 34015 +1, 33825 +2, 33827 +3, 33955 +4, 34013 +5, 33825 +6, 34019 +7, 33949 +0, 34589 +1, 34409 +2, 34427 +3, 34569 +4, 34617 +5, 34473 +6, 34647 +7, 34619 +0, 41639 +1, 41463 +2, 41485 +3, 41597 +4, 41651 +5, 41465 +6, 41641 +7, 41587 +0, 41639 +1, 41463 +2, 41485 +3, 41597 +4, 41651 +5, 41465 +6, 41641 +7, 41587 +0, 563 +1, 566 +2, 566 +3, 566 +4, 566 +5, 884 +6, 41107 +7, 2166 diff --git a/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__cap2p00/rep1/run.env b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__cap2p00/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..eb861a41a26f482d8d422d3f2f50ca1744e324fc --- /dev/null +++ b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__cap2p00/rep1/run.env @@ -0,0 +1,102 @@ +backend=echo +config=OLMOE_1B_7B_L16 +family=tierB +source_model=OLMoE-1B-7B-0924 +standard_num_layers=16 +base_rung= +repo=/workspace/Megatron-MoE-Echo +branch=token-drop-analysis +commit=b099fd84 +num_layers=16 +hidden=2048 +ffn_hidden=1024 +moe_ffn_hidden=1024 +num_experts=64 +topk=8 +experts_per_rank=8 +expert_params=6291456 +heads=16 +kv_groups=16 +kv_channels=128 +qk_layernorm=0 +shared_ffn=0 +shared_expert_overlap=0 +moe_token_dispatcher=flex +mla=0 +q_lora_rank= +kv_lora_rank=512 +qk_head_dim=128 +qk_pos_emb_head_dim=64 +v_head_dim=128 +dense_first_layers=0 +moe_layers=16 +rotary_base=10000 +rotary_percent=1.0 +rotary_scaling= +mscale= +mscale_all_dim= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-2 +router_pre_softmax=1 +router_fusion=0 +router_dtype=fp32 +router_groups= +router_group_topk= +router_score_function=softmax +router_expert_bias=0 +router_bias_update_rate= +router_topk_scaling_factor= +shared_expert_gate=0 +apply_layernorm_1p=0 +attention_output_gate=0 +linear_attention_type= +linear_attention_freq= +linear_conv_kernel_dim= +linear_key_head_dim= +linear_value_head_dim= +linear_num_key_heads= +linear_num_value_heads= +mtp_num_layers=0 +mtp_loss_scaling_factor=0.1 +no_weight_decay_cond_type= +max_position_embeddings=4096 +init_method_std=0.02 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-5 +seq_length=4096 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=32768 +mean_tokens_per_expert=4096 +slot_policy=matched +slots_per_rank=8 +capacity_mode=static +capacity=2.00 +cuda_graph=0 +probe_mode=1 +async_grad_reduce=0 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_mode=mxfp8 +fp8_recipe=mxfp8 +training_seed=1234 +learning_rate=1e-4 +min_learning_rate=1.0e-5 +lr_warmup_iters=5 +state_dump_enable=0 +state_dump_dir= +moonep_assert_no_pending_reduce=0 diff --git a/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__cap2p00/rep1/train.log b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__cap2p00/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..d42235c8e7223e66bd0f2e8a234cd6218ebfea3d --- /dev/null +++ b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__cap2p00/rep1/train.log @@ -0,0 +1,8907 @@ +[08:23:18] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +============================================================== + OLMOE_1B_7B_L16 / echo (slots/rank=8 policy=matched) + E=64 K=8 H'=1024 m=8 + L=16 seq=4096 gbs=8 (1 microbatches) + cap_mode=static cap=2.00 cg=0 recompute=none + memory_dump=none snapshots=/workspace/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__cap2p00/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ b099fd84 +============================================================== +W0921 08:23:19.760000 632486 torch/distributed/run.py:766] +W0921 08:23:19.760000 632486 torch/distributed/run.py:766] ***************************************** +W0921 08:23:19.760000 632486 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0921 08:23:19.760000 632486 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 16 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 1024 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 2048 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.02 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 4096 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.01 + moe_debug_drop_logging .......................... True + moe_debug_drop_logging_interval ................. 1 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ True + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. True + moe_enable_hybridep ............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 1024 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ 64 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 2.0 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... True + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-05 + normalization ................................... RMSNorm + num_attention_heads ............................. 16 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 64 + num_layers ...................................... 16 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 16 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 4[Gloo] Rank 6 is connected to 7 is connected to 7 is connected to 7 peer ranks. 7 peer ranks. Expected number of connected peer ranks is : peer ranks. Expected number of connected peer ranks is : 7Expected number of connected peer ranks is : 7 +7 + +[Gloo] Rank [Gloo] Rank 0 is connected to 17 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7[Gloo] Rank +4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 6 is connected to 77 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank3]:[W921 08:23:36.671242972 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W921 08:23:36.671291645 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W921 08:23:36.679593311 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W921 08:23:36.679636074 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W921 08:23:36.679713075 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W921 08:23:36.679736553 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W921 08:23:36.682635894 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.035 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W921 08:23:36.836370227 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 4.006 seconds +time to initialize megatron (seconds): 8.227 +[after megatron is initialized] datetime: 2026-09-21 08:23:43 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 1281951744 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 7 +Params for bucket 1 (103022592 elements, 103022592 padded size): + module.output_layer.weight +Params for bucket 2 (46542848 elements, 46542848 padded size): + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.self_attention.linear_qkv.weight + module.decoder.final_layernorm.weight + module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.self_attention.linear_qkv.weight + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.linear_qkv.weight +Params for bucket 3 (50737152 elements, 50737152 padded size): + module.decoder.layers.12.self_attention.linear_qkv.weight + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.10.self_attention.linear_qkv.weight + module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.pre_mlp_layernorm.weight +Params for bucket 4 (50737152 elements, 50737152 padded size): + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.self_attention.linear_proj.weight +Params for bucket 5 (50737152 elements, 50737152 padded size): + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.weight +Params for bucket 6 (50737152 elements, 50737152 padded size): + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.1.self_attention.linear_qkv.weight +Params for bucket 7 (124131328 elements, 124131328 padded size): + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 20 +Params for bucket 1 (41943040 elements, 41943040 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 +Params for bucket 2 (41943040 elements, 41943040 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 +Params for bucket 3 (41943040 elements, 41943040 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 +Params for bucket 4 (41943040 elements, 41943040 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 +Params for bucket 5 (41943040 elements, 41943040 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 +Params for bucket 6 (41943040 elements, 41943040 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 +Params for bucket 7 (41943040 elements, 41943040 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 +Params for bucket 8 (41943040 elements, 41943040 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 +Params for bucket 9 (41943040 elements, 41943040 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 +Params for bucket 10 (41943040 elements, 41943040 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 +Params for bucket 11 (41943040 elements, 41943040 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 +Params for bucket 12 (41943040 elements, 41943040 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 +Params for bucket 13 (41943040 elements, 41943040 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 +Params for bucket 14 (41943040 elements, 41943040 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 +Params for bucket 15 (41943040 elements, 41943040 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 +Params for bucket 16 (41943040 elements, 41943040 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 +Params for bucket 17 (41943040 elements, 41943040 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 +Params for bucket 18 (41943040 elements, 41943040 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 +Params for bucket 19 (41943040 elements, 41943040 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 +Params for bucket 20 (8388608 elements, 8388608 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-09-21 08:23:44 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 96 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-09-21 08:23:44 +done with setup ... +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (306.21, 315.25) + train/valid/test-data-iterators-setup ..........: (63.96, 77.97) +[before the start of training step] datetime: 2026-09-21 08:23:44 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[2126, 707, 143, 317, 315, 311, 165, 115, 498, 255, 141, 10, 45, 381, 187, 1690, 278, 332, 1325, 1410, 330, 791, 1422, 177, 911, 249, 249, 1136, 581, 763, 127, 277, 214, 628, 497, 61, 880, 226, 601, 546, 399, 562, 539, 70, 1536, 606, 71, 577, 663, 705, 582, 92, 1793, 332, 41, 250, 554, 22, 867, 327, 41, 296, 321, 105] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1914, 572, 134, 548, 126, 525, 145, 17, 423, 333, 82, 9, 53, 687, 320, 1764, 172, 446, 1405, 1359, 85, 839, 913, 475, 913, 159, 70, 1429, 459, 342, 117, 219, 233, 965, 831, 172, 1028, 293, 334, 450, 278, 330, 271, 11, 1667, 817, 94, 460, 690, 526, 329, 171, 1878, 103, 227, 708, 1076, 83, 625, 236, 47, 305, 379, 97] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[2244, 680, 90, 345, 167, 407, 65, 25, 481, 219, 168, 3, 86, 415, 155, 1009, 68, 624, 1413, 1291, 117, 740, 1270, 406, 1268, 169, 54, 1331, 1019, 299, 25, 249, 482, 446, 635, 68, 723, 142, 729, 531, 261, 300, 318, 29, 1541, 652, 91, 499, 715, 1000, 788, 173, 1836, 308, 104, 427, 569, 98, 848, 210, 69, 667, 506, 101] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[2135, 755, 166, 224, 233, 591, 194, 25, 385, 280, 83, 2, 91, 395, 112, 1369, 137, 482, 914, 1662, 121, 579, 1120, 515, 1252, 338, 108, 1786, 577, 515, 42, 218, 145, 770, 513, 178, 1151, 218, 491, 428, 471, 206, 262, 10, 2142, 559, 104, 424, 331, 506, 709, 109, 2068, 172, 217, 447, 997, 32, 738, 299, 46, 222, 334, 63] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1798, 806, 133, 502, 195, 538, 55, 48, 165, 170, 110, 3, 111, 318, 244, 1391, 129, 389, 1577, 1960, 128, 567, 877, 113, 768, 257, 228, 1691, 663, 486, 143, 331, 242, 755, 766, 61, 1153, 258, 632, 446, 643, 533, 427, 20, 1744, 752, 93, 621, 279, 465, 723, 82, 1779, 299, 33, 388, 806, 13, 651, 430, 80, 401, 156, 143] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[2216, 727, 142, 227, 419, 315, 71, 29, 512, 305, 70, 1, 47, 462, 280, 1302, 51, 424, 1622, 1990, 148, 857, 1062, 573, 1060, 227, 154, 1361, 504, 229, 130, 225, 122, 957, 673, 159, 730, 170, 595, 522, 240, 172, 357, 8, 1668, 759, 69, 585, 208, 1023, 633, 59, 1983, 357, 143, 385, 492, 33, 748, 281, 60, 267, 370, 198] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[2142, 651, 177, 407, 263, 614, 99, 21, 513, 254, 40, 0, 75, 636, 331, 1414, 99, 737, 1615, 1624, 131, 632, 1102, 424, 995, 240, 147, 1083, 987, 319, 149, 314, 169, 652, 811, 72, 1012, 127, 530, 444, 313, 135, 588, 11, 1301, 529, 34, 786, 458, 756, 791, 106, 1523, 173, 250, 575, 429, 19, 630, 228, 43, 600, 351, 87] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1849, 669, 102, 339, 84, 432, 107, 50, 539, 163, 86, 4, 97, 413, 271, 1902, 340, 537, 855, 1781, 59, 618, 1510, 195, 1008, 186, 143, 1369, 722, 417, 121, 242, 212, 730, 594, 41, 1268, 373, 405, 553, 348, 559, 431, 18, 1801, 445, 108, 629, 457, 362, 337, 195, 1773, 204, 117, 625, 1044, 65, 490, 238, 130, 523, 325, 158] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2135, 755, 166, 224, 233, 591, 194, 25, 2, 0, 0, 0, 0, 0, 0, 0, 385, 280, 83, 2, 91, 395, 112, 1369, 533, 178, 251, 89, 0, 0, 0, 0, 137, 482, 736, 0, 121, 579, 1120, 515, 0, 0, 0, 0, 0, 0, 0, 0, 1252, 338, 108, 1566, 577, 515, 42, 218, 0, 0, 0, 0, 0, 0, 0, 0, 145, 770, 513, 178, 1151, 218, 491, 428, 131, 16, 0, 0, 0, 0, 0, 0, 471, 206, 262, 10, 2124, 559, 104, 424, 0, 0, 0, 0, 0, 0, 0, 0, 331, 506, 709, 109, 1535, 172, 217, 447, 0, 0, 0, 0, 0, 0, 0, 0, 997, 32, 738, 299, 46, 222, 334, 63, 1411, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4325, 3768, 3690, 4616, 4041, 4160, 4026, 4142] avg=4096.0000 max=4616 max_rank=3 min=3690 min_rank=2 imbalance_ratio=1.126953 max_load_reduction=28152 max_load_reduction_pct=0.859131 moved_to_echo=16369 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1914, 572, 134, 548, 126, 525, 145, 17, 1, 0, 0, 0, 0, 0, 0, 0, 423, 333, 82, 9, 53, 687, 320, 1764, 484, 274, 205, 71, 0, 0, 0, 0, 172, 446, 1131, 0, 85, 839, 913, 475, 0, 0, 0, 0, 0, 0, 0, 0, 913, 159, 70, 1253, 459, 342, 117, 219, 0, 0, 0, 0, 0, 0, 0, 0, 233, 965, 831, 172, 1028, 293, 334, 450, 105, 12, 0, 0, 0, 0, 0, 0, 278, 330, 271, 11, 1654, 817, 94, 460, 0, 0, 0, 0, 0, 0, 0, 0, 690, 526, 329, 171, 1394, 103, 227, 708, 0, 0, 0, 0, 0, 0, 0, 0, 1076, 83, 625, 236, 47, 305, 379, 97, 1154, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3982, 4705, 4061, 3532, 4423, 3915, 4148, 4002] avg=4096.0000 max=4705 max_rank=1 min=3532 min_rank=3 imbalance_ratio=1.148682 max_load_reduction=28063 max_load_reduction_pct=0.856415 moved_to_echo=16488 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2216, 727, 142, 227, 419, 315, 71, 29, 1, 0, 0, 0, 0, 0, 0, 0, 512, 305, 70, 1, 47, 462, 280, 1302, 511, 317, 302, 67, 0, 0, 0, 0, 51, 424, 1305, 0, 148, 857, 1062, 573, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 227, 154, 1194, 504, 229, 130, 225, 0, 0, 0, 0, 0, 0, 0, 0, 122, 957, 673, 159, 730, 170, 595, 522, 100, 12, 0, 0, 0, 0, 0, 0, 240, 172, 357, 8, 1655, 759, 69, 585, 0, 0, 0, 0, 0, 0, 0, 0, 208, 1023, 633, 59, 1472, 357, 143, 385, 0, 0, 0, 0, 0, 0, 0, 0, 492, 33, 748, 281, 60, 267, 370, 198, 1688, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4147, 4176, 4420, 3723, 4040, 3845, 4280, 4137] avg=4096.0000 max=4420 max_rank=2 min=3723 min_rank=3 imbalance_ratio=1.079102 max_load_reduction=28348 max_load_reduction_pct=0.865112 moved_to_echo=16302 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1798, 806, 133, 502, 195, 538, 55, 48, 1, 0, 0, 0, 0, 0, 0, 0, 165, 170, 110, 3, 111, 318, 244, 1391, 458, 308, 296, 84, 0, 0, 0, 0, 129, 389, 1269, 0, 128, 567, 877, 113, 0, 0, 0, 0, 0, 0, 0, 0, 768, 257, 228, 1483, 663, 486, 143, 331, 0, 0, 0, 0, 0, 0, 0, 0, 242, 755, 766, 61, 1153, 258, 632, 446, 124, 13, 0, 0, 0, 0, 0, 0, 643, 533, 427, 20, 1730, 752, 93, 621, 0, 0, 0, 0, 0, 0, 0, 0, 279, 465, 723, 82, 1321, 299, 33, 388, 0, 0, 0, 0, 0, 0, 0, 0, 806, 13, 651, 430, 80, 401, 156, 143, 1664, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4076, 3658, 3472, 4359, 4450, 4819, 3590, 4344] avg=4096.0000 max=4819 max_rank=5 min=3472 min_rank=2 imbalance_ratio=1.176514 max_load_reduction=27949 max_load_reduction_pct=0.852936 moved_to_echo=17203 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2126, 707, 143, 317, 315, 311, 165, 115, 1, 0, 0, 0, 0, 0, 0, 0, 498, 255, 141, 10, 45, 381, 187, 1690, 462, 259, 213, 56, 0, 0, 0, 0, 278, 332, 1066, 0, 330, 791, 1422, 177, 0, 0, 0, 0, 0, 0, 0, 0, 911, 249, 249, 997, 581, 763, 127, 277, 0, 0, 0, 0, 0, 0, 0, 0, 214, 628, 497, 61, 880, 226, 601, 546, 83, 11, 0, 0, 0, 0, 0, 0, 399, 562, 539, 70, 1524, 606, 71, 577, 0, 0, 0, 0, 0, 0, 0, 0, 663, 705, 582, 92, 1331, 332, 41, 250, 0, 0, 0, 0, 0, 0, 0, 0, 554, 22, 867, 327, 41, 296, 321, 105, 1197, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4200, 4197, 4396, 4154, 3747, 4348, 3996, 3730] avg=4096.0000 max=4396 max_rank=2 min=3730 min_rank=7 imbalance_ratio=1.073242 max_load_reduction=28372 max_load_reduction_pct=0.865845 moved_to_echo=15821 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2142, 651, 177, 407, 263, 614, 99, 21, 1, 0, 0, 0, 0, 0, 0, 0, 513, 254, 40, 0, 75, 636, 331, 1414, 392, 316, 245, 54, 0, 0, 0, 0, 99, 737, 1299, 0, 131, 632, 1102, 424, 0, 0, 0, 0, 0, 0, 0, 0, 995, 240, 147, 950, 987, 319, 149, 314, 0, 0, 0, 0, 0, 0, 0, 0, 169, 652, 811, 72, 1012, 127, 530, 444, 79, 9, 0, 0, 0, 0, 0, 0, 313, 135, 588, 11, 1291, 529, 34, 786, 0, 0, 0, 0, 0, 0, 0, 0, 458, 756, 791, 106, 1131, 173, 250, 575, 0, 0, 0, 0, 0, 0, 0, 0, 429, 19, 630, 228, 43, 600, 351, 87, 1379, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4375, 4270, 4424, 4101, 3905, 3687, 4240, 3766] avg=4096.0000 max=4424 max_rank=2 min=3687 min_rank=5 imbalance_ratio=1.080078 max_load_reduction=28344 max_load_reduction_pct=0.864990 moved_to_echo=15598 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2244, 680, 90, 345, 167, 407, 65, 25, 6, 0, 0, 0, 0, 0, 0, 0, 481, 219, 168, 3, 86, 415, 155, 1009, 476, 280, 196, 69, 0, 0, 0, 0, 68, 624, 1133, 0, 117, 740, 1270, 406, 0, 0, 0, 0, 0, 0, 0, 0, 1268, 169, 54, 1160, 1019, 299, 25, 249, 0, 0, 0, 0, 0, 0, 0, 0, 482, 446, 635, 68, 723, 142, 729, 531, 102, 15, 0, 0, 0, 0, 0, 0, 261, 300, 318, 29, 1520, 652, 91, 499, 0, 0, 0, 0, 0, 0, 0, 0, 715, 1000, 788, 173, 1360, 308, 104, 427, 0, 0, 0, 0, 0, 0, 0, 0, 569, 98, 848, 210, 69, 667, 506, 101, 1095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4029, 3557, 4358, 4243, 3873, 3670, 4875, 4163] avg=4096.0000 max=4875 max_rank=6 min=3557 min_rank=1 imbalance_ratio=1.190186 max_load_reduction=27893 max_load_reduction_pct=0.851227 moved_to_echo=16581 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1849, 669, 102, 339, 84, 432, 107, 50, 2, 0, 0, 0, 0, 0, 0, 0, 539, 163, 86, 4, 97, 413, 271, 1902, 457, 167, 270, 68, 0, 0, 0, 0, 340, 537, 688, 0, 59, 618, 1510, 195, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 186, 143, 1201, 722, 417, 121, 242, 0, 0, 0, 0, 0, 0, 0, 0, 212, 730, 594, 41, 1268, 373, 405, 553, 100, 13, 0, 0, 0, 0, 0, 0, 348, 559, 431, 18, 1786, 445, 108, 629, 0, 0, 0, 0, 0, 0, 0, 0, 457, 362, 337, 195, 1316, 204, 117, 625, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 65, 490, 238, 130, 523, 325, 158, 1511, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3634, 4437, 3947, 4040, 4289, 4324, 3613, 4484] avg=4096.0000 max=4484 max_rank=7 min=3613 min_rank=6 imbalance_ratio=1.094727 max_load_reduction=28284 max_load_reduction_pct=0.863159 moved_to_echo=16710 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[6016, 384, 5632, 2304, 640, 3328, 2816, 1024, 11136, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3840, 5376, 4992, 1024, 10880, 2048, 1152, 3840, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8192, 1920, 1280, 9856, 5632, 3456, 896, 2176, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1280, 4096, 8704, 0, 1152, 5632, 9344, 2944, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3072, 2816, 3200, 256, 13312, 5120, 768, 4608, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[3584, 2048, 896, 128, 640, 3712, 1920, 11904, 3840, 2176, 2048, 640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[1920, 6016, 5376, 896, 8064, 1920, 4352, 3968, 896, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[16512, 5632, 1152, 2944, 1920, 3840, 1024, 384, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 239, 1861, 3, 49, 0, 1331, 10, 3787, 9, 718, 0, 38, 42, 13, 0, 0, 5, 3, 12, 3, 2542, 0, 107, 81, 48, 0, 2564, 2737, 190, 76, 4072, 0, 48, 67, 7, 7, 0, 16, 3321, 24, 134, 110, 1, 0, 0, 22, 3467, 11, 1, 0, 366, 0, 1, 130, 10, 377, 162, 11, 315, 2462, 1155, 2, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1, 1967, 1868, 85, 0, 0, 0, 0, 2971, 2, 1628, 9, 0, 3, 21, 1, 7, 0, 0, 6, 0, 1344, 0, 590, 53, 0, 0, 3479, 21, 2817, 151, 3279, 6, 47, 22, 320, 0, 0, 0, 3754, 57, 1, 296, 2, 0, 49, 8, 1034, 2, 59, 0, 952, 8, 433, 0, 99, 1046, 5, 15, 250, 1956, 2011, 32, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1, 373, 119, 5, 0, 2, 1087, 28, 3032, 0, 24, 19, 531, 1, 29, 12, 0, 0, 0, 22, 4, 3548, 0, 1836, 160, 0, 0, 2898, 43, 455, 693, 3952, 0, 3, 2, 623, 0, 0, 0, 3923, 33, 717, 9, 0, 0, 0, 19, 3954, 65, 39, 0, 106, 20, 45, 348, 65, 36, 485, 0, 5, 1490, 1907, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[8, 1, 95, 0, 2, 483, 953, 0, 1145, 5, 0, 0, 1608, 0, 334, 1, 7, 1, 0, 732, 17, 3959, 12, 132, 1500, 6, 0, 1702, 1436, 814, 1341, 3929, 73, 0, 17, 51, 23, 0, 1, 3693, 0, 490, 4, 0, 0, 9, 25, 626, 11, 24, 82, 1096, 0, 1523, 273, 0, 608, 20, 1614, 2, 1387, 752, 141, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 1, 1565, 0, 115, 207, 380, 259, 2124, 0, 1986, 12, 134, 0, 3556, 148, 45, 0, 0, 4, 0, 1770, 0, 0, 253, 545, 0, 3663, 2196, 1713, 319, 3256, 0, 2, 33, 765, 44, 1, 8, 2248, 10, 313, 23, 14, 19, 0, 5, 2414, 18, 0, 0, 492, 263, 19, 75, 5, 683, 319, 156, 69, 79, 440, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1, 870, 33, 0, 0, 34, 391, 7, 1864, 3, 55, 0, 29, 162, 2, 0, 0, 5, 20, 23, 203, 3843, 0, 1790, 272, 0, 0, 3332, 1720, 1122, 133, 3301, 29, 0, 49, 532, 0, 0, 1, 2016, 380, 1191, 146, 249, 8, 0, 757, 2900, 26, 1, 98, 511, 0, 1, 3, 2109, 50, 0, 17, 180, 1654, 624, 19, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 1110, 2835, 0, 0, 1, 1124, 0, 1137, 9, 3, 88, 101, 20, 320, 0, 0, 0, 0, 0, 0, 3540, 2, 215, 4, 0, 10, 1280, 2412, 1983, 197, 3965, 0, 0, 0, 1558, 0, 0, 2, 4000, 3, 337, 15, 6, 0, 0, 12, 1362, 2, 18, 0, 33, 167, 1509, 1, 0, 583, 88, 878, 121, 1261, 456, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[161, 102, 525, 120, 5, 0, 3345, 0, 816, 3, 6, 0, 15, 0, 396, 9, 0, 0, 0, 0, 0, 3632, 8, 1259, 507, 0, 0, 1272, 1140, 1998, 766, 4046, 0, 6, 0, 1747, 45, 0, 0, 3735, 854, 7, 86, 107, 1, 1, 145, 3334, 0, 307, 0, 246, 6, 33, 51, 0, 7, 251, 15, 119, 236, 1298, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 373, 119, 5, 0, 2, 1087, 28, 1166, 30, 0, 0, 0, 0, 0, 0, 3032, 0, 24, 19, 531, 1, 29, 12, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4, 3548, 0, 1836, 207, 11, 0, 0, 0, 0, 0, 0, 160, 0, 0, 412, 43, 455, 693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 623, 0, 0, 0, 3912, 0, 0, 0, 0, 0, 0, 0, 0, 33, 717, 9, 0, 0, 0, 19, 3954, 1001, 0, 0, 0, 0, 0, 0, 0, 65, 39, 0, 106, 20, 45, 348, 65, 2786, 0, 0, 0, 0, 0, 0, 0, 36, 485, 0, 5, 1490, 1907, 0, 0, 705, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2811, 4191, 5628, 1763, 4540, 5733, 3474, 4628] avg=4096.0000 max=5733 max_rank=5 min=1763 min_rank=3 imbalance_ratio=1.399658 max_load_reduction=27035 max_load_reduction_pct=0.825043 moved_to_echo=18375 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 870, 33, 0, 0, 34, 391, 7, 974, 34, 0, 0, 0, 0, 0, 0, 1864, 3, 55, 0, 29, 162, 2, 0, 624, 0, 0, 0, 0, 0, 0, 0, 0, 5, 20, 23, 203, 3843, 0, 1790, 238, 5, 0, 0, 0, 0, 0, 0, 272, 0, 0, 474, 1720, 1122, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 49, 532, 0, 0, 1, 2011, 0, 0, 0, 0, 0, 0, 0, 0, 380, 1191, 146, 249, 8, 0, 757, 2900, 1151, 0, 0, 0, 0, 0, 0, 0, 26, 1, 98, 511, 0, 1, 3, 2109, 2327, 0, 0, 0, 0, 0, 0, 0, 50, 0, 17, 180, 1654, 624, 19, 2, 811, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2344, 2739, 6127, 3721, 2622, 6782, 5076, 3357] avg=4096.0000 max=6782 max_rank=5 min=2344 min_rank=0 imbalance_ratio=1.655762 max_load_reduction=25986 max_load_reduction_pct=0.793030 moved_to_echo=17837 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8, 1, 95, 0, 2, 483, 953, 0, 1159, 17, 0, 0, 0, 0, 0, 0, 1145, 5, 0, 0, 1608, 0, 334, 1, 319, 0, 0, 0, 0, 0, 0, 0, 7, 1, 0, 732, 17, 3959, 12, 132, 121, 10, 0, 0, 0, 0, 0, 0, 1500, 6, 0, 243, 1436, 814, 1341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 17, 51, 23, 0, 1, 3683, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 4, 0, 0, 9, 25, 626, 588, 0, 0, 0, 0, 0, 0, 0, 11, 24, 82, 1096, 0, 1523, 273, 0, 2770, 0, 0, 0, 0, 0, 0, 0, 608, 20, 1614, 2, 1387, 752, 141, 0, 414, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2718, 3412, 4991, 5340, 3848, 1742, 5779, 4938] avg=4096.0000 max=5779 max_rank=6 min=1742 min_rank=5 imbalance_ratio=1.410889 max_load_reduction=26989 max_load_reduction_pct=0.823639 moved_to_echo=16307 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 239, 1861, 3, 49, 0, 1331, 10, 1202, 26, 0, 0, 0, 0, 0, 0, 3787, 9, 718, 0, 38, 42, 13, 0, 480, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3, 12, 3, 2542, 0, 107, 183, 9, 0, 0, 0, 0, 0, 0, 81, 48, 0, 365, 2737, 190, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 67, 7, 7, 0, 16, 3312, 0, 0, 0, 0, 0, 0, 0, 0, 24, 134, 110, 1, 0, 0, 22, 3467, 886, 0, 0, 0, 0, 0, 0, 0, 11, 1, 0, 366, 0, 1, 130, 10, 2870, 0, 0, 0, 0, 0, 0, 0, 377, 162, 11, 315, 2462, 1155, 2, 1, 624, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4721, 5087, 2864, 3497, 3457, 4644, 3389, 5109] avg=4096.0000 max=5109 max_rank=7 min=2864 min_rank=2 imbalance_ratio=1.247314 max_load_reduction=27659 max_load_reduction_pct=0.844086 moved_to_echo=16599 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 1967, 1868, 85, 0, 0, 0, 0, 968, 36, 0, 0, 0, 0, 0, 0, 2971, 2, 1628, 9, 0, 3, 21, 1, 652, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 6, 0, 1344, 0, 590, 248, 10, 0, 0, 0, 0, 0, 0, 53, 0, 0, 495, 21, 2817, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 47, 22, 320, 0, 0, 0, 3744, 0, 0, 0, 0, 0, 0, 0, 0, 57, 1, 296, 2, 0, 49, 8, 1034, 1202, 0, 0, 0, 0, 0, 0, 0, 2, 59, 0, 952, 8, 433, 0, 99, 2311, 0, 0, 0, 0, 0, 0, 0, 1046, 5, 15, 250, 1956, 2011, 32, 1, 846, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4925, 5287, 2205, 3537, 4139, 2649, 3864, 6162] avg=4096.0000 max=6162 max_rank=7 min=2205 min_rank=2 imbalance_ratio=1.504395 max_load_reduction=26606 max_load_reduction_pct=0.811951 moved_to_echo=16814 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1565, 0, 115, 207, 380, 259, 960, 38, 0, 0, 0, 0, 0, 0, 2124, 0, 1986, 12, 134, 0, 3556, 148, 687, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 4, 0, 1770, 0, 0, 261, 6, 0, 0, 0, 0, 0, 0, 253, 545, 0, 521, 2196, 1713, 319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 33, 765, 44, 1, 8, 2242, 0, 0, 0, 0, 0, 0, 0, 0, 10, 313, 23, 14, 19, 0, 5, 2414, 1265, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 492, 263, 19, 75, 5, 2296, 0, 0, 0, 0, 0, 0, 0, 683, 319, 156, 69, 79, 440, 0, 0, 891, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3525, 8647, 2086, 5547, 3095, 4063, 3168, 2637] avg=4096.0000 max=8647 max_rank=1 min=2086 min_rank=2 imbalance_ratio=2.111084 max_load_reduction=24121 max_load_reduction_pct=0.736115 moved_to_echo=12963 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1110, 2835, 0, 0, 1, 1124, 0, 1171, 17, 0, 0, 0, 0, 0, 0, 1137, 9, 3, 88, 101, 20, 320, 0, 244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3540, 2, 215, 95, 16, 0, 0, 0, 0, 0, 0, 4, 0, 10, 163, 2412, 1983, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1558, 0, 0, 2, 3984, 0, 0, 0, 0, 0, 0, 0, 0, 3, 337, 15, 6, 0, 0, 12, 1362, 446, 0, 0, 0, 0, 0, 0, 0, 2, 18, 0, 33, 167, 1509, 1, 0, 2794, 0, 0, 0, 0, 0, 0, 0, 583, 88, 878, 121, 1261, 456, 0, 0, 315, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6258, 1922, 3868, 4769, 5544, 2181, 4524, 3702] avg=4096.0000 max=6258 max_rank=0 min=1922 min_rank=1 imbalance_ratio=1.527832 max_load_reduction=26510 max_load_reduction_pct=0.809021 moved_to_echo=15951 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[161, 102, 525, 120, 5, 0, 3345, 0, 1195, 13, 0, 0, 0, 0, 0, 0, 816, 3, 6, 0, 15, 0, 396, 9, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3632, 8, 1259, 90, 10, 0, 0, 0, 0, 0, 0, 507, 0, 0, 183, 1140, 1998, 766, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1747, 45, 0, 0, 3725, 0, 0, 0, 0, 0, 0, 0, 0, 854, 7, 86, 107, 1, 1, 145, 3334, 439, 0, 0, 0, 0, 0, 0, 0, 0, 307, 0, 246, 6, 33, 51, 0, 2851, 0, 0, 0, 0, 0, 0, 0, 7, 251, 15, 119, 236, 1298, 0, 0, 309, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5466, 1483, 4999, 4594, 5523, 4974, 3494, 2235] avg=4096.0000 max=5523 max_rank=4 min=1483 min_rank=1 imbalance_ratio=1.348389 max_load_reduction=27245 max_load_reduction_pct=0.831451 moved_to_echo=16226 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[256, 4736, 8960, 256, 256, 768, 8704, 384, 8832, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[2944, 640, 128, 2944, 11776, 11136, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[3456, 1408, 2816, 1152, 10624, 8704, 256, 128, 4992, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1408, 3200, 768, 384, 128, 128, 1024, 19200, 7040, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 896, 256, 24192, 128, 6016, 1536, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[16896, 128, 4480, 128, 2560, 256, 4736, 256, 3840, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 512, 256, 3840, 512, 3584, 896, 2304, 21120, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 256, 5632, 128, 128, 128, 26624, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[23, 276, 0, 2, 0, 0, 0, 198, 4, 1678, 0, 0, 0, 1, 0, 20, 188, 1, 0, 0, 7, 1, 64, 3051, 0, 39, 0, 3715, 160, 5, 0, 20, 4000, 0, 2, 62, 0, 601, 45, 2, 0, 0, 72, 0, 3471, 20, 2010, 3558, 2918, 25, 0, 0, 1038, 1, 0, 0, 3238, 9, 15, 0, 2173, 0, 1, 54] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[18, 40, 0, 0, 0, 0, 29, 0, 2917, 3386, 0, 141, 0, 0, 0, 0, 0, 2, 1, 3950, 0, 15, 7, 4076, 0, 0, 0, 2185, 2, 6, 4050, 8, 2, 0, 10, 0, 27, 88, 1701, 0, 0, 7, 696, 0, 2312, 0, 0, 45, 618, 0, 1, 0, 1774, 26, 1, 0, 50, 0, 0, 541, 41, 3995, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 3, 21, 0, 0, 7, 75, 0, 0, 4083, 5, 1, 0, 0, 0, 0, 0, 7, 70, 40, 0, 31, 0, 3811, 324, 0, 0, 3867, 0, 43, 5, 4, 4082, 0, 279, 5, 61, 108, 2806, 3, 0, 0, 27, 0, 327, 234, 912, 3, 1525, 113, 1724, 0, 4009, 0, 1, 0, 0, 0, 17, 0, 79, 4056, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[6, 0, 0, 6, 0, 60, 0, 0, 0, 4092, 10, 3274, 28, 12, 8, 0, 13, 1, 0, 0, 0, 0, 4, 2011, 0, 2, 2, 3811, 155, 2, 116, 0, 1, 2845, 0, 0, 0, 0, 113, 0, 0, 37, 187, 0, 1531, 31, 338, 1, 3193, 11, 37, 0, 3121, 5, 0, 0, 0, 0, 0, 18, 4042, 3644, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 5, 0, 20, 3, 2984, 4027, 0, 479, 0, 0, 1, 0, 0, 98, 3, 0, 0, 1, 0, 3933, 0, 23, 0, 3470, 133, 0, 711, 0, 3986, 133, 10, 0, 19, 423, 467, 13, 1, 0, 83, 0, 2761, 785, 1, 13, 2565, 403, 966, 1, 1, 0, 0, 0, 199, 2, 57, 477, 3409, 98, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 91, 0, 2, 1249, 61, 0, 4007, 9, 3975, 2103, 0, 0, 0, 19, 0, 0, 0, 0, 9, 0, 2905, 0, 1277, 4, 3849, 0, 2, 401, 3, 1, 106, 0, 0, 0, 0, 1608, 0, 2, 9, 0, 7, 1642, 0, 109, 447, 799, 15, 2064, 0, 1347, 0, 0, 1, 4009, 0, 0, 0, 80, 553, 0, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 1134, 7, 1, 8, 0, 2233, 4090, 53, 0, 3015, 0, 3, 1, 0, 89, 0, 51, 0, 0, 0, 1683, 0, 9, 0, 3470, 18, 46, 29, 460, 1, 0, 2, 0, 0, 144, 2242, 20, 3, 2, 3340, 1, 358, 288, 1454, 0, 1022, 0, 249, 0, 108, 0, 0, 0, 3312, 0, 2, 0, 82, 3717, 0, 21] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 6, 17, 15, 0, 3762, 2485, 261, 1171, 52, 0, 0, 0, 0, 1084, 0, 9, 0, 0, 0, 4048, 0, 0, 0, 3481, 12, 0, 133, 0, 3997, 2381, 6, 1, 2, 0, 125, 0, 0, 2, 45, 0, 1794, 0, 1043, 0, 1852, 14, 36, 14, 218, 0, 1, 0, 33, 0, 0, 0, 3142, 1388, 0, 136] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 21, 0, 0, 7, 75, 0, 3167, 1807, 0, 0, 0, 0, 0, 0, 0, 916, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 70, 40, 0, 31, 0, 3811, 206, 0, 0, 0, 0, 0, 0, 0, 324, 0, 0, 3412, 0, 43, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 279, 5, 61, 108, 2806, 3, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 327, 234, 912, 3, 499, 84, 0, 0, 0, 0, 0, 0, 1525, 113, 1724, 0, 4009, 0, 1, 0, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 79, 1750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5080, 922, 4165, 3788, 7377, 2086, 7504, 1846] avg=4096.0000 max=7504 max_rank=6 min=922 min_rank=1 imbalance_ratio=1.832031 max_load_reduction=25264 max_load_reduction_pct=0.770996 moved_to_echo=18813 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[18, 40, 0, 0, 0, 0, 29, 0, 2626, 1780, 0, 0, 0, 0, 0, 0, 2917, 760, 0, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 3950, 0, 15, 7, 4076, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1930, 2, 6, 4050, 8, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 10, 0, 27, 88, 1701, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 7, 696, 0, 2312, 0, 0, 45, 491, 47, 0, 0, 0, 0, 0, 0, 618, 0, 1, 0, 1774, 26, 1, 0, 74, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 541, 41, 1724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4493, 3818, 8167, 5996, 1846, 3598, 2494, 2356] avg=4096.0000 max=8167 max_rank=2 min=1846 min_rank=4 imbalance_ratio=1.993896 max_load_reduction=24601 max_load_reduction_pct=0.750763 moved_to_echo=10294 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1134, 7, 1, 8, 0, 3172, 1656, 0, 0, 0, 0, 0, 0, 2233, 918, 53, 0, 3015, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 51, 0, 0, 0, 1683, 185, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 3062, 18, 46, 29, 460, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 144, 2242, 20, 29, 0, 0, 0, 0, 0, 0, 0, 3, 2, 3340, 1, 358, 288, 1454, 0, 457, 76, 0, 0, 0, 0, 0, 0, 1022, 0, 249, 0, 108, 0, 0, 0, 118, 0, 0, 0, 0, 0, 0, 0, 3312, 0, 2, 0, 82, 1604, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5978, 6223, 2008, 3624, 2438, 5979, 1497, 5021] avg=4096.0000 max=6223 max_rank=1 min=1497 min_rank=6 imbalance_ratio=1.519287 max_load_reduction=26545 max_load_reduction_pct=0.810089 moved_to_echo=14935 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[23, 276, 0, 2, 0, 0, 0, 198, 1301, 0, 0, 0, 0, 0, 0, 0, 4, 377, 0, 0, 0, 1, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 188, 1, 0, 0, 7, 1, 64, 3051, 198, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 3279, 160, 5, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 4000, 0, 2, 62, 0, 601, 45, 2, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 3471, 20, 2010, 3558, 0, 81, 0, 0, 0, 0, 0, 0, 2918, 25, 0, 0, 1038, 1, 0, 0, 126, 0, 0, 0, 0, 0, 0, 0, 3238, 9, 15, 0, 2173, 0, 1, 54, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1800, 402, 3510, 3503, 4743, 9212, 4108, 5490] avg=4096.0000 max=9212 max_rank=5 min=402 min_rank=1 imbalance_ratio=2.249023 max_load_reduction=23556 max_load_reduction_pct=0.718872 moved_to_echo=23553 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 0, 6, 0, 60, 0, 0, 3174, 1623, 0, 0, 0, 0, 0, 0, 0, 918, 10, 3274, 28, 12, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1, 0, 0, 0, 0, 4, 2011, 203, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 3363, 155, 2, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2845, 0, 0, 0, 0, 113, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 37, 187, 0, 1531, 31, 338, 1, 448, 83, 0, 0, 0, 0, 0, 0, 3193, 11, 37, 0, 3121, 5, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 4042, 1573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4869, 4250, 2232, 3640, 2991, 2656, 6497, 5633] avg=4096.0000 max=6497 max_rank=6 min=2232 min_rank=2 imbalance_ratio=1.586182 max_load_reduction=26271 max_load_reduction_pct=0.801727 moved_to_echo=17777 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 5, 0, 20, 3, 3123, 43, 0, 0, 0, 0, 0, 0, 2984, 904, 0, 479, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 3, 0, 0, 1, 0, 3933, 185, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 3062, 133, 0, 711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3986, 133, 10, 0, 19, 423, 467, 13, 29, 0, 0, 0, 0, 0, 0, 0, 1, 0, 83, 0, 2761, 785, 1, 13, 12, 76, 0, 0, 0, 0, 0, 0, 2565, 403, 966, 1, 1, 0, 0, 0, 118, 0, 0, 0, 0, 0, 0, 0, 199, 2, 57, 477, 3409, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3198, 4368, 4220, 3929, 5080, 3732, 4054, 4187] avg=4096.0000 max=5080 max_rank=4 min=3198 min_rank=0 imbalance_ratio=1.240234 max_load_reduction=27688 max_load_reduction_pct=0.844971 moved_to_echo=17053 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 6, 17, 15, 0, 1927, 618, 0, 0, 0, 0, 0, 0, 3762, 558, 261, 1171, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1084, 0, 9, 0, 0, 0, 4048, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3073, 12, 0, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3997, 2381, 6, 1, 2, 0, 125, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 2, 45, 0, 1794, 0, 1043, 0, 170, 76, 0, 0, 0, 0, 0, 0, 1852, 14, 36, 14, 218, 0, 1, 0, 118, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 3142, 600, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2585, 5804, 5326, 3218, 6541, 3130, 2253, 3911] avg=4096.0000 max=6541 max_rank=4 min=2253 min_rank=6 imbalance_ratio=1.596924 max_load_reduction=26227 max_load_reduction_pct=0.800385 moved_to_echo=15835 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 91, 0, 2, 1249, 61, 3113, 249, 0, 0, 0, 0, 0, 0, 0, 894, 9, 3975, 2103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 9, 0, 2905, 207, 0, 0, 0, 0, 0, 0, 0, 0, 1277, 4, 3383, 0, 2, 401, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 106, 0, 0, 0, 0, 1608, 0, 37, 0, 0, 0, 0, 0, 0, 0, 2, 9, 0, 7, 1642, 0, 109, 447, 72, 87, 0, 0, 0, 0, 0, 0, 799, 15, 2064, 0, 1347, 0, 0, 1, 135, 0, 0, 0, 0, 0, 0, 0, 4009, 0, 0, 0, 80, 232, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4765, 6981, 3140, 5070, 1752, 2375, 4361, 4324] avg=4096.0000 max=6981 max_rank=1 min=1752 min_rank=4 imbalance_ratio=1.704346 max_load_reduction=25787 max_load_reduction_pct=0.786957 moved_to_echo=12812 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[14592, 640, 5120, 128, 11648, 128, 128, 128, 1024, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 4480, 128, 14208, 1408, 5888, 4096, 2176, 640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[11904, 6272, 384, 9088, 5248, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 384, 128, 1280, 128, 128, 1408, 384, 21632, 7808, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[16128, 5504, 384, 128, 128, 1408, 9216, 128, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[10880, 128, 128, 1152, 13056, 7552, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[256, 1408, 128, 4096, 128, 128, 128, 25600, 1536, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[384, 1408, 128, 24576, 512, 128, 5504, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 1367, 0, 0, 3996, 0, 21, 1701, 29, 4074, 145, 0, 1130, 1557, 0, 0, 0, 4077, 91, 0, 0, 0, 0, 0, 0, 0, 249, 371, 231, 0, 1, 0, 0, 0, 1034, 3, 1, 2, 0, 0, 12, 0, 1836, 2561, 7, 26, 0, 0, 0, 3658, 0, 360, 3975, 6, 0, 0, 189, 0, 0, 10, 48, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 3, 4083, 55, 3084, 0, 1, 1578, 0, 2883, 0, 2, 0, 1034, 0, 0, 0, 2009, 290, 37, 0, 2357, 0, 0, 0, 0, 963, 0, 0, 7, 0, 0, 0, 1257, 3725, 265, 0, 6, 0, 0, 10, 22, 0, 0, 5, 0, 0, 0, 1, 1288, 62, 4086, 2606, 28, 228, 0, 781, 6, 0, 3, 3, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 1217, 155, 195, 0, 62, 1, 0, 0, 3488, 49, 0, 2560, 8, 0, 809, 0, 607, 145, 2, 0, 384, 0, 0, 7, 0, 0, 159, 0, 0, 22, 0, 0, 990, 1578, 0, 0, 0, 3, 1320, 1601, 0, 59, 0, 3783, 0, 2911, 16, 3700, 0, 0, 174, 3691, 0, 0, 0, 871, 0, 0, 2007, 69, 1, 1, 123] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[6, 1359, 2, 54, 0, 0, 1, 167, 0, 0, 0, 0, 0, 571, 55, 0, 0, 3203, 0, 0, 11, 8, 0, 0, 1870, 22, 125, 0, 0, 0, 0, 0, 279, 175, 1941, 4, 7, 0, 0, 1, 2701, 2, 3991, 3, 2137, 0, 3828, 0, 4093, 245, 0, 132, 4077, 4, 1, 0, 23, 0, 0, 8, 1657, 0, 0, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 95, 0, 0, 1220, 1475, 0, 1658, 595, 2073, 0, 6, 173, 6, 0, 2, 0, 3562, 618, 0, 0, 504, 0, 0, 0, 0, 558, 12, 0, 0, 0, 3, 928, 1445, 4086, 0, 0, 16, 0, 0, 45, 0, 4076, 0, 0, 0, 1053, 153, 21, 72, 0, 6, 3571, 0, 1, 115, 1812, 0, 2, 1954, 848, 3, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 208, 2, 0, 9, 3233, 0, 0, 0, 4078, 0, 0, 8, 0, 8, 0, 8, 3372, 0, 0, 0, 154, 0, 0, 5, 0, 0, 639, 0, 0, 0, 0, 151, 446, 4084, 0, 0, 7, 0, 34, 0, 0, 628, 0, 3420, 0, 30, 0, 0, 0, 0, 0, 4028, 0, 0, 1171, 3840, 0, 0, 3204, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1, 12, 3, 1069, 7, 2, 84, 1397, 1, 60, 0, 0, 0, 1125, 0, 973, 0, 1988, 7, 0, 180, 1984, 0, 0, 0, 0, 2594, 34, 3, 12, 74, 14, 823, 259, 3691, 0, 0, 0, 0, 0, 0, 0, 251, 4, 2591, 0, 3879, 0, 0, 3778, 87, 0, 3948, 0, 0, 0, 1669, 0, 4, 146, 7, 0, 0, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[2, 4071, 0, 12, 0, 1, 0, 17, 246, 3524, 0, 0, 603, 0, 1106, 0, 0, 3526, 0, 0, 0, 0, 0, 0, 3, 17, 2, 0, 0, 13, 0, 480, 1286, 16, 3916, 123, 0, 0, 0, 0, 34, 0, 2779, 0, 2958, 0, 3883, 1, 0, 118, 0, 0, 3953, 0, 0, 0, 0, 0, 0, 0, 13, 0, 65, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 4083, 55, 3084, 0, 1, 1578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2853, 0, 2, 0, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2009, 290, 37, 0, 2357, 0, 0, 0, 175, 0, 30, 0, 0, 0, 0, 0, 0, 963, 0, 0, 7, 0, 0, 1791, 0, 0, 0, 0, 0, 0, 0, 0, 1257, 3550, 265, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 22, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1288, 62, 4086, 815, 28, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 781, 6, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8804, 3889, 4898, 2761, 5078, 37, 6508, 793] avg=4096.0000 max=8804 max_rank=0 min=37 min_rank=5 imbalance_ratio=2.149414 max_load_reduction=23964 max_load_reduction_pct=0.731323 moved_to_echo=12416 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[6, 1210, 2, 54, 0, 0, 1, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 571, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3203, 0, 0, 11, 8, 0, 0, 197, 91, 149, 0, 0, 0, 0, 0, 1870, 22, 125, 0, 0, 0, 0, 0, 2802, 924, 0, 0, 0, 0, 0, 0, 279, 175, 1850, 4, 7, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2701, 2, 3991, 3, 1716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 245, 0, 132, 1275, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 8, 1657, 0, 0, 5, 2904, 224, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1440, 626, 3659, 5743, 2316, 8413, 5750, 4821] avg=4096.0000 max=8413 max_rank=5 min=626 min_rank=1 imbalance_ratio=2.053955 max_load_reduction=24355 max_load_reduction_pct=0.743256 moved_to_echo=21300 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1217, 0, 0, 3996, 0, 21, 1701, 0, 0, 0, 0, 0, 0, 0, 0, 29, 4031, 145, 0, 1130, 1557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 91, 0, 0, 0, 0, 0, 0, 48, 150, 43, 0, 0, 0, 0, 0, 0, 249, 371, 231, 0, 1, 0, 2732, 0, 0, 0, 0, 0, 0, 0, 0, 0, 986, 3, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 1836, 2561, 7, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3658, 0, 360, 1243, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 0, 0, 10, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6935, 6892, 4409, 3584, 992, 4442, 5267, 247] avg=4096.0000 max=6935 max_rank=0 min=247 min_rank=7 imbalance_ratio=1.693115 max_load_reduction=25833 max_load_reduction_pct=0.788361 moved_to_echo=10948 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 85, 0, 0, 1220, 1475, 0, 1658, 0, 0, 0, 0, 0, 0, 0, 0, 595, 2052, 0, 6, 173, 6, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3562, 618, 0, 0, 504, 0, 0, 0, 192, 10, 21, 0, 0, 0, 0, 0, 0, 558, 12, 0, 0, 0, 3, 2454, 254, 0, 0, 0, 0, 0, 0, 928, 1445, 3894, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 4076, 0, 0, 0, 0, 153, 0, 0, 0, 0, 0, 0, 0, 0, 21, 72, 0, 6, 1117, 0, 1, 115, 0, 0, 0, 0, 0, 0, 0, 0, 1812, 0, 2, 1954, 848, 3, 1, 0, 799, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4438, 2834, 4907, 3281, 6283, 4274, 1332, 5419] avg=4096.0000 max=6283 max_rank=4 min=1332 min_rank=6 imbalance_ratio=1.533936 max_load_reduction=26485 max_load_reduction_pct=0.808258 moved_to_echo=17308 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1083, 155, 195, 0, 62, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3452, 49, 0, 2560, 8, 0, 809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 607, 145, 2, 0, 384, 0, 0, 350, 74, 134, 36, 0, 0, 0, 0, 7, 0, 0, 159, 0, 0, 22, 0, 2536, 702, 0, 0, 0, 0, 0, 0, 0, 990, 1504, 0, 0, 0, 3, 1320, 0, 0, 0, 0, 0, 0, 0, 0, 1601, 0, 59, 0, 3036, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 3700, 0, 0, 174, 1155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 871, 0, 0, 2007, 69, 1, 1, 123, 2209, 397, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1496, 6878, 1732, 3426, 3817, 4712, 5029, 5678] avg=4096.0000 max=6878 max_rank=1 min=1496 min_rank=0 imbalance_ratio=1.679199 max_load_reduction=25890 max_load_reduction_pct=0.790100 moved_to_echo=19236 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 11, 3, 1069, 7, 2, 84, 1397, 0, 0, 0, 0, 0, 0, 0, 0, 1, 60, 0, 0, 0, 1125, 0, 973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1988, 7, 0, 180, 1984, 0, 0, 239, 174, 1, 0, 0, 0, 0, 0, 0, 0, 2594, 34, 3, 12, 74, 14, 2713, 936, 0, 0, 0, 0, 0, 0, 823, 259, 3517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 4, 2080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3778, 87, 0, 1235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1669, 0, 4, 146, 7, 0, 0, 7, 2943, 272, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2574, 2159, 4573, 6380, 4599, 2335, 5100, 5048] avg=4096.0000 max=6380 max_rank=3 min=2159 min_rank=1 imbalance_ratio=1.557617 max_load_reduction=26388 max_load_reduction_pct=0.805298 moved_to_echo=17082 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 182, 2, 0, 9, 3233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 0, 0, 8, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3372, 0, 0, 0, 154, 0, 0, 320, 196, 26, 46, 0, 0, 0, 0, 5, 0, 0, 639, 0, 0, 0, 0, 2771, 8, 0, 0, 0, 0, 0, 0, 151, 446, 3888, 0, 0, 7, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 628, 0, 2738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1257, 0, 0, 1171, 0, 0, 0, 0, 0, 0, 0, 0, 3840, 0, 0, 3204, 0, 0, 0, 1, 22, 362, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3426, 4048, 4122, 3423, 4526, 3366, 2428, 7429] avg=4096.0000 max=7429 max_rank=7 min=2428 min_rank=6 imbalance_ratio=1.813721 max_load_reduction=25339 max_load_reduction_pct=0.773285 moved_to_echo=17749 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2, 3623, 0, 12, 0, 1, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 246, 3487, 0, 0, 603, 0, 1106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3526, 0, 0, 0, 0, 0, 0, 273, 184, 448, 37, 0, 0, 0, 0, 3, 17, 2, 0, 0, 13, 0, 480, 2717, 938, 0, 0, 0, 0, 0, 0, 1286, 16, 3732, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 2779, 0, 2375, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 0, 0, 1236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 65, 0, 2945, 310, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3655, 5442, 4468, 4170, 5157, 5189, 1354, 3333] avg=4096.0000 max=5442 max_rank=1 min=1354 min_rank=6 imbalance_ratio=1.328613 max_load_reduction=27326 max_load_reduction_pct=0.833923 moved_to_echo=15033 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[9216, 128, 128, 7424, 2688, 128, 128, 256, 11904, 1664, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1920, 128, 4608, 1280, 256, 128, 128, 512, 20608, 3840, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 7424, 4352, 1408, 8320, 4864, 128, 6528, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4480, 128, 13696, 2688, 12032, 128, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[3584, 4608, 23040, 512, 128, 128, 128, 1408, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[7936, 9216, 256, 4864, 9344, 128, 256, 1408, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 22400, 1152, 128, 256, 5504, 0, 0, 1408, 1152, 1024, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[896, 19968, 256, 128, 4480, 4352, 1280, 1792, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 16, 4042, 1131, 0, 0, 13, 0, 2, 0, 0, 0, 0, 0, 2, 493, 0, 12, 15, 0, 0, 3953, 0, 0, 0, 0, 11, 71, 252, 4075, 653, 0, 0, 0, 2, 14, 168, 0, 0, 0, 0, 4, 0, 0, 402, 3922, 0, 5, 110, 0, 3186, 0, 0, 2, 0, 11, 0, 2030, 4078, 0, 4092, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 24, 43, 1204, 2, 2539, 0, 0, 0, 0, 0, 1, 0, 2029, 4080, 224, 0, 0, 0, 4005, 1, 0, 0, 17, 0, 0, 762, 372, 3849, 0, 126, 0, 1, 0, 0, 172, 132, 27, 1, 640, 0, 108, 3533, 242, 0, 0, 138, 6, 4, 0, 0, 442, 0, 2, 2, 3988, 3874, 0, 176, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[4, 178, 0, 3, 94, 0, 221, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 3886, 0, 3449, 17, 0, 2, 0, 2, 75, 0, 0, 0, 28, 0, 274, 3421, 1, 0, 1483, 0, 0, 0, 0, 0, 0, 0, 2096, 81, 5, 0, 1849, 1, 2416, 0, 3, 634, 790, 182, 7, 0, 0, 0, 3848, 3531, 0, 34, 58, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[3, 11, 201, 522, 4033, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 512, 35, 113, 4091, 0, 76, 0, 3989, 0, 0, 2, 0, 992, 0, 0, 141, 1658, 0, 0, 2, 4, 0, 98, 0, 0, 0, 1, 1140, 1, 37, 501, 2892, 0, 0, 0, 15, 15, 0, 1319, 0, 0, 0, 0, 2958, 4087, 4, 3314, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 1113, 0, 49, 9, 6, 26, 0, 3743, 0, 15, 24, 8, 2216, 78, 0, 0, 0, 42, 68, 0, 0, 32, 3368, 35, 0, 276, 4080, 0, 1, 35, 19, 244, 0, 0, 0, 1, 6, 0, 3907, 0, 0, 95, 982, 0, 0, 0, 0, 4058, 3514, 0, 0, 1, 2, 270, 4029, 130, 0, 0, 0, 283] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 1451, 36, 880, 0, 0, 494, 0, 2, 2975, 0, 0, 0, 0, 1181, 160, 0, 1000, 0, 726, 4078, 0, 9, 0, 0, 7, 0, 0, 4, 3348, 0, 4, 0, 0, 0, 0, 0, 1, 299, 10, 2638, 5, 0, 466, 3729, 27, 6, 0, 0, 6, 977, 41, 0, 0, 0, 0, 4091, 648, 0, 3465, 4, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 574, 3577, 5, 48, 0, 0, 13, 6, 0, 0, 3, 0, 2, 0, 1726, 0, 60, 0, 333, 3749, 1, 0, 0, 0, 0, 0, 398, 3356, 3070, 15, 95, 0, 5, 0, 186, 2, 0, 2603, 0, 18, 686, 1, 0, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 681, 0, 3790, 4051, 0, 3602, 9, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 161, 29, 2, 0, 0, 3257, 335, 0, 0, 1, 0, 346, 0, 3691, 0, 0, 3969, 394, 607, 4040, 0, 0, 8, 661, 0, 0, 1, 2136, 1255, 63, 0, 0, 0, 0, 0, 0, 254, 513, 0, 4, 0, 2, 0, 51, 0, 5, 2133, 0, 983, 15, 0, 240, 61, 0, 13, 3645, 3893, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 16, 4042, 1131, 0, 0, 13, 1367, 809, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 2377, 0, 0, 0, 0, 0, 0, 0, 423, 0, 12, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 71, 252, 4075, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 14, 168, 0, 0, 2030, 209, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 402, 3678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 110, 0, 3186, 0, 0, 2, 0, 224, 457, 70, 244, 0, 0, 0, 0, 11, 0, 0, 2812, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7379, 2381, 450, 4838, 2423, 4084, 4298, 6915] avg=4096.0000 max=7379 max_rank=0 min=450 min_rank=2 imbalance_ratio=1.801514 max_load_reduction=25389 max_load_reduction_pct=0.774811 moved_to_echo=17720 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3, 11, 201, 522, 4033, 0, 0, 0, 1379, 811, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 512, 2398, 0, 0, 0, 0, 0, 0, 0, 30, 113, 4091, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 992, 0, 0, 141, 1089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 0, 98, 0, 0, 2958, 212, 0, 0, 0, 0, 0, 0, 0, 1, 1140, 1, 37, 501, 2712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 15, 0, 1319, 0, 0, 569, 458, 5, 180, 0, 0, 0, 0, 0, 0, 0, 2818, 4, 3314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6960, 2911, 4310, 2224, 3274, 4392, 2561, 6136] avg=4096.0000 max=6960 max_rank=0 min=2224 min_rank=3 imbalance_ratio=1.699219 max_load_reduction=25808 max_load_reduction_pct=0.787598 moved_to_echo=16363 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 161, 29, 2, 0, 0, 1397, 772, 0, 0, 0, 0, 0, 0, 3257, 335, 0, 0, 1, 0, 346, 0, 2429, 0, 0, 0, 0, 0, 0, 0, 3162, 0, 0, 3969, 394, 607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 661, 0, 0, 1, 2136, 824, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 254, 3645, 214, 0, 0, 0, 0, 0, 0, 513, 0, 4, 0, 2, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2133, 0, 983, 15, 0, 240, 61, 431, 436, 529, 3, 0, 0, 0, 0, 0, 13, 0, 2685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2361, 6368, 8132, 3630, 4176, 567, 4836, 2698] avg=4096.0000 max=8132 max_rank=2 min=567 min_rank=5 imbalance_ratio=1.985352 max_load_reduction=24636 max_load_reduction_pct=0.751831 moved_to_echo=12277 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 24, 43, 1204, 2, 1386, 768, 0, 0, 0, 0, 0, 0, 2539, 0, 0, 0, 0, 0, 1, 0, 2408, 0, 0, 0, 0, 0, 0, 0, 1738, 4080, 224, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 762, 372, 2527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 0, 1, 0, 0, 172, 132, 3988, 211, 0, 0, 0, 0, 0, 0, 27, 1, 640, 0, 108, 3533, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 6, 4, 0, 0, 442, 0, 1322, 434, 291, 15, 0, 0, 0, 0, 2, 2, 0, 2672, 0, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3429, 4948, 6043, 3678, 4630, 4536, 2652, 2852] avg=4096.0000 max=6043 max_rank=2 min=2652 min_rank=6 imbalance_ratio=1.475342 max_load_reduction=26725 max_load_reduction_pct=0.815582 moved_to_echo=14670 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4, 178, 0, 3, 94, 0, 221, 1, 0, 700, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3329, 0, 3449, 17, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 28, 0, 274, 2246, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1483, 0, 0, 0, 0, 0, 3848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2096, 81, 5, 0, 1734, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2416, 0, 3, 634, 790, 182, 7, 0, 1175, 395, 557, 115, 0, 0, 0, 0, 0, 0, 0, 2436, 0, 34, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1201, 4094, 6799, 2623, 5332, 3917, 6274, 2528] avg=4096.0000 max=6799 max_rank=2 min=1201 min_rank=0 imbalance_ratio=1.659912 max_load_reduction=25969 max_load_reduction_pct=0.792511 moved_to_echo=18051 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 1113, 0, 49, 9, 24, 25, 0, 0, 0, 0, 0, 0, 6, 26, 0, 3743, 0, 15, 24, 8, 41, 0, 0, 0, 0, 0, 0, 0, 1899, 78, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 3368, 35, 0, 276, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 35, 19, 244, 0, 0, 0, 1, 4029, 3, 0, 0, 0, 0, 0, 0, 6, 0, 3907, 0, 0, 95, 921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 3514, 0, 0, 1, 0, 14, 317, 61, 0, 0, 0, 0, 2, 270, 0, 91, 0, 0, 0, 283, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1223, 3863, 2019, 7791, 4332, 4929, 7965, 646] avg=4096.0000 max=7965 max_rank=6 min=646 min_rank=7 imbalance_ratio=1.944580 max_load_reduction=24803 max_load_reduction_pct=0.756927 moved_to_echo=17872 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 574, 3577, 5, 48, 0, 1296, 804, 0, 0, 0, 0, 0, 0, 0, 13, 6, 0, 0, 3, 0, 2, 2256, 0, 0, 0, 0, 0, 0, 0, 0, 1726, 0, 60, 0, 333, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 398, 3356, 2016, 0, 0, 0, 0, 0, 0, 0, 0, 15, 95, 0, 5, 0, 186, 2, 0, 3790, 197, 0, 0, 0, 0, 0, 0, 2603, 0, 18, 686, 1, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 454, 0, 6, 0, 0, 0, 0, 681, 0, 0, 2793, 0, 3602, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6304, 2280, 2120, 5770, 4290, 3405, 1514, 7085] avg=4096.0000 max=7085 max_rank=7 min=1514 min_rank=6 imbalance_ratio=1.729736 max_load_reduction=25683 max_load_reduction_pct=0.783783 moved_to_echo=16294 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1451, 36, 880, 0, 0, 1411, 133, 0, 0, 0, 0, 0, 0, 494, 0, 2, 2975, 0, 0, 0, 0, 2452, 0, 0, 0, 0, 0, 0, 0, 1009, 160, 0, 1000, 0, 726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 7, 0, 0, 4, 2194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 4091, 215, 0, 0, 0, 0, 0, 0, 299, 10, 2638, 5, 0, 466, 3493, 27, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 6, 977, 41, 0, 0, 1154, 76, 172, 236, 0, 0, 0, 0, 0, 0, 0, 439, 0, 3465, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3911, 5923, 2895, 2214, 4311, 6938, 2668, 3908] avg=4096.0000 max=6938 max_rank=5 min=2214 min_rank=3 imbalance_ratio=1.693848 max_load_reduction=25830 max_load_reduction_pct=0.788269 moved_to_echo=17825 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 256, 256, 6784, 10112, 1024, 1536, 128, 8320, 4864, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[2432, 2432, 128, 8960, 5376, 1664, 768, 128, 6016, 2816, 2048, 896, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[11648, 6272, 7808, 5120, 512, 1792, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[768, 384, 0, 16768, 128, 14720, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3456, 128, 10496, 896, 256, 5120, 12928, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[10496, 384, 128, 6784, 128, 128, 384, 640, 14464, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 4096, 1152, 128, 1792, 14464, 11392, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 384, 1536, 256, 128, 512, 256, 512, 28416, 1280, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 1674, 3159, 0, 0, 0, 0, 0, 0, 3844, 0, 0, 0, 20, 0, 0, 0, 2, 4096, 26, 18, 958, 0, 2404, 1, 840, 0, 0, 0, 11, 0, 1, 850, 120, 21, 421, 1204, 0, 14, 0, 1113, 2, 0, 2555, 268, 0, 0, 12, 0, 1016, 0, 0, 4, 1550, 4084, 0, 1468, 195, 813, 2, 2, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 2, 0, 3863, 2, 0, 2, 0, 3, 2, 5, 31, 0, 0, 0, 0, 0, 1699, 0, 1, 0, 4096, 1779, 0, 0, 1, 4092, 67, 26, 4094, 0, 0, 0, 0, 0, 1, 0, 3944, 0, 3, 0, 0, 0, 0, 2579, 1654, 0, 0, 0, 23, 3113, 0, 0, 0, 0, 1357, 0, 205, 123, 0, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 1, 647, 791, 26, 0, 0, 0, 0, 0, 3231, 0, 14, 0, 216, 2471, 0, 696, 0, 0, 200, 0, 185, 1, 0, 1630, 9, 0, 36, 239, 0, 3, 0, 0, 0, 1, 3591, 1376, 0, 0, 0, 0, 3, 10, 3936, 58, 389, 0, 0, 0, 4035, 0, 0, 0, 56, 3670, 0, 251, 0, 929, 0, 0, 0, 4067] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 3818, 0, 0, 0, 0, 109, 1, 517, 19, 16, 0, 1, 24, 0, 4006, 8, 19, 0, 3, 127, 0, 2199, 0, 0, 0, 90, 3744, 7, 6, 0, 3405, 0, 0, 4096, 4071, 0, 9, 3, 0, 540, 3, 1004, 0, 0, 33, 0, 0, 0, 0, 293, 5, 0, 652, 0, 869, 1, 929, 74, 0, 1, 2066] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 154, 0, 2, 0, 0, 0, 1, 39, 0, 3158, 0, 0, 947, 1, 0, 0, 1845, 3, 11, 1749, 13, 3, 0, 631, 0, 2809, 1478, 0, 0, 0, 97, 0, 0, 3, 27, 226, 4092, 0, 0, 0, 17, 3947, 0, 3911, 0, 0, 1, 0, 213, 332, 0, 0, 2, 0, 3116, 0, 1, 0, 0, 0, 0, 0, 3939] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 4033, 0, 11, 337, 0, 0, 2, 25, 8, 0, 0, 71, 8, 0, 0, 0, 0, 47, 15, 3845, 0, 3759, 0, 3829, 0, 0, 0, 0, 0, 5, 0, 3, 0, 1, 3382, 793, 0, 41, 0, 1, 0, 331, 0, 1, 0, 0, 0, 37, 1, 0, 0, 0, 4095, 0, 3974, 0, 3878, 225, 8, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 42, 1236, 15, 0, 0, 21, 0, 0, 4061, 0, 1, 0, 0, 218, 1, 2627, 0, 96, 0, 1088, 0, 0, 3997, 0, 3882, 6, 0, 0, 0, 0, 0, 0, 82, 3997, 9, 3510, 1, 1, 0, 7, 518, 0, 0, 1, 0, 62, 0, 0, 3864, 0, 0, 0, 6, 2106, 0, 0, 0, 0, 0, 0, 1313, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 13, 655, 0, 0, 0, 2, 0, 0, 1, 0, 6, 0, 0, 0, 4, 2852, 0, 2327, 0, 0, 125, 5, 30, 0, 12, 0, 0, 4094, 15, 10, 0, 117, 0, 0, 1171, 4094, 0, 0, 0, 1, 48, 0, 4049, 3939, 0, 0, 0, 0, 3598, 2, 0, 0, 0, 4061, 0, 15, 0, 1163, 1, 0, 7, 351] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 3863, 2, 0, 2, 0, 2527, 158, 0, 0, 0, 0, 0, 0, 3, 2, 5, 31, 0, 0, 0, 0, 1824, 482, 0, 0, 0, 0, 0, 0, 0, 1476, 0, 1, 0, 4096, 1779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1083, 67, 26, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2120, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2579, 1654, 0, 0, 215, 0, 0, 0, 0, 0, 0, 0, 0, 23, 3113, 0, 0, 0, 0, 858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 205, 123, 0, 0, 1, 0, 0, 341, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6554, 2347, 7352, 5271, 2124, 4448, 3994, 678] avg=4096.0000 max=7352 max_rank=2 min=678 min_rank=7 imbalance_ratio=1.794922 max_load_reduction=25416 max_load_reduction_pct=0.775635 moved_to_echo=11244 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1674, 3159, 0, 0, 0, 0, 518, 477, 0, 0, 0, 0, 0, 0, 0, 0, 3844, 0, 0, 0, 20, 0, 556, 99, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 26, 18, 958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2404, 1, 223, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 850, 120, 21, 421, 648, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1113, 2, 0, 2555, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 1016, 0, 0, 4, 1550, 2579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1468, 195, 813, 2, 2, 0, 0, 1028, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5828, 4519, 5100, 2639, 2075, 3938, 5161, 3508] avg=4096.0000 max=5828 max_rank=0 min=2075 min_rank=4 imbalance_ratio=1.422852 max_load_reduction=26940 max_load_reduction_pct=0.822144 moved_to_echo=14682 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4033, 0, 11, 337, 0, 2364, 478, 0, 0, 0, 0, 0, 0, 0, 2, 25, 8, 0, 0, 71, 8, 1564, 451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 15, 3845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3759, 0, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3, 0, 1, 1818, 793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 1, 0, 331, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 1, 0, 0, 0, 2586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3974, 0, 3878, 225, 8, 0, 2, 1031, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7223, 2129, 3907, 4773, 2620, 374, 2624, 9118] avg=4096.0000 max=9118 max_rank=7 min=374 min_rank=5 imbalance_ratio=2.226074 max_load_reduction=23650 max_load_reduction_pct=0.721741 moved_to_echo=14736 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 42, 1236, 15, 0, 0, 21, 2398, 246, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 1, 0, 0, 218, 1623, 459, 0, 0, 0, 0, 0, 0, 1, 2282, 0, 96, 0, 1088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3997, 0, 1025, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 3997, 9, 1887, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 518, 0, 0, 1, 0, 62, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3864, 0, 0, 0, 6, 1330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1313, 0, 530, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3958, 6362, 3467, 5028, 5977, 921, 5200, 1855] avg=4096.0000 max=6362 max_rank=1 min=921 min_rank=5 imbalance_ratio=1.553223 max_load_reduction=26406 max_load_reduction_pct=0.805847 moved_to_echo=13953 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3818, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 109, 1, 517, 19, 16, 0, 1, 24, 1882, 0, 0, 0, 0, 0, 0, 0, 0, 3480, 8, 19, 0, 3, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2199, 0, 0, 0, 90, 3744, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3405, 0, 0, 4096, 2189, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 540, 3, 1004, 0, 0, 33, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 5, 0, 412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 869, 1, 929, 74, 0, 1, 2066, 164, 18, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3894, 2569, 3637, 6046, 9699, 2091, 710, 4122] avg=4096.0000 max=9699 max_rank=4 min=710 min_rank=6 imbalance_ratio=2.367920 max_load_reduction=23069 max_load_reduction_pct=0.704010 moved_to_echo=16622 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 154, 0, 2, 0, 0, 0, 1, 1734, 364, 0, 0, 0, 0, 0, 0, 39, 0, 3158, 0, 0, 947, 1, 0, 1892, 331, 0, 0, 0, 0, 0, 0, 0, 1603, 3, 11, 1749, 13, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 631, 0, 744, 1478, 0, 0, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 27, 226, 2200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 3947, 0, 3911, 0, 0, 1, 234, 0, 0, 0, 0, 0, 0, 0, 0, 213, 332, 0, 0, 2, 0, 1967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3939, 785, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2255, 6368, 3382, 2950, 2456, 8110, 2514, 4733] avg=4096.0000 max=8110 max_rank=5 min=2255 min_rank=0 imbalance_ratio=1.979980 max_load_reduction=24658 max_load_reduction_pct=0.752502 moved_to_echo=17813 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 13, 655, 0, 0, 0, 2, 7, 474, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, 0, 1893, 1, 0, 0, 0, 0, 0, 0, 4, 2477, 0, 2327, 0, 0, 125, 5, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 4, 0, 0, 4094, 15, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 0, 0, 1171, 2201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 48, 0, 4049, 3939, 0, 0, 362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3598, 2, 0, 0, 0, 2564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 1163, 1, 0, 7, 351, 1023, 13, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1151, 1901, 4938, 4153, 3489, 8399, 6164, 2573] avg=4096.0000 max=8399 max_rank=5 min=1151 min_rank=0 imbalance_ratio=2.050537 max_load_reduction=24369 max_load_reduction_pct=0.743683 moved_to_echo=20625 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 647, 791, 26, 0, 0, 0, 8, 432, 0, 0, 0, 0, 0, 0, 0, 0, 3231, 0, 14, 0, 216, 2471, 640, 1, 0, 0, 0, 0, 0, 0, 0, 599, 0, 0, 200, 0, 185, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1630, 0, 0, 36, 239, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3591, 736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 10, 3936, 58, 389, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 0, 0, 0, 56, 2310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 0, 929, 0, 0, 0, 4067, 928, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1905, 6573, 985, 1908, 4328, 4487, 6401, 6181] avg=4096.0000 max=6573 max_rank=1 min=985 min_rank=2 imbalance_ratio=1.604736 max_load_reduction=26195 max_load_reduction_pct=0.799408 moved_to_echo=21397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 4480, 256, 4096, 9600, 13824, 896, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 256, 2432, 17664, 128, 128, 384, 128, 9600, 2816, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 1152, 5120, 128, 18432, 6016, 512, 128, 1792, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 256, 16000, 128, 384, 128, 1664, 14720, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[256, 128, 14848, 128, 128, 1024, 384, 2816, 11904, 1920, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 12032, 128, 6656, 2048, 5248, 7040, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[13056, 1664, 4096, 1664, 256, 12288, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 6784, 384, 7808, 384, 128, 1408, 10496, 5888, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1991, 2, 0, 0, 1437, 0, 19, 149, 20, 3, 0, 0, 0, 0, 138, 0, 0, 0, 11, 37, 0, 7, 4089, 0, 3295, 0, 0, 0, 0, 0, 63, 0, 3836, 0, 0, 65, 3658, 1209, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 3, 0, 462, 4051, 0, 1, 4092, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 0, 1805, 0, 9, 0, 108, 0, 0, 0, 16, 0, 264, 0, 0, 0, 4002, 0, 0, 0, 0, 0, 0, 87, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 3772, 0, 0, 110, 1054, 4018, 382, 0, 5, 0, 284, 0, 0, 3, 0, 0, 705, 0, 0, 0, 215, 4084, 0, 4092, 3656, 0, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 44, 0, 0, 6, 0, 0, 0, 1084, 0, 0, 0, 44, 1, 3, 0, 2, 27, 0, 0, 0, 2398, 2164, 81, 3912, 4, 130, 0, 0, 0, 0, 2726, 770, 159, 0, 0, 0, 449, 1333, 1304, 0, 1, 0, 0, 15, 0, 2278, 151, 1, 0, 0, 178, 2070, 3999, 0, 3376, 4057, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1, 6, 0, 0, 3, 2, 1, 29, 0, 0, 421, 0, 0, 0, 0, 3980, 15, 67, 0, 2992, 0, 0, 0, 0, 10, 1633, 0, 4096, 0, 3613, 60, 0, 3823, 0, 0, 0, 32, 0, 96, 0, 0, 0, 1648, 0, 476, 12, 0, 1, 0, 288, 5, 36, 0, 392, 54, 0, 39, 2470, 3612, 5, 44, 2805, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1371, 0, 0, 3500, 0, 1493, 616, 210, 0, 65, 0, 0, 0, 0, 0, 0, 161, 4, 0, 3083, 45, 0, 3991, 0, 0, 0, 3362, 3419, 0, 3868, 0, 0, 41, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 3943, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2399, 0, 0, 0, 1188, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[517, 3786, 0, 0, 2, 229, 2, 101, 0, 0, 0, 31, 56, 0, 1217, 0, 68, 0, 0, 444, 0, 80, 4070, 141, 10, 3, 0, 506, 73, 3290, 41, 0, 549, 0, 0, 610, 0, 3912, 51, 0, 0, 0, 1823, 150, 3818, 0, 0, 1754, 1, 0, 0, 5, 1092, 0, 21, 0, 0, 212, 0, 0, 0, 399, 0, 3704] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 1297, 2425, 0, 0, 0, 160, 0, 0, 0, 0, 1, 0, 1, 607, 2623, 3, 911, 0, 0, 1928, 0, 0, 160, 0, 3672, 376, 3, 2471, 0, 45, 15, 0, 109, 0, 0, 0, 0, 3535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 239, 4012, 12, 0, 0, 4063, 1, 0, 46, 3869, 0, 182] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3, 0, 0, 0, 374, 0, 0, 1, 2, 0, 1179, 0, 0, 0, 3773, 134, 1126, 0, 1, 4096, 0, 1, 1556, 1, 3918, 0, 0, 0, 0, 0, 0, 16, 10, 37, 0, 3188, 0, 4071, 754, 208, 0, 8, 0, 2, 0, 0, 3, 199, 34, 0, 0, 0, 18, 3428, 0, 161, 4070, 0, 396] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1991, 2, 0, 528, 1016, 0, 0, 0, 0, 0, 0, 0, 1437, 0, 19, 149, 20, 3, 0, 3076, 448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 0, 0, 0, 11, 887, 0, 0, 0, 0, 0, 0, 0, 37, 0, 7, 4089, 0, 272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 3836, 0, 0, 1341, 235, 0, 0, 0, 0, 0, 0, 65, 3658, 446, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 3, 2575, 0, 0, 0, 0, 0, 0, 0, 0, 462, 1823, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3537, 5152, 1036, 4405, 5475, 8264, 2611, 2288] avg=4096.0000 max=8264 max_rank=5 min=1036 min_rank=2 imbalance_ratio=2.017578 max_load_reduction=24504 max_load_reduction_pct=0.747803 moved_to_echo=18638 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 44, 0, 196, 1008, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 1084, 0, 0, 3049, 532, 0, 0, 0, 0, 0, 0, 0, 44, 1, 3, 0, 2, 27, 0, 875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2398, 2164, 81, 322, 4, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2726, 770, 159, 0, 1323, 87, 0, 0, 0, 0, 0, 0, 0, 0, 166, 1333, 1304, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 2278, 151, 1, 0, 0, 3058, 0, 0, 0, 0, 0, 0, 0, 178, 2070, 1801, 0, 3376, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1248, 4671, 952, 5099, 5065, 2804, 5503, 7426] avg=4096.0000 max=7426 max_rank=7 min=952 min_rank=2 imbalance_ratio=1.812988 max_load_reduction=25342 max_load_reduction_pct=0.773376 moved_to_echo=20798 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 6, 0, 0, 3, 2, 1, 29, 719, 697, 0, 0, 0, 0, 0, 0, 0, 0, 421, 0, 0, 0, 0, 3980, 2108, 491, 0, 0, 0, 0, 0, 0, 15, 67, 0, 2992, 0, 0, 0, 0, 791, 0, 0, 0, 0, 0, 0, 0, 10, 1633, 0, 4096, 0, 298, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3823, 0, 0, 0, 32, 0, 96, 0, 1195, 321, 0, 0, 0, 0, 0, 0, 0, 0, 608, 0, 476, 12, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 288, 5, 36, 0, 392, 54, 0, 2824, 0, 0, 0, 0, 0, 0, 0, 39, 2470, 1626, 5, 44, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1458, 7000, 3865, 6097, 5467, 1097, 3599, 4185] avg=4096.0000 max=7000 max_rank=1 min=1097 min_rank=5 imbalance_ratio=1.708984 max_load_reduction=25768 max_load_reduction_pct=0.786377 moved_to_echo=14348 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1371, 0, 0, 3500, 0, 1493, 616, 210, 0, 295, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 893, 526, 0, 0, 0, 0, 0, 0, 161, 4, 0, 3083, 45, 0, 3991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3362, 3419, 0, 319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3023, 0, 0, 0, 0, 0, 0, 0, 0, 2399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7485, 1484, 7284, 7100, 49, 3943, 3024, 2399] avg=4096.0000 max=7485 max_rank=0 min=49 min_rank=4 imbalance_ratio=1.827393 max_load_reduction=25283 max_load_reduction_pct=0.771576 moved_to_echo=9415 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[517, 3786, 0, 0, 2, 229, 2, 101, 796, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 56, 0, 1217, 0, 300, 447, 0, 0, 0, 0, 0, 0, 68, 0, 0, 444, 0, 80, 4070, 141, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3, 0, 506, 73, 272, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 0, 0, 610, 0, 3912, 51, 0, 0, 355, 0, 0, 0, 0, 0, 0, 0, 0, 672, 150, 3818, 0, 0, 1754, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5, 1092, 0, 21, 0, 2571, 0, 0, 0, 0, 0, 0, 0, 0, 212, 0, 0, 0, 0, 0, 3704, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5532, 2051, 4803, 905, 5477, 6394, 3690, 3916] avg=4096.0000 max=6394 max_rank=5 min=905 min_rank=3 imbalance_ratio=1.561035 max_load_reduction=26374 max_load_reduction_pct=0.804871 moved_to_echo=19477 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 0, 1805, 0, 9, 1758, 909, 0, 0, 0, 0, 0, 0, 0, 108, 0, 0, 0, 16, 0, 264, 2747, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4002, 0, 0, 0, 0, 897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3772, 0, 0, 1355, 788, 0, 0, 0, 0, 0, 0, 110, 1054, 1472, 382, 0, 5, 0, 284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 705, 0, 0, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 215, 1832, 0, 4092, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4486, 3695, 4899, 415, 5915, 3307, 3908, 6143] avg=4096.0000 max=6143 max_rank=7 min=415 min_rank=3 imbalance_ratio=1.499756 max_load_reduction=26625 max_load_reduction_pct=0.812531 moved_to_echo=19273 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 1297, 2425, 0, 1544, 962, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 0, 0, 0, 1, 2907, 499, 0, 0, 0, 0, 0, 0, 0, 1, 607, 2623, 3, 911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1928, 0, 0, 160, 0, 303, 376, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2471, 0, 45, 15, 0, 109, 0, 0, 0, 689, 0, 0, 0, 0, 0, 0, 0, 0, 1302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 239, 4012, 12, 0, 2870, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 1, 0, 46, 0, 0, 182, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6230, 3567, 4145, 2770, 3329, 1302, 7133, 4292] avg=4096.0000 max=7133 max_rank=6 min=1302 min_rank=5 imbalance_ratio=1.741455 max_load_reduction=25635 max_load_reduction_pct=0.782318 moved_to_echo=16056 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3, 0, 0, 1778, 1011, 0, 0, 0, 0, 0, 0, 0, 374, 0, 0, 1, 2, 0, 1179, 3059, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3773, 134, 1126, 0, 1, 750, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 1556, 1, 323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 10, 37, 0, 1134, 794, 0, 0, 0, 0, 0, 0, 3188, 0, 1499, 754, 208, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3, 199, 34, 0, 0, 3062, 0, 0, 0, 0, 0, 0, 0, 0, 18, 1544, 0, 161, 0, 0, 396, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2792, 5148, 5784, 5977, 1991, 5657, 3300, 2119] avg=4096.0000 max=5977 max_rank=3 min=1991 min_rank=4 imbalance_ratio=1.459229 max_load_reduction=26791 max_load_reduction_pct=0.817596 moved_to_echo=13067 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1920, 3840, 0, 3584, 128, 6912, 3200, 384, 7424, 6016, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[6912, 0, 128, 768, 2816, 12544, 384, 0, 6400, 3328, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3456, 4736, 6272, 10752, 5888, 128, 128, 2048, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[6144, 1664, 5888, 16000, 256, 2560, 512, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 128, 640, 17152, 256, 2176, 8192, 256, 4224, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 12032, 8704, 128, 7808, 0, 0, 4352, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 2048, 640, 128, 256, 1152, 1280, 5504, 18176, 4096, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 384, 128, 2432, 1792, 5248, 128, 128, 23296, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[8, 243, 0, 0, 4093, 2911, 0, 545, 1, 851, 26, 0, 0, 92, 0, 1, 1, 4050, 1, 0, 3, 1721, 1071, 3736, 0, 25, 5, 0, 0, 0, 0, 22, 0, 0, 1948, 3, 3154, 50, 0, 0, 272, 886, 0, 0, 3, 0, 0, 89, 3717, 0, 0, 230, 0, 0, 0, 0, 0, 0, 1635, 1, 943, 430, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 650, 0, 255, 3159, 12, 0, 24, 0, 0, 0, 0, 0, 267, 1, 1631, 0, 2497, 0, 0, 13, 56, 1489, 4038, 0, 0, 1514, 0, 0, 3, 2410, 0, 31, 1276, 2306, 0, 150, 0, 0, 2820, 0, 72, 182, 3919, 0, 7, 0, 0, 3808, 0, 0, 1, 0, 0, 0, 0, 177, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 2953, 0, 0, 3, 4042, 1195, 0, 0, 0, 0, 0, 7, 8, 0, 0, 0, 2742, 0, 0, 0, 11, 0, 2392, 0, 0, 0, 1, 32, 0, 1593, 0, 12, 8, 4091, 0, 3830, 0, 46, 0, 0, 0, 29, 0, 13, 0, 0, 3649, 4078, 0, 0, 10, 0, 0, 5, 0, 0, 0, 0, 1, 82, 0, 0, 1935] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 89, 0, 431, 0, 2901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3976, 188, 0, 0, 6, 48, 0, 0, 0, 203, 0, 982, 0, 2610, 0, 0, 0, 3994, 0, 4017, 0, 0, 0, 0, 63, 25, 3038, 0, 0, 0, 24, 3966, 0, 0, 0, 0, 9, 4013, 0, 0, 0, 0, 0, 2185, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 274, 217, 3156, 403, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1033, 0, 3853, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 5, 2829, 0, 0, 0, 0, 0, 216, 1, 3624, 0, 1, 43, 0, 0, 418, 2, 0, 0, 0, 0, 3981, 0, 0, 0, 224, 0, 4086, 273, 75, 0, 0, 0, 280, 3682, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 153, 639, 1, 4023, 2623, 95, 0, 0, 0, 0, 0, 21, 937, 0, 0, 0, 2992, 4, 0, 0, 2, 0, 25, 0, 0, 0, 4, 220, 0, 39, 0, 655, 1283, 3708, 1709, 4096, 0, 0, 130, 0, 0, 1, 3963, 5, 2700, 0, 0, 1634, 1, 13, 0, 811, 9, 1, 0, 0, 3, 0, 3, 261, 0, 0, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 3992, 2971, 3989, 0, 75, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 1526, 0, 1, 12, 4, 1901, 0, 391, 0, 512, 0, 2, 11, 3598, 0, 73, 16, 3646, 8, 3871, 12, 1, 3, 0, 191, 1, 0, 0, 14, 0, 0, 2759, 0, 0, 0, 0, 160, 0, 0, 299, 0, 24, 5, 284, 2336, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 148, 4085, 3452, 15, 2709, 0, 0, 69, 0, 1, 0, 0, 0, 625, 100, 1123, 0, 63, 0, 0, 0, 2629, 0, 0, 0, 2, 0, 0, 53, 0, 364, 121, 1356, 3, 4072, 0, 115, 0, 5, 0, 167, 0, 5, 0, 0, 0, 4019, 1, 0, 0, 1933, 0, 1582, 19, 1, 3, 22, 15, 2204, 1687, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[8, 243, 0, 0, 0, 2487, 0, 545, 0, 0, 0, 0, 0, 0, 0, 0, 1, 851, 26, 0, 0, 92, 0, 1, 2834, 554, 0, 0, 0, 0, 0, 0, 1, 1631, 1, 0, 3, 1721, 1071, 3736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 5, 0, 0, 0, 0, 22, 2882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1948, 3, 320, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 272, 886, 0, 0, 3, 0, 0, 89, 1139, 532, 424, 0, 0, 0, 0, 0, 2578, 0, 0, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1635, 1, 943, 430, 0, 1, 1887, 657, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3283, 4359, 8164, 2934, 2321, 3345, 2808, 5554] avg=4096.0000 max=8164 max_rank=2 min=2321 min_rank=4 imbalance_ratio=1.993164 max_load_reduction=24604 max_load_reduction_pct=0.750854 moved_to_echo=14028 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 2953, 0, 0, 0, 3453, 1195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8, 0, 0, 3441, 0, 0, 0, 0, 0, 0, 0, 0, 1105, 0, 0, 0, 11, 0, 2392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 32, 0, 1593, 0, 2, 0, 0, 0, 0, 0, 0, 0, 12, 8, 4091, 0, 389, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 13, 0, 0, 3649, 1250, 360, 589, 0, 0, 0, 0, 0, 2828, 0, 0, 10, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 82, 0, 0, 1935, 1277, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7601, 3456, 3508, 1628, 4546, 5890, 2843, 3296] avg=4096.0000 max=7601 max_rank=0 min=1628 min_rank=3 imbalance_ratio=1.855713 max_load_reduction=25167 max_load_reduction_pct=0.768036 moved_to_echo=16575 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 650, 0, 255, 0, 11, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 267, 1, 1631, 134, 427, 0, 0, 0, 0, 0, 0, 0, 1006, 0, 0, 13, 56, 1489, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1514, 0, 0, 3, 2410, 0, 2224, 0, 0, 0, 0, 0, 0, 0, 31, 1276, 2306, 0, 16, 0, 0, 2820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 182, 3919, 0, 7, 0, 0, 1167, 328, 1, 0, 0, 0, 0, 0, 2641, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177, 0, 0, 0, 0, 0, 0, 0, 1163, 508, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[940, 2460, 6602, 6151, 6449, 5676, 2642, 1848] avg=4096.0000 max=6602 max_rank=2 min=940 min_rank=0 imbalance_ratio=1.611816 max_load_reduction=26166 max_load_reduction_pct=0.798523 moved_to_echo=16615 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 89, 0, 0, 0, 2901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3609, 58, 0, 0, 0, 0, 0, 0, 0, 1602, 188, 0, 0, 6, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 203, 0, 982, 0, 2610, 0, 303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3994, 0, 408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 25, 3038, 0, 0, 0, 24, 1216, 522, 0, 0, 0, 0, 0, 0, 2750, 0, 0, 0, 0, 9, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2185, 0, 0, 0, 1852, 70, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2990, 3667, 1844, 4098, 4402, 4888, 6772, 4107] avg=4096.0000 max=6772 max_rank=6 min=1844 min_rank=2 imbalance_ratio=1.653320 max_load_reduction=25996 max_load_reduction_pct=0.793335 moved_to_echo=20169 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 148, 4085, 0, 13, 2709, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 1, 0, 0, 0, 625, 3659, 468, 0, 0, 0, 0, 0, 0, 100, 453, 0, 63, 0, 0, 0, 2629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 53, 0, 2430, 0, 0, 0, 0, 0, 0, 0, 364, 121, 1356, 3, 413, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 167, 0, 5, 0, 0, 0, 1232, 147, 2, 0, 0, 0, 0, 0, 2787, 1, 0, 0, 1933, 0, 1582, 19, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 22, 15, 2204, 1687, 0, 0, 523, 554, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6955, 4822, 3245, 2485, 2372, 1558, 6322, 5009] avg=4096.0000 max=6955 max_rank=0 min=1558 min_rank=5 imbalance_ratio=1.697998 max_load_reduction=25813 max_load_reduction_pct=0.787750 moved_to_echo=15261 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3992, 2971, 0, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 0, 0, 3478, 540, 0, 0, 0, 0, 0, 0, 0, 615, 0, 1, 12, 4, 1901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 391, 0, 512, 0, 2, 11, 3598, 0, 2809, 0, 0, 0, 0, 0, 0, 0, 73, 16, 3646, 8, 393, 12, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 191, 1, 0, 0, 14, 0, 0, 846, 200, 0, 0, 0, 0, 0, 0, 1913, 0, 0, 0, 0, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 299, 0, 24, 5, 284, 2336, 0, 2, 711, 640, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7038, 4096, 2533, 7323, 4152, 1252, 2073, 4301] avg=4096.0000 max=7323 max_rank=3 min=1252 min_rank=5 imbalance_ratio=1.787842 max_load_reduction=25445 max_load_reduction_pct=0.776520 moved_to_echo=11778 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 274, 217, 0, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1033, 3260, 429, 0, 0, 0, 0, 0, 0, 0, 1546, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2829, 0, 0, 0, 2221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 1, 364, 0, 1, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 418, 2, 0, 0, 0, 0, 1224, 508, 62, 0, 0, 0, 0, 0, 2757, 0, 0, 0, 224, 0, 4086, 273, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 280, 3682, 0, 0, 1799, 506, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[832, 4724, 5636, 5055, 625, 2214, 7340, 6342] avg=4096.0000 max=7340 max_rank=6 min=625 min_rank=4 imbalance_ratio=1.791992 max_load_reduction=25428 max_load_reduction_pct=0.776001 moved_to_echo=16521 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 153, 639, 1, 0, 2241, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 937, 0, 0, 3680, 546, 0, 0, 0, 0, 0, 0, 0, 1205, 4, 0, 0, 2, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 220, 0, 39, 0, 2831, 0, 0, 0, 0, 0, 0, 0, 655, 1283, 3708, 1709, 416, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3963, 5, 2700, 0, 0, 501, 393, 382, 0, 0, 0, 0, 0, 1133, 1, 13, 0, 811, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 261, 0, 0, 4, 1394, 646, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3129, 5184, 1236, 3094, 7901, 7945, 1968, 2311] avg=4096.0000 max=7945 max_rank=5 min=1236 min_rank=2 imbalance_ratio=1.939697 max_load_reduction=24823 max_load_reduction_pct=0.757538 moved_to_echo=20125 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[384, 1280, 896, 11008, 128, 2816, 0, 3840, 8576, 3072, 1536, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 4096, 5248, 7552, 0, 8576, 7040, 640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[640, 128, 1792, 128, 6272, 8192, 0, 2048, 10624, 3584, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[19456, 128, 128, 256, 3072, 256, 9728, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[512, 128, 2304, 128, 4096, 128, 10368, 128, 15744, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1152, 2816, 21376, 1792, 2816, 128, 256, 3072, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 1024, 128, 128, 128, 1408, 128, 3328, 24192, 3072, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 9216, 256, 128, 128, 1920, 4608, 17024, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 1232, 3292, 0, 1447, 0, 0, 0, 3003, 3053, 3710, 0, 13, 0, 0, 31, 0, 2, 8, 0, 0, 0, 19, 0, 3, 2223, 0, 0, 0, 17, 6, 54, 3960, 0, 0, 725, 4, 208, 207, 0, 0, 46, 0, 0, 0, 0, 0, 3920, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1488, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 43, 3599, 191, 0, 0, 0, 0, 0, 0, 0, 0, 3711, 0, 0, 233, 0, 5, 0, 0, 1, 0, 0, 5, 3907, 3181, 1182, 0, 0, 0, 1324, 4092, 0, 0, 2767, 6, 4090, 0, 30, 0, 0, 32, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 3743, 0, 529, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 2, 2603, 4, 0, 3805, 0, 1, 0, 0, 0, 563, 1, 0, 7, 643, 3, 37, 567, 4054, 0, 0, 338, 0, 926, 0, 230, 0, 178, 0, 3, 37, 0, 0, 98, 3172, 0, 5, 0, 3485, 0, 0, 0, 1, 0, 0, 3345, 11, 0, 233, 362, 0, 0, 0, 3946, 0, 1, 0, 24, 4083, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 1839, 0, 5, 4091, 71, 3821, 0, 0, 63, 0, 0, 0, 34, 0, 0, 4029, 3996, 233, 2163, 3984, 0, 6, 93, 142, 0, 0, 0, 10, 0, 0, 0, 824, 0, 2, 14, 2185, 0, 0, 0, 3015, 0, 0, 0, 0, 4, 0, 168, 0, 0, 36, 1888, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 33, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 422, 5, 3722, 3724, 245, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3321, 3929, 0, 166, 13, 0, 0, 21, 1, 8, 0, 0, 0, 3661, 0, 0, 2575, 0, 0, 0, 1688, 0, 0, 0, 3416, 0, 0, 49, 0, 0, 0, 0, 1529, 0, 0, 9, 2, 0, 8, 6, 4, 0, 0, 0, 147, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 9, 1, 3819, 0, 3019, 0, 0, 6, 1244, 0, 588, 0, 106, 0, 0, 2580, 0, 0, 0, 0, 0, 36, 0, 0, 2, 0, 2, 4085, 0, 7, 0, 0, 0, 0, 4043, 0, 0, 6, 3840, 2194, 0, 10, 66, 1, 159, 0, 1, 4021, 0, 0, 0, 6, 0, 2091, 0, 0, 0, 808, 1, 17, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1065, 46, 4054, 96, 153, 0, 0, 0, 69, 0, 0, 0, 3985, 3003, 0, 202, 4035, 0, 0, 2, 0, 0, 0, 0, 0, 4066, 0, 0, 15, 3, 0, 16, 4072, 0, 0, 3, 4082, 0, 0, 3, 0, 0, 0, 3760, 6, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 1, 4093, 3765, 11, 1282, 1, 1, 0, 0, 0, 1, 75, 0, 0, 0, 0, 0, 0, 0, 242, 0, 0, 1, 4, 0, 0, 3994, 2257, 0, 0, 3149, 2300, 0, 2, 1190, 0, 0, 136, 3, 0, 0, 0, 0, 23, 0, 3432, 0, 87, 0, 0, 0, 0, 0, 2310, 4, 0, 0, 0, 4096, 0, 0, 308] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 2603, 4, 0, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 563, 1, 0, 7, 2630, 0, 0, 0, 0, 0, 0, 0, 115, 3, 37, 567, 4054, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 926, 0, 230, 0, 178, 0, 3, 424, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 98, 2279, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3024, 0, 0, 0, 1, 0, 0, 3345, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 233, 362, 0, 0, 0, 3946, 254, 893, 37, 0, 0, 0, 0, 0, 0, 1, 0, 24, 4083, 0, 0, 0, 274, 940, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2844, 3202, 5114, 1761, 2419, 6370, 5736, 5322] avg=4096.0000 max=6370 max_rank=5 min=1761 min_rank=3 imbalance_ratio=1.555176 max_load_reduction=26398 max_load_reduction_pct=0.805603 moved_to_echo=19847 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 422, 5, 3722, 3724, 245, 253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2831, 0, 0, 0, 0, 0, 0, 0, 591, 3929, 0, 166, 13, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8, 0, 0, 0, 3661, 0, 0, 415, 0, 0, 0, 0, 0, 0, 0, 2575, 0, 0, 0, 1213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2965, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1529, 0, 0, 9, 2, 0, 8, 6, 1314, 475, 36, 0, 0, 0, 0, 0, 4, 0, 0, 0, 147, 0, 0, 1, 1416, 1012, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8371, 2831, 4720, 4085, 3788, 3014, 3379, 2580] avg=4096.0000 max=8371 max_rank=0 min=2580 min_rank=7 imbalance_ratio=2.043701 max_load_reduction=24397 max_load_reduction_pct=0.744537 moved_to_echo=12761 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 43, 223, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2487, 0, 0, 0, 0, 0, 0, 0, 653, 0, 0, 233, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5, 3907, 3181, 1182, 0, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1324, 4092, 0, 0, 2767, 6, 0, 0, 0, 0, 0, 0, 0, 0, 3545, 0, 30, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 1472, 0, 46, 0, 0, 0, 0, 0, 0, 0, 15, 0, 3743, 0, 529, 0, 1586, 889, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[459, 2487, 891, 8775, 8189, 3607, 1598, 6762] avg=4096.0000 max=8775 max_rank=3 min=459 min_rank=0 imbalance_ratio=2.142334 max_load_reduction=23993 max_load_reduction_pct=0.732208 moved_to_echo=20156 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1839, 0, 5, 4091, 71, 236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 34, 0, 0, 2641, 0, 0, 0, 0, 0, 0, 0, 716, 3996, 233, 2163, 3984, 0, 6, 93, 0, 0, 0, 0, 0, 0, 0, 0, 142, 0, 0, 0, 10, 0, 0, 0, 366, 0, 0, 0, 0, 0, 0, 0, 824, 0, 2, 14, 1570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2617, 0, 0, 0, 0, 4, 0, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 1888, 0, 0, 19, 0, 1595, 615, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 1718, 944, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6242, 2738, 11191, 518, 2410, 2789, 4185, 2695] avg=4096.0000 max=11191 max_rank=2 min=518 min_rank=3 imbalance_ratio=2.732178 max_load_reduction=21577 max_load_reduction_pct=0.658478 moved_to_echo=12079 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1065, 46, 250, 96, 0, 0, 0, 0, 0, 0, 0, 0, 153, 0, 0, 0, 69, 0, 0, 0, 2802, 0, 0, 0, 0, 0, 0, 0, 708, 3003, 0, 202, 4035, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 496, 0, 0, 0, 0, 0, 0, 0, 15, 3, 0, 16, 2926, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3543, 0, 0, 3, 0, 0, 0, 3760, 0, 0, 0, 0, 0, 0, 0, 0, 6, 27, 0, 0, 0, 0, 0, 0, 1577, 1146, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1700, 1002, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1457, 3024, 7950, 4562, 2963, 7306, 2799, 2707] avg=4096.0000 max=7950 max_rank=2 min=1457 min_rank=0 imbalance_ratio=1.940918 max_load_reduction=24818 max_load_reduction_pct=0.757385 moved_to_echo=15775 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 1232, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1447, 0, 0, 0, 3003, 3053, 3710, 0, 2275, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 31, 0, 2, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 3, 2223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 6, 54, 2846, 0, 0, 725, 0, 0, 0, 0, 0, 0, 0, 0, 4, 208, 207, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3920, 0, 0, 0, 0, 5, 1114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1488, 0, 0, 5, 813, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1437, 13488, 44, 2245, 3648, 465, 5039, 6402] avg=4096.0000 max=13488 max_rank=1 min=44 min_rank=2 imbalance_ratio=3.292969 max_load_reduction=19280 max_load_reduction_pct=0.588379 moved_to_echo=15554 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 9, 1, 3819, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1244, 0, 588, 0, 106, 0, 2087, 0, 0, 0, 0, 0, 0, 0, 0, 2580, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 4085, 0, 7, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2905, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 3332, 2194, 0, 10, 66, 1, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4021, 0, 0, 0, 6, 0, 2091, 0, 1138, 41, 0, 0, 0, 0, 0, 0, 0, 0, 808, 1, 17, 0, 0, 0, 746, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4015, 4031, 2616, 4563, 2911, 5762, 7298, 1572] avg=4096.0000 max=7298 max_rank=6 min=1572 min_rank=7 imbalance_ratio=1.781738 max_load_reduction=25470 max_load_reduction_pct=0.777283 moved_to_echo=17543 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 4093, 3765, 11, 72, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 75, 0, 0, 890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 3994, 2257, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3149, 2300, 0, 2, 853, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 3432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 0, 0, 0, 2310, 0, 337, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 308, 0, 320, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7943, 967, 242, 6259, 6440, 3455, 2734, 4728] avg=4096.0000 max=7943 max_rank=0 min=242 min_rank=2 imbalance_ratio=1.939209 max_load_reduction=24825 max_load_reduction_pct=0.757599 moved_to_echo=17357 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[2816, 13568, 384, 3456, 12160, 256, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 2304, 128, 10496, 16512, 1664, 1664, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[1664, 4224, 384, 6272, 128, 128, 128, 8448, 6272, 5760, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 896, 16256, 1536, 640, 384, 6784, 6784, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[256, 1024, 128, 256, 7936, 19712, 1280, 128, 2688, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[19072, 2432, 256, 128, 128, 128, 256, 10752, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1664, 128, 1280, 0, 4224, 3200, 3840, 128, 18688, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[6656, 2432, 1408, 4352, 14592, 0, 2816, 896, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 116, 0, 0, 7, 0, 3, 0, 0, 0, 0, 0, 266, 0, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4046, 105, 0, 0, 0, 0, 4078, 0, 0, 242, 0, 0, 0, 721, 0, 3655, 5, 10, 5, 4091, 0, 0, 1, 3854, 0, 2671, 0, 0, 0, 2, 0, 0, 0, 907, 3886, 4093, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 1605, 54, 0, 15, 0, 0, 3880, 86, 356, 0, 0, 142, 0, 0, 22, 293, 0, 0, 0, 0, 0, 0, 4067, 0, 4061, 0, 44, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 8, 0, 1481, 717, 0, 0, 0, 2, 3990, 1288, 3730, 0, 0, 0, 0, 3908, 1, 0, 0, 2975, 0, 0, 28] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 981, 0, 0, 1220, 0, 0, 0, 2184, 14, 361, 1017, 101, 3, 0, 0, 6, 0, 0, 0, 2, 640, 0, 65, 1317, 4092, 0, 2, 0, 0, 530, 0, 0, 1233, 4078, 0, 0, 6, 261, 3391, 0, 0, 0, 3692, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 2454, 3175, 1713, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[54, 0, 26, 1416, 0, 0, 0, 476, 0, 0, 0, 0, 0, 12, 0, 0, 11, 0, 0, 3, 1, 0, 0, 0, 3311, 198, 4093, 3392, 0, 0, 0, 0, 0, 3754, 1748, 4073, 0, 38, 0, 4088, 0, 638, 49, 0, 78, 0, 0, 325, 0, 2131, 348, 0, 2390, 0, 0, 0, 0, 0, 0, 0, 112, 3, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[2, 0, 0, 29, 68, 0, 0, 1968, 7, 1, 0, 3081, 0, 0, 0, 1, 0, 0, 797, 0, 0, 0, 7, 11, 4060, 0, 3203, 0, 1215, 0, 2396, 0, 0, 0, 1, 0, 3147, 0, 4019, 2, 0, 0, 0, 643, 4065, 14, 0, 2, 0, 1977, 0, 1962, 0, 0, 0, 0, 3, 48, 0, 0, 0, 39, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 71, 2, 0, 0, 0, 0, 0, 6, 3, 0, 237, 0, 12, 0, 168, 0, 0, 0, 0, 0, 1272, 0, 1, 4043, 156, 2067, 3227, 1, 0, 416, 0, 202, 0, 0, 2941, 0, 0, 0, 0, 1, 0, 1488, 32, 95, 0, 0, 8, 178, 2579, 1308, 3451, 0, 0, 1, 1347, 3, 3841, 3611, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 41, 0, 0, 183, 5, 4095, 53, 0, 0, 0, 4096, 0, 0, 0, 172, 0, 0, 0, 0, 2083, 1, 0, 4068, 3476, 0, 0, 0, 0, 4068, 0, 23, 0, 0, 0, 339, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 1107, 0, 0, 0, 0, 0, 0, 853, 0, 3987, 4092, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 10, 3404, 0, 0, 0, 0, 0, 1551, 4, 208, 0, 264, 3932, 0, 822, 12, 0, 0, 0, 0, 3969, 93, 0, 0, 0, 0, 2704, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3, 23, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 1752, 17, 4096, 0, 0, 0, 0, 0, 0, 3058, 2751, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 29, 68, 0, 0, 1968, 0, 596, 261, 0, 0, 0, 0, 0, 7, 1, 0, 3081, 0, 0, 0, 1, 1969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 797, 0, 0, 0, 7, 11, 3799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 638, 0, 1215, 0, 2396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3147, 0, 4019, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 643, 4065, 14, 0, 2, 0, 434, 0, 0, 0, 0, 0, 0, 0, 1543, 0, 1962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 48, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2924, 5059, 4614, 4249, 7169, 5158, 3505, 90] avg=4096.0000 max=7169 max_rank=4 min=90 min_rank=7 imbalance_ratio=1.750244 max_load_reduction=25599 max_load_reduction_pct=0.781219 moved_to_echo=15922 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1605, 54, 0, 15, 0, 0, 0, 758, 263, 0, 0, 0, 0, 0, 3880, 86, 356, 0, 0, 142, 0, 0, 2499, 0, 0, 0, 0, 0, 0, 0, 22, 293, 0, 0, 0, 0, 0, 0, 3804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 804, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 1481, 717, 0, 0, 0, 0, 876, 0, 0, 0, 0, 0, 0, 2, 3114, 1288, 3730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3908, 1, 0, 0, 2975, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2695, 6963, 4119, 848, 15, 3082, 8134, 6912] avg=4096.0000 max=8134 max_rank=6 min=15 min_rank=4 imbalance_ratio=1.985840 max_load_reduction=24634 max_load_reduction_pct=0.751770 moved_to_echo=18143 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 116, 0, 0, 7, 0, 3, 0, 2928, 0, 260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, 0, 3, 0, 0, 538, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3786, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 242, 0, 0, 0, 721, 0, 1937, 0, 0, 0, 0, 0, 0, 0, 0, 5, 10, 5, 4091, 0, 0, 1, 3854, 1180, 586, 0, 0, 0, 0, 0, 0, 0, 2085, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 907, 958, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3314, 807, 3787, 4183, 2900, 9732, 2087, 5958] avg=4096.0000 max=9732 max_rank=5 min=807 min_rank=1 imbalance_ratio=2.375977 max_load_reduction=23036 max_load_reduction_pct=0.703003 moved_to_echo=20677 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 71, 2, 0, 0, 0, 0, 0, 2721, 385, 261, 0, 0, 0, 0, 0, 6, 3, 0, 237, 0, 12, 0, 168, 1271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1272, 0, 1, 3782, 0, 0, 0, 0, 0, 0, 0, 0, 156, 411, 3227, 1, 0, 416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 202, 0, 0, 2941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1488, 32, 95, 0, 0, 8, 0, 566, 0, 0, 0, 0, 0, 0, 178, 2013, 1308, 3451, 0, 0, 1, 1347, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3841, 890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3440, 1697, 5055, 4211, 3143, 2190, 8298, 4734] avg=4096.0000 max=8298 max_rank=6 min=1697 min_rank=1 imbalance_ratio=2.025879 max_load_reduction=24470 max_load_reduction_pct=0.746765 moved_to_echo=18365 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[54, 0, 26, 1416, 0, 0, 0, 476, 0, 762, 214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 2516, 602, 0, 0, 0, 0, 0, 0, 11, 0, 0, 3, 1, 0, 0, 0, 3097, 0, 0, 0, 0, 0, 0, 0, 0, 198, 815, 3392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3754, 1748, 4073, 0, 38, 0, 2166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 638, 49, 0, 78, 0, 0, 325, 1320, 467, 0, 0, 0, 0, 0, 0, 0, 1664, 348, 0, 2390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2948, 3130, 3112, 4405, 11779, 2877, 4402, 115] avg=4096.0000 max=11779 max_rank=4 min=115 min_rank=7 imbalance_ratio=2.875732 max_load_reduction=20989 max_load_reduction_pct=0.640533 moved_to_echo=19173 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 981, 0, 0, 1220, 0, 0, 0, 1849, 0, 85, 0, 0, 0, 0, 0, 2184, 14, 361, 1017, 101, 3, 0, 0, 0, 502, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 2, 640, 0, 65, 1232, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 2, 0, 0, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1233, 4078, 0, 0, 6, 261, 1793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3692, 193, 0, 0, 0, 1096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 605, 3175, 1713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4135, 4182, 1945, 4624, 7371, 4981, 0, 5530] avg=4096.0000 max=7371 max_rank=4 min=0 min_rank=6 imbalance_ratio=1.799561 max_load_reduction=25397 max_load_reduction_pct=0.775055 moved_to_echo=17882 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 41, 0, 0, 183, 5, 4095, 3005, 0, 262, 0, 0, 0, 0, 0, 53, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 0, 0, 0, 0, 2083, 1, 0, 3806, 0, 0, 0, 0, 0, 0, 0, 0, 3476, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 243, 0, 0, 0, 0, 0, 0, 0, 864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 853, 0, 982, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7591, 4149, 6062, 7544, 362, 269, 864, 5927] avg=4096.0000 max=7591 max_rank=0 min=269 min_rank=5 imbalance_ratio=1.853271 max_load_reduction=25177 max_load_reduction_pct=0.768341 moved_to_echo=7422 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 10, 3404, 0, 0, 0, 0, 0, 2307, 0, 0, 0, 0, 0, 0, 0, 1551, 4, 208, 0, 264, 3932, 0, 822, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 3969, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 387, 0, 0, 0, 0, 0, 0, 0, 1365, 17, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 751, 2751, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5721, 6781, 4074, 2704, 29, 4479, 5478, 3502] avg=4096.0000 max=6781 max_rank=1 min=29 min_rank=4 imbalance_ratio=1.655518 max_load_reduction=25987 max_load_reduction_pct=0.793060 moved_to_echo=13488 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 5248, 5888, 7040, 3584, 768, 4352, 6016, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[7808, 128, 1024, 4352, 4736, 4224, 128, 1024, 8320, 1664, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 768, 1664, 14080, 5248, 128, 128, 4224, 3712, 3584, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[256, 12672, 3072, 13312, 2432, 128, 128, 1408, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4864, 4864, 4224, 14208, 4864, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 1280, 5120, 1536, 1408, 256, 128, 6656, 12928, 2560, 1664, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 8064, 2688, 9344, 1280, 0, 11520, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[256, 384, 896, 128, 128, 8064, 128, 128, 23424, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[246, 0, 0, 4096, 10, 0, 0, 0, 0, 60, 272, 0, 2343, 1, 1647, 0, 0, 4068, 0, 0, 3441, 0, 2918, 0, 440, 1631, 4027, 15, 0, 5, 20, 0, 0, 0, 0, 115, 0, 0, 0, 1948, 0, 0, 19, 0, 19, 0, 70, 0, 0, 0, 0, 3345, 0, 0, 0, 0, 0, 0, 2010, 0, 0, 2, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3909, 352, 0, 0, 0, 0, 137, 0, 0, 0, 0, 23, 0, 359, 0, 192, 0, 19, 0, 3773, 0, 2264, 29, 21, 0, 1, 250, 0, 0, 0, 4086, 36, 0, 1134, 0, 0, 1301, 2311, 0, 1, 0, 3974, 0, 543, 1, 0, 0, 0, 0, 4049, 0, 0, 0, 0, 0, 101, 0, 3902, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 2263, 0, 8, 2, 0, 8, 0, 0, 0, 608, 0, 4055, 0, 381, 53, 0, 0, 1347, 0, 6, 0, 0, 0, 4049, 1, 0, 0, 0, 982, 0, 0, 5, 0, 0, 0, 0, 3985, 0, 2828, 165, 0, 4014, 0, 0, 0, 0, 30, 0, 4044, 52, 163, 3693, 0, 0, 0, 26, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1, 229, 0, 340, 3450, 3357, 47, 0, 0, 0, 4006, 0, 472, 0, 9, 3, 7, 0, 0, 1, 1648, 0, 2912, 0, 0, 0, 3303, 0, 0, 0, 249, 0, 0, 1482, 0, 0, 0, 0, 0, 2681, 1, 4090, 33, 0, 0, 0, 1311, 0, 0, 3, 190, 0, 0, 0, 0, 270, 0, 2396, 0, 165, 0, 0, 112, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[272, 159, 14, 4089, 3720, 2, 0, 0, 0, 0, 2846, 0, 0, 2, 456, 5, 59, 3987, 0, 47, 3816, 0, 613, 0, 0, 0, 3740, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 4085, 4092, 1, 0, 757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 1242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3706, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 2, 0, 1271, 269, 0, 0, 3829, 0, 0, 8, 322, 0, 0, 0, 3621, 4094, 1, 0, 2, 1670, 4042, 16, 1, 0, 918, 0, 0, 0, 0, 8, 1, 51, 62, 0, 2, 112, 1, 0, 3426, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 1, 3772, 144, 104, 0, 0, 2500, 0, 1, 0, 54, 1, 0, 0, 42, 283, 0, 0, 4068, 0, 0, 132, 0, 28, 4015, 424, 0, 0, 2999, 1, 0, 0, 390, 1, 0, 0, 0, 0, 0, 0, 0, 10, 3945, 3748, 0, 1, 2095, 0, 0, 3948, 0, 0, 2, 0, 0, 0, 0, 39, 0, 0, 20, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 3742, 71, 318, 0, 47, 0, 3547, 0, 0, 0, 0, 0, 0, 0, 182, 0, 392, 3679, 516, 0, 0, 3674, 1700, 4012, 33, 0, 3, 3530, 125, 0, 0, 0, 0, 0, 0, 0, 3595, 0, 0, 0, 173, 2595, 504, 290, 1, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 10, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1889, 0, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 608, 0, 4055, 0, 360, 569, 0, 0, 0, 0, 0, 0, 381, 53, 0, 0, 954, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1326, 1, 0, 0, 0, 982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 3416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2828, 165, 0, 2607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 4044, 52, 163, 3693, 0, 1407, 247, 14, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 2723, 146, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1899, 5600, 1394, 2309, 3421, 5600, 9650, 2895] avg=4096.0000 max=9650 max_rank=6 min=1394 min_rank=2 imbalance_ratio=2.355957 max_load_reduction=23118 max_load_reduction_pct=0.705505 moved_to_echo=21566 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[246, 0, 0, 3418, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 272, 0, 2343, 1, 1647, 0, 652, 278, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 2436, 0, 2918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 440, 1631, 1319, 15, 0, 5, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 0, 1670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 13, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3345, 0, 0, 0, 0, 6, 632, 26, 0, 0, 0, 0, 0, 0, 0, 2010, 0, 0, 2, 0, 0, 2708, 373, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3674, 5253, 9422, 3430, 1785, 102, 4009, 5093] avg=4096.0000 max=9422 max_rank=2 min=102 min_rank=5 imbalance_ratio=2.300293 max_load_reduction=23346 max_load_reduction_pct=0.712463 moved_to_echo=10989 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3909, 352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 359, 0, 137, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3773, 0, 742, 29, 21, 0, 1, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 36, 0, 1134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1301, 2311, 0, 1, 0, 3974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 1, 0, 0, 0, 0, 4049, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 3902, 0, 1522, 20, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4261, 137, 538, 4816, 5256, 7587, 4628, 5545] avg=4096.0000 max=7587 max_rank=5 min=137 min_rank=1 imbalance_ratio=1.852295 max_load_reduction=25181 max_load_reduction_pct=0.768463 moved_to_echo=23016 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[272, 159, 14, 3412, 3720, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2846, 0, 0, 2, 456, 5, 651, 587, 0, 0, 0, 0, 0, 0, 59, 3987, 0, 47, 2702, 0, 613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 4085, 3505, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 701, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2515, 413, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7579, 4547, 7408, 1225, 7592, 758, 730, 2929] avg=4096.0000 max=7592 max_rank=4 min=730 min_rank=6 imbalance_ratio=1.853516 max_load_reduction=25176 max_load_reduction_pct=0.768311 moved_to_echo=12009 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 229, 0, 284, 3450, 3357, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 0, 472, 0, 9, 3, 54, 383, 0, 0, 0, 0, 0, 0, 7, 0, 0, 1, 1168, 0, 2912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1082, 0, 0, 0, 249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1482, 0, 0, 0, 0, 0, 2298, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4090, 33, 0, 0, 0, 1311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 190, 0, 0, 0, 0, 270, 0, 302, 2, 0, 0, 0, 0, 0, 0, 2396, 0, 165, 0, 0, 112, 0, 2221, 178, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7368, 4927, 4088, 1331, 3780, 5435, 767, 5072] avg=4096.0000 max=7368 max_rank=0 min=767 min_rank=6 imbalance_ratio=1.798828 max_load_reduction=25400 max_load_reduction_pct=0.775146 moved_to_echo=15054 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3706, 0, 197, 584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 417, 269, 0, 0, 3829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 322, 0, 0, 0, 3621, 3510, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 1670, 2625, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 918, 0, 0, 0, 0, 8, 1, 51, 1417, 751, 8, 0, 0, 0, 0, 0, 62, 0, 2, 112, 1, 0, 3426, 0, 854, 443, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1037, 4489, 2895, 4517, 7461, 4315, 3154, 4900] avg=4096.0000 max=7461 max_rank=4 min=1037 min_rank=0 imbalance_ratio=1.821533 max_load_reduction=25307 max_load_reduction_pct=0.772308 moved_to_echo=19830 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 3143, 144, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2500, 0, 1, 0, 54, 1, 0, 0, 603, 0, 0, 0, 0, 0, 0, 0, 42, 283, 0, 0, 2871, 0, 0, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 1309, 424, 0, 0, 2999, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 390, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 2559, 3748, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2095, 0, 0, 3948, 0, 0, 2, 0, 1386, 752, 26, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 20, 0, 2706, 445, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3392, 3159, 3328, 4761, 391, 6318, 8209, 3210] avg=4096.0000 max=8209 max_rank=6 min=391 min_rank=4 imbalance_ratio=2.004150 max_load_reduction=24559 max_load_reduction_pct=0.749481 moved_to_echo=18128 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3122, 71, 318, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3547, 0, 0, 0, 0, 0, 0, 596, 513, 0, 0, 0, 0, 0, 0, 0, 182, 0, 392, 2605, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3674, 1700, 1314, 33, 0, 3, 3530, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173, 1685, 504, 290, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 910, 676, 24, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 10, 0, 0, 2698, 398, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3558, 4656, 3695, 10379, 3082, 2653, 1621, 3124] avg=4096.0000 max=10379 max_rank=3 min=1621 min_rank=6 imbalance_ratio=2.533936 max_load_reduction=22389 max_load_reduction_pct=0.683258 moved_to_echo=10480 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 8320, 3328, 1920, 9600, 4352, 5760, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[512, 8576, 384, 512, 15872, 640, 6528, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3584, 128, 256, 11392, 128, 256, 7808, 384, 5248, 4096, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 1536, 768, 4224, 128, 0, 8960, 17536, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 2432, 2176, 384, 128, 128, 7552, 0, 18048, 2432, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[640, 512, 128, 16384, 7424, 7808, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[2560, 3712, 7168, 256, 3584, 128, 9984, 128, 3200, 2944, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[7936, 3456, 8832, 896, 128, 128, 10752, 1408, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[205, 0, 34, 12, 0, 0, 0, 0, 0, 0, 3582, 0, 0, 147, 3908, 0, 0, 4072, 0, 13, 0, 0, 0, 0, 707, 0, 1, 5, 521, 742, 2548, 3462, 73, 0, 0, 0, 0, 0, 0, 3859, 0, 0, 0, 0, 6, 0, 5, 1, 0, 588, 3977, 63, 4093, 0, 0, 112, 0, 32, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 1, 2, 4072, 0, 38, 0, 20, 0, 0, 51, 0, 0, 208, 0, 843, 0, 0, 0, 0, 94, 6, 0, 2315, 2, 0, 1527, 0, 0, 0, 3919, 0, 0, 0, 0, 3627, 0, 0, 4038, 0, 1, 0, 2315, 3857, 96, 0, 0, 3, 0, 3076, 13, 0, 0, 1122, 0, 0, 34, 1150, 0, 0, 0, 71, 267] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 279, 4068, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3657, 1235, 0, 0, 0, 15, 2, 1217, 0, 438, 2, 109, 0, 0, 0, 3156, 0, 0, 2139, 4096, 0, 9, 3757, 150, 0, 4095, 0, 0, 48, 0, 0, 0, 80, 0, 0, 116, 3, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[789, 415, 4094, 0, 1, 184, 0, 0, 0, 0, 119, 0, 0, 0, 0, 0, 1, 0, 19, 27, 4094, 11, 0, 0, 3176, 0, 0, 0, 0, 303, 0, 871, 0, 0, 0, 0, 12, 0, 0, 3866, 0, 0, 3507, 0, 0, 0, 0, 0, 3781, 0, 4066, 0, 0, 0, 0, 48, 0, 168, 3176, 2, 0, 0, 0, 38] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[241, 0, 27, 0, 44, 20, 998, 0, 0, 0, 0, 0, 3929, 0, 1, 0, 0, 0, 3, 0, 0, 1, 0, 0, 3768, 0, 5, 62, 0, 0, 0, 32, 1, 0, 0, 0, 0, 1, 0, 4055, 0, 0, 4096, 4092, 0, 1638, 746, 0, 3843, 262, 3998, 0, 0, 377, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 1894, 2, 0, 6, 12, 0, 0, 1707, 0, 0, 6, 14, 31, 0, 0, 408, 0, 4077, 0, 5, 0, 4047, 3253, 0, 8, 0, 0, 0, 2688, 0, 0, 0, 2, 0, 0, 0, 1275, 0, 589, 1, 2818, 2747, 0, 4008, 0, 0, 0, 552, 0, 0, 0, 0, 2192, 0, 0, 0, 426, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 3250, 0, 0, 80, 30, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4073, 263, 4092, 125, 0, 0, 0, 1200, 0, 0, 0, 0, 8, 0, 2845, 0, 0, 0, 6, 1, 3856, 0, 1082, 12, 0, 2913, 5, 964, 0, 0, 22, 0, 0, 3846, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[1, 0, 1035, 2, 3, 0, 0, 0, 2, 0, 0, 1308, 0, 0, 3188, 0, 0, 0, 0, 507, 4093, 0, 259, 15, 0, 0, 0, 118, 0, 0, 93, 3972, 0, 368, 0, 6, 0, 9, 0, 289, 0, 0, 0, 0, 3514, 0, 4092, 0, 0, 17, 2392, 3569, 0, 0, 0, 3911, 3, 1, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 279, 4068, 0, 3358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1784, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2523, 623, 0, 0, 0, 0, 0, 0, 0, 1235, 0, 0, 0, 15, 2, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 438, 2, 109, 0, 0, 0, 3156, 179, 340, 0, 0, 0, 0, 0, 0, 0, 0, 2139, 0, 0, 9, 1633, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 116, 3, 0, 1, 0, 0, 1134, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7705, 5995, 3146, 2290, 4224, 3931, 4143, 1334] avg=4096.0000 max=7705 max_rank=0 min=1334 min_rank=7 imbalance_ratio=1.881104 max_load_reduction=25063 max_load_reduction_pct=0.764862 moved_to_echo=13632 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[205, 0, 34, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3582, 0, 0, 147, 3908, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 13, 0, 0, 0, 0, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 521, 742, 2548, 2951, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 3859, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 588, 588, 63, 4093, 0, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 3389, 220, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[251, 7639, 4572, 6768, 4443, 10, 5444, 3641] avg=4096.0000 max=7639 max_rank=1 min=10 min_rank=5 imbalance_ratio=1.864990 max_load_reduction=25129 max_load_reduction_pct=0.766876 moved_to_echo=13538 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 2, 4072, 0, 38, 0, 1898, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 51, 0, 0, 208, 0, 0, 64, 0, 0, 0, 0, 0, 0, 843, 0, 0, 0, 0, 94, 6, 0, 1597, 353, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1527, 0, 0, 0, 3341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3627, 0, 0, 4038, 578, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3857, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 455, 13, 0, 0, 1122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 1150, 0, 0, 0, 71, 267, 2621, 718, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6011, 343, 2893, 4870, 8243, 3954, 1593, 4861] avg=4096.0000 max=8243 max_rank=4 min=343 min_rank=1 imbalance_ratio=2.012451 max_load_reduction=24525 max_load_reduction_pct=0.748444 moved_to_echo=18651 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[789, 415, 4094, 0, 1, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 19, 27, 4094, 11, 0, 0, 2192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 303, 0, 743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 3866, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3781, 0, 601, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 3176, 2, 0, 0, 0, 38, 3465, 984, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5483, 119, 6344, 1046, 4006, 3507, 4430, 7833] avg=4096.0000 max=7833 max_rank=7 min=119 min_rank=1 imbalance_ratio=1.912354 max_load_reduction=24935 max_load_reduction_pct=0.760956 moved_to_echo=19776 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[241, 0, 27, 0, 44, 20, 998, 0, 3355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3929, 0, 1, 0, 354, 113, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 2599, 624, 0, 0, 0, 0, 0, 0, 0, 0, 5, 62, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 4055, 4, 67, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1638, 325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3843, 262, 591, 0, 0, 377, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3407, 1169, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4685, 4397, 3227, 95, 4128, 6059, 5601, 4576] avg=4096.0000 max=6059 max_rank=5 min=95 min_rank=3 imbalance_ratio=1.479248 max_load_reduction=26709 max_load_reduction_pct=0.815094 moved_to_echo=20364 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1035, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1308, 0, 0, 3188, 0, 1943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 4093, 0, 259, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 0, 0, 93, 3386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 368, 0, 6, 0, 9, 0, 289, 586, 370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3514, 0, 1779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 354, 3569, 0, 0, 0, 3911, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 1, 0, 0, 0, 0, 0, 2038, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1041, 6441, 4874, 3597, 1628, 5293, 7851, 2043] avg=4096.0000 max=7851 max_rank=6 min=1041 min_rank=0 imbalance_ratio=1.916748 max_load_reduction=24917 max_load_reduction_pct=0.760406 moved_to_echo=16815 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3250, 0, 0, 80, 30, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 263, 4092, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1200, 0, 0, 0, 0, 8, 0, 2845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3856, 0, 1082, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 429, 5, 964, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3846, 0, 0, 0, 0, 0, 2484, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3366, 4094, 1, 8553, 4053, 4939, 1432, 6330] avg=4096.0000 max=8553 max_rank=3 min=1 min_rank=2 imbalance_ratio=2.088135 max_load_reduction=24215 max_load_reduction_pct=0.738983 moved_to_echo=16754 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1894, 2, 0, 6, 12, 2312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1707, 0, 0, 6, 14, 31, 1904, 78, 0, 0, 0, 0, 0, 0, 0, 0, 408, 0, 4077, 0, 5, 0, 2793, 428, 0, 0, 0, 0, 0, 0, 0, 3253, 0, 8, 0, 0, 0, 2288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1275, 400, 366, 0, 0, 0, 0, 0, 0, 0, 589, 1, 0, 2747, 0, 1738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 2192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 426, 0, 0, 0, 0, 470, 1254, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4226, 3740, 7711, 5549, 2043, 5075, 2274, 2150] avg=4096.0000 max=7711 max_rank=2 min=2043 min_rank=4 imbalance_ratio=1.882568 max_load_reduction=25057 max_load_reduction_pct=0.764679 moved_to_echo=11542 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 8576, 384, 5888, 768, 1152, 2688, 13824, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1280, 512, 8448, 1920, 4224, 640, 5248, 128, 11008, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 5504, 1408, 12160, 256, 7424, 128, 6016, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[7680, 4992, 3200, 3712, 5120, 384, 1664, 6400, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[896, 4096, 512, 640, 12288, 128, 384, 128, 12288, 2048, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 640, 9856, 0, 10240, 5632, 5504, 1280, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 256, 8192, 640, 128, 0, 128, 384, 17920, 5504, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1280, 896, 128, 128, 3712, 128, 0, 23424, 2432, 1152, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[10, 0, 0, 25, 202, 0, 35, 0, 0, 0, 265, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4014, 3405, 15, 32, 11, 1013, 0, 0, 36, 21, 0, 895, 0, 0, 0, 4022, 645, 0, 410, 0, 0, 41, 2540, 919, 0, 3941, 1, 0, 1324, 0, 0, 0, 73, 0, 4076, 0, 0, 0, 869, 0, 0, 3927, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[3481, 0, 0, 181, 0, 0, 3, 0, 386, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3068, 2548, 0, 0, 0, 3794, 1, 117, 0, 0, 24, 0, 0, 0, 23, 717, 0, 3750, 460, 0, 44, 4, 0, 3002, 953, 1742, 0, 0, 0, 3286, 735, 33, 31, 256] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[339, 0, 0, 0, 0, 3862, 1, 0, 1569, 0, 0, 20, 3822, 0, 0, 12, 102, 9, 0, 2674, 0, 3967, 113, 0, 0, 0, 0, 0, 80, 0, 0, 1427, 0, 238, 0, 3803, 0, 0, 75, 0, 0, 0, 0, 0, 4, 784, 0, 0, 0, 1784, 0, 18, 0, 0, 0, 1, 2637, 2, 0, 4030, 13, 0, 1382, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[3511, 0, 0, 226, 0, 0, 0, 35, 88, 0, 0, 0, 0, 0, 105, 0, 60, 0, 0, 1234, 0, 3984, 27, 0, 0, 0, 0, 0, 787, 0, 2, 501, 0, 0, 0, 2870, 0, 0, 13, 0, 0, 7, 313, 0, 0, 3653, 0, 4096, 1, 0, 0, 0, 9, 3662, 0, 0, 4088, 31, 0, 3, 3462, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[4088, 2, 0, 2498, 0, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 2, 3741, 3876, 4059, 75, 0, 4091, 0, 0, 1705, 0, 0, 0, 0, 12, 0, 0, 0, 0, 2, 0, 0, 28, 0, 0, 192, 1, 12, 4033, 1, 0, 0, 0, 0, 0, 0, 199, 0, 0, 0, 5, 0, 3990, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[1253, 0, 1, 2677, 4, 3495, 9, 97, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3836, 2706, 0, 0, 0, 0, 0, 0, 0, 191, 186, 24, 0, 0, 0, 3059, 0, 0, 0, 14, 0, 0, 4, 10, 4071, 2, 4089, 4007, 0, 83, 0, 0, 0, 0, 0, 0, 0, 1237, 1468, 10, 0, 103, 69] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 98, 192, 0, 3875, 0, 0, 17, 0, 0, 44, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4054, 0, 0, 0, 21, 20, 0, 0, 0, 0, 3865, 0, 0, 5, 672, 0, 1, 0, 0, 0, 0, 1, 63, 0, 0, 0, 4, 286, 3958, 40, 0, 0, 17, 1479, 0, 0, 0, 1972, 4092, 0, 3948, 4042, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[7, 0, 0, 0, 0, 0, 0, 0, 4032, 4091, 0, 7, 0, 1, 0, 0, 0, 0, 0, 2145, 2179, 4092, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 128, 0, 368, 3575, 1, 935, 0, 0, 0, 0, 1, 747, 0, 1361, 5, 0, 0, 0, 0, 0, 4095, 1476, 0, 0, 0, 2, 0, 0, 3430, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 0, 25, 202, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 265, 1, 0, 0, 0, 0, 1963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4014, 1043, 15, 32, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1013, 0, 0, 36, 21, 0, 895, 2186, 399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 645, 0, 410, 0, 1741, 0, 0, 0, 0, 0, 0, 0, 0, 41, 2540, 919, 0, 3941, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1324, 0, 0, 0, 73, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[272, 2229, 5104, 4561, 6818, 7442, 5473, 869] avg=4096.0000 max=7442 max_rank=5 min=272 min_rank=0 imbalance_ratio=1.816895 max_load_reduction=25326 max_load_reduction_pct=0.772888 moved_to_echo=20602 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[339, 0, 0, 0, 0, 3862, 1, 0, 0, 330, 0, 0, 0, 0, 0, 0, 1569, 0, 0, 20, 3822, 0, 0, 12, 2287, 0, 0, 0, 0, 0, 0, 0, 102, 9, 0, 2674, 0, 1215, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 1427, 0, 465, 0, 0, 0, 0, 0, 0, 0, 238, 0, 3803, 0, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1784, 0, 18, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2637, 2, 0, 3700, 13, 0, 1382, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4532, 7710, 4113, 1972, 4116, 788, 1803, 7734] avg=4096.0000 max=7734 max_rank=7 min=788 min_rank=5 imbalance_ratio=1.888184 max_load_reduction=25034 max_load_reduction_pct=0.763977 moved_to_echo=14441 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3511, 0, 0, 226, 0, 0, 0, 35, 378, 0, 8, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 105, 0, 2297, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 1234, 0, 1220, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 787, 0, 2, 501, 0, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2870, 0, 0, 13, 0, 0, 308, 0, 0, 0, 0, 0, 0, 0, 7, 313, 0, 0, 3653, 0, 3410, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 9, 3654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 31, 0, 3, 3462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4158, 2490, 2541, 1757, 3191, 7383, 3664, 7584] avg=4096.0000 max=7584 max_rank=7 min=1757 min_rank=3 imbalance_ratio=1.851562 max_load_reduction=25184 max_load_reduction_pct=0.768555 moved_to_echo=21822 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3481, 0, 0, 181, 0, 0, 3, 0, 346, 269, 7, 0, 0, 0, 0, 0, 386, 0, 0, 0, 0, 0, 33, 0, 2362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3068, 2548, 18, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3794, 1, 117, 0, 0, 15, 282, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 23, 717, 0, 3122, 0, 0, 0, 0, 0, 0, 0, 0, 460, 0, 44, 4, 0, 2995, 953, 1742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3017, 735, 0, 31, 256, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4287, 2781, 1254, 6114, 4209, 3886, 6198, 4039] avg=4096.0000 max=6198 max_rank=6 min=1254 min_rank=2 imbalance_ratio=1.513184 max_load_reduction=26570 max_load_reduction_pct=0.810852 moved_to_echo=18332 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4088, 2, 0, 2498, 0, 0, 0, 0, 373, 3, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 51, 2235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3741, 1187, 4059, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 1705, 0, 0, 0, 2222, 454, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 2, 0, 1768, 305, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 192, 1, 12, 3355, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6964, 2391, 9064, 8472, 2087, 3588, 200, 2] avg=4096.0000 max=9064 max_rank=2 min=2 min_rank=7 imbalance_ratio=2.212891 max_load_reduction=23704 max_load_reduction_pct=0.723389 moved_to_echo=5877 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1253, 0, 1, 2677, 4, 3495, 9, 97, 378, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 2212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1174, 2706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 191, 186, 0, 450, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 3059, 0, 0, 0, 0, 308, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4, 10, 4071, 2, 3403, 0, 0, 0, 0, 0, 0, 0, 0, 4007, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1237, 1348, 10, 0, 103, 69, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8034, 2275, 3880, 827, 3391, 7504, 4090, 2767] avg=4096.0000 max=8034 max_rank=0 min=827 min_rank=3 imbalance_ratio=1.961426 max_load_reduction=24734 max_load_reduction_pct=0.754822 moved_to_echo=17752 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 98, 192, 0, 3875, 0, 0, 0, 335, 0, 0, 0, 0, 0, 0, 17, 0, 0, 44, 0, 0, 0, 0, 2337, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 20, 0, 0, 0, 0, 3865, 2199, 475, 0, 0, 0, 0, 0, 0, 0, 0, 5, 672, 0, 1, 0, 0, 1749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 63, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 286, 3958, 40, 0, 0, 17, 1479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1972, 3757, 0, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4500, 2398, 1244, 6580, 2427, 68, 5780, 9771] avg=4096.0000 max=9771 max_rank=7 min=68 min_rank=5 imbalance_ratio=2.385498 max_load_reduction=22997 max_load_reduction_pct=0.701813 moved_to_echo=18046 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 0, 0, 0, 0, 0, 0, 3, 0, 11, 0, 0, 0, 0, 0, 4032, 4091, 0, 7, 0, 1, 0, 0, 2363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2145, 2179, 1244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 1910, 485, 0, 0, 0, 0, 0, 0, 0, 128, 0, 368, 3575, 1, 935, 0, 1520, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 747, 0, 1361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 1476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21, 10494, 5568, 2485, 6529, 2109, 5560, 2] avg=4096.0000 max=10494 max_rank=1 min=2 min_rank=7 imbalance_ratio=2.562012 max_load_reduction=22274 max_load_reduction_pct=0.679749 moved_to_echo=14200 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[6784, 128, 4096, 11904, 4224, 0, 5632, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 128, 0, 6144, 9984, 9600, 7040, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[6272, 4096, 384, 256, 3840, 128, 256, 128, 18176, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 5248, 128, 0, 2816, 128, 3328, 9472, 8576, 3712, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 2944, 1024, 1024, 13184, 1408, 13312, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[6144, 5760, 256, 128, 128, 10752, 8064, 2048, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[12800, 128, 128, 5888, 256, 11264, 128, 256, 1536, 1152, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 384, 128, 15616, 7296, 128, 1536, 0, 6912, 1280, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[1, 0, 4011, 3791, 36, 131, 3, 106, 0, 0, 0, 0, 3480, 52, 0, 0, 0, 0, 0, 0, 0, 3923, 0, 25, 1, 12, 15, 0, 3706, 803, 0, 0, 0, 5, 0, 596, 0, 2995, 0, 309, 0, 0, 0, 0, 11, 0, 148, 3952, 0, 0, 56, 279, 0, 3, 2, 4075, 224, 0, 0, 0, 0, 0, 0, 17] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[2039, 43, 0, 4072, 0, 0, 0, 11, 0, 457, 0, 0, 0, 0, 0, 4022, 0, 0, 0, 2993, 0, 3908, 0, 3731, 4056, 142, 215, 0, 3133, 1, 0, 0, 0, 0, 1184, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 166, 0, 0, 0, 834, 121, 0, 1, 0, 933, 690, 0, 0, 14, 0, 1, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[4095, 123, 3406, 2, 1665, 2, 0, 0, 67, 0, 14, 2, 0, 0, 0, 0, 0, 0, 31, 43, 1141, 0, 0, 0, 1265, 0, 0, 6, 0, 0, 3532, 0, 1782, 0, 0, 820, 0, 3852, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 135, 0, 0, 0, 0, 3, 0, 4074, 3765, 0, 0, 0, 2835, 0, 0, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 3892, 558, 4096, 114, 0, 0, 0, 0, 0, 641, 0, 0, 0, 189, 0, 0, 30, 14, 19, 183, 3, 0, 0, 147, 0, 4084, 0, 4023, 3706, 0, 0, 0, 0, 0, 3798, 0, 3030, 0, 649, 0, 0, 3, 184, 0, 6, 56, 0, 0, 0, 0, 2, 0, 49, 22, 0, 0, 0, 0, 0, 0, 3270] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4049, 2249, 3897, 4094, 0, 0, 0, 0, 0, 1, 0, 2214, 0, 1654, 0, 0, 0, 2, 4, 1436, 0, 0, 384, 0, 0, 0, 4051, 0, 0, 0, 86, 0, 0, 3, 0, 0, 0, 4095, 0, 308, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 73, 0, 71, 0, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 1, 3902, 2373, 0, 0, 0, 0, 0, 0, 0, 1537, 0, 0, 0, 14, 0, 1, 3820, 0, 3554, 351, 0, 2, 0, 301, 5, 4089, 0, 4, 0, 641, 2441, 0, 0, 0, 3317, 1, 4083, 1, 0, 0, 0, 0, 0, 76, 0, 0, 347, 0, 0, 0, 0, 0, 0, 92, 0, 10, 0, 0, 0, 1805, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[4, 0, 0, 0, 0, 3792, 0, 0, 0, 16, 0, 0, 1, 0, 0, 3466, 0, 0, 0, 0, 0, 1095, 0, 1, 1034, 0, 2, 0, 512, 3519, 369, 28, 1, 0, 0, 0, 0, 0, 0, 897, 0, 2, 75, 52, 793, 0, 2, 0, 2, 0, 0, 183, 0, 390, 41, 4089, 3396, 0, 76, 2012, 4005, 0, 2913, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[3956, 0, 0, 11, 27, 2740, 0, 0, 58, 0, 145, 4071, 44, 0, 0, 0, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3861, 3648, 54, 0, 109, 1, 0, 0, 0, 0, 43, 0, 0, 3568, 2063, 0, 0, 4007, 0, 0, 0, 0, 0, 2, 0, 0, 1632, 187, 0, 665, 0, 105, 0, 11, 686, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 4011, 0, 36, 21, 3, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3480, 52, 0, 0, 77, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3923, 0, 25, 33, 26, 0, 0, 0, 0, 0, 0, 1, 12, 15, 0, 298, 803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 596, 0, 2995, 0, 218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 148, 3952, 3408, 447, 0, 0, 0, 0, 0, 0, 0, 0, 56, 279, 0, 3, 2, 4075, 2915, 0, 0, 0, 0, 0, 0, 0, 224, 0, 0, 0, 0, 0, 0, 17, 65, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4178, 4038, 4007, 1129, 3814, 7966, 7330, 306] avg=4096.0000 max=7966 max_rank=5 min=306 min_rank=7 imbalance_ratio=1.944824 max_load_reduction=24802 max_load_reduction_pct=0.756897 moved_to_echo=19416 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4095, 123, 3406, 0, 1665, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 14, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 43, 1141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1265, 0, 0, 6, 0, 0, 3532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1782, 0, 0, 820, 0, 3852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 0, 0, 0, 0, 3, 0, 4074, 2, 0, 0, 0, 0, 0, 0, 0, 3765, 0, 0, 0, 2835, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9290, 84, 1215, 4803, 6454, 100, 4214, 6608] avg=4096.0000 max=9290 max_rank=0 min=84 min_rank=1 imbalance_ratio=2.268066 max_load_reduction=23478 max_load_reduction_pct=0.716492 moved_to_echo=17376 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 558, 621, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 641, 0, 0, 0, 2425, 440, 0, 0, 0, 0, 0, 0, 189, 0, 0, 30, 14, 19, 183, 3, 1050, 260, 0, 0, 0, 0, 0, 0, 0, 0, 147, 0, 329, 0, 4023, 3706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3798, 0, 2125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 649, 0, 0, 3, 184, 0, 6, 3755, 460, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 2, 0, 49, 2992, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 3270, 645, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1293, 3506, 1748, 8205, 5923, 5057, 3099, 3937] avg=4096.0000 max=8205 max_rank=3 min=1293 min_rank=0 imbalance_ratio=2.003174 max_load_reduction=24563 max_load_reduction_pct=0.749603 moved_to_echo=18016 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 0, 0, 0, 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 1, 0, 0, 3466, 2245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1095, 0, 1, 972, 77, 0, 0, 0, 0, 0, 0, 1034, 0, 2, 0, 42, 3519, 369, 28, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 75, 52, 793, 0, 2, 0, 470, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 183, 0, 390, 41, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 3396, 0, 76, 2012, 4005, 0, 2913, 0, 191, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[579, 5728, 2145, 4994, 630, 1394, 4705, 12593] avg=4096.0000 max=12593 max_rank=7 min=579 min_rank=0 imbalance_ratio=3.074463 max_load_reduction=20175 max_load_reduction_pct=0.615692 moved_to_echo=19322 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2039, 43, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 457, 0, 0, 0, 0, 0, 4022, 0, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2993, 0, 3908, 0, 3731, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 142, 215, 0, 252, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 166, 0, 0, 2881, 481, 0, 0, 0, 0, 0, 0, 0, 834, 121, 0, 1, 0, 933, 690, 3129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2093, 4941, 10632, 4666, 1184, 3529, 5708, 15] avg=4096.0000 max=10632 max_rank=2 min=15 min_rank=7 imbalance_ratio=2.595703 max_load_reduction=22136 max_load_reduction_pct=0.675537 moved_to_echo=10436 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3956, 0, 0, 0, 27, 411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 145, 4071, 44, 0, 0, 0, 1624, 2, 0, 0, 0, 0, 0, 0, 1074, 0, 0, 0, 0, 0, 0, 0, 705, 306, 0, 0, 0, 0, 0, 0, 0, 0, 3861, 3648, 5, 0, 109, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 2502, 0, 0, 0, 0, 0, 0, 0, 0, 2063, 0, 0, 4007, 0, 0, 0, 0, 49, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1632, 187, 0, 665, 8, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 11, 686, 0, 0, 0, 760, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4394, 5944, 2085, 7624, 2545, 6120, 2494, 1562] avg=4096.0000 max=7624 max_rank=3 min=1562 min_rank=7 imbalance_ratio=1.861328 max_load_reduction=25144 max_load_reduction_pct=0.767334 moved_to_echo=12721 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4049, 0, 3897, 621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2214, 0, 1654, 2424, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 1436, 0, 0, 1049, 352, 0, 0, 0, 0, 0, 0, 384, 0, 0, 0, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 3, 0, 0, 0, 2871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 308, 0, 0, 0, 0, 0, 1, 3725, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 71, 1728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 872, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8567, 6547, 2843, 710, 2960, 4301, 1872, 4968] avg=4096.0000 max=8567 max_rank=0 min=710 min_rank=3 imbalance_ratio=2.091553 max_load_reduction=24201 max_load_reduction_pct=0.738556 moved_to_echo=14101 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 2373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1537, 0, 0, 0, 0, 443, 0, 0, 0, 0, 0, 0, 14, 0, 1, 3820, 0, 3554, 351, 0, 0, 353, 0, 0, 0, 0, 0, 0, 2, 0, 301, 5, 325, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 641, 2441, 0, 0, 0, 3317, 1, 2858, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 76, 0, 3764, 460, 0, 0, 0, 0, 0, 0, 0, 347, 0, 0, 0, 0, 0, 0, 2999, 0, 0, 0, 0, 0, 0, 0, 92, 0, 10, 0, 0, 0, 1805, 0, 872, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2374, 1980, 8093, 637, 9258, 4301, 3346, 2779] avg=4096.0000 max=9258 max_rank=4 min=637 min_rank=3 imbalance_ratio=2.260254 max_load_reduction=23510 max_load_reduction_pct=0.717468 moved_to_echo=19684 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1280, 0, 128, 6912, 1280, 13952, 640, 3840, 3840, 1408, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[7552, 128, 128, 2048, 7552, 0, 8832, 3328, 3456, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[10112, 256, 11520, 0, 8576, 2304, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[6784, 256, 4608, 3712, 1664, 4352, 8064, 3840, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 512, 256, 4096, 5760, 2304, 0, 9216, 8832, 2048, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[2560, 2560, 1280, 1536, 128, 14080, 128, 11264, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[2176, 1024, 128, 4096, 896, 512, 256, 3968, 18176, 2176, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[256, 1280, 256, 512, 1664, 768, 1024, 13824, 13824, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[8, 3876, 0, 0, 14, 0, 333, 0, 0, 4095, 0, 0, 0, 0, 6, 0, 0, 2039, 0, 0, 0, 630, 0, 1682, 0, 116, 3823, 0, 0, 0, 0, 0, 0, 0, 34, 2, 4088, 0, 13, 0, 109, 0, 3, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 1001, 3973, 36, 0, 0, 6, 0, 217, 0, 132, 0, 2454] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 1556, 12, 0, 0, 901, 0, 3032, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3406, 0, 0, 2, 0, 0, 0, 0, 0, 0, 186, 0, 0, 0, 0, 0, 281, 0, 0, 0, 817, 0, 0, 2028, 657, 0, 3361, 4096, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 4, 0, 0, 0, 1815, 2458, 0, 11] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 828, 0, 0, 0, 0, 796, 52, 0, 0, 0, 0, 324, 2146, 0, 0, 0, 0, 0, 0, 553, 16, 3708, 3903, 376, 1744, 0, 87, 2, 1681, 138, 0, 230, 3, 0, 0, 214, 0, 4, 0, 0, 0, 0, 0, 0, 3901, 3688, 0, 0, 0, 0, 0, 0, 0, 171, 0, 0, 4096, 11, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 18, 1, 21, 1049, 0, 1, 1609, 4061, 0, 0, 0, 3416, 3565, 0, 0, 0, 0, 0, 0, 0, 76, 3, 4011, 0, 0, 545, 62, 0, 0, 1, 253, 1, 0, 0, 2805, 14, 5, 150, 358, 0, 7, 0, 0, 0, 0, 3572, 0, 0, 681, 0, 0, 2413, 0, 0, 0, 0, 3, 0, 0, 19, 4048, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[220, 4093, 4084, 535, 22, 0, 51, 0, 19, 0, 0, 0, 4049, 2198, 0, 0, 0, 0, 0, 0, 0, 0, 258, 260, 0, 1117, 0, 0, 0, 586, 10, 0, 4, 0, 339, 0, 0, 7, 0, 0, 808, 0, 0, 4081, 4, 3638, 2256, 0, 1, 0, 0, 0, 0, 0, 3166, 0, 0, 0, 659, 0, 0, 0, 0, 303] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[2, 0, 3598, 0, 37, 0, 0, 0, 0, 4044, 0, 7, 4048, 4033, 0, 0, 160, 0, 1, 320, 0, 0, 0, 3969, 0, 93, 4096, 0, 0, 0, 0, 48, 0, 0, 0, 82, 0, 0, 0, 0, 0, 126, 3, 0, 500, 1550, 2628, 0, 0, 1120, 0, 0, 2256, 0, 0, 0, 0, 0, 8, 37, 0, 2, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 4036, 3197, 0, 0, 14, 0, 7, 226, 0, 0, 3111, 4096, 0, 4, 2687, 0, 253, 392, 1107, 0, 21, 0, 52, 4064, 0, 9, 0, 1236, 0, 0, 14, 0, 0, 0, 3794, 2454, 0, 0, 0, 0, 0, 0, 1987, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 3680, 0, 0, 47, 0, 0, 0, 5, 20, 0, 0, 4012, 0, 0, 0, 57, 0, 511, 0, 0, 2, 0, 322, 0, 0, 3759, 0, 10, 157, 2806, 0, 0, 0, 0, 0, 770, 0, 0, 0, 0, 0, 0, 0, 0, 3820, 3170, 0, 561, 0, 0, 0, 0, 4090, 99, 1, 3185, 68, 0, 0, 0, 1616, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[8, 3876, 0, 0, 14, 0, 333, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2039, 0, 0, 0, 630, 0, 1629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 3823, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 2, 4088, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 0, 3, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 3973, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 217, 0, 132, 0, 2454, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4284, 4101, 4298, 3939, 4137, 4190, 5010, 2809] avg=4096.0000 max=5010 max_rank=6 min=2809 min_rank=7 imbalance_ratio=1.223145 max_load_reduction=27758 max_load_reduction_pct=0.847107 moved_to_echo=16146 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 828, 0, 0, 0, 0, 796, 52, 27, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 2146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 16, 3708, 3780, 0, 0, 0, 0, 0, 0, 0, 0, 376, 1744, 0, 87, 2, 1681, 138, 0, 108, 0, 0, 0, 0, 0, 0, 0, 230, 3, 0, 0, 214, 0, 4, 0, 2728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3901, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 171, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 4096, 701, 50, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1826, 2169, 8057, 4136, 3179, 4160, 287, 8954] avg=4096.0000 max=8954 max_rank=7 min=287 min_rank=6 imbalance_ratio=2.186035 max_load_reduction=23814 max_load_reduction_pct=0.726746 moved_to_echo=16580 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1556, 12, 0, 0, 901, 0, 3032, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3406, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 281, 0, 0, 0, 817, 3029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2028, 657, 0, 3361, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1815, 2458, 0, 11, 778, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5501, 4095, 3408, 186, 4127, 6335, 4050, 5066] avg=4096.0000 max=6335 max_rank=5 min=186 min_rank=3 imbalance_ratio=1.546631 max_load_reduction=26433 max_load_reduction_pct=0.806671 moved_to_echo=19578 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 2, 0, 1, 0, 347, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 261, 3197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 7, 226, 0, 0, 3111, 3967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2687, 0, 253, 392, 1107, 0, 1345, 0, 0, 0, 0, 0, 0, 0, 21, 0, 52, 4064, 0, 9, 0, 1236, 2806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 267, 2454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1987, 0, 1453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 721, 630, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[483, 3458, 7325, 5788, 8188, 2735, 3440, 1351] avg=4096.0000 max=8188 max_rank=4 min=483 min_rank=0 imbalance_ratio=1.999023 max_load_reduction=24580 max_load_reduction_pct=0.750122 moved_to_echo=15714 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 3680, 0, 0, 47, 0, 0, 0, 345, 10, 0, 0, 0, 0, 0, 0, 5, 20, 0, 0, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 511, 0, 0, 2, 0, 312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3759, 0, 10, 157, 2806, 0, 1337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 770, 0, 0, 0, 2344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3820, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 0, 0, 0, 0, 4090, 99, 1, 1445, 0, 0, 0, 0, 0, 0, 0, 3185, 68, 0, 0, 0, 1616, 0, 0, 602, 626, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 284, 882, 8069, 3114, 4044, 6196, 6097] avg=4096.0000 max=8069 max_rank=3 min=284 min_rank=1 imbalance_ratio=1.969971 max_load_reduction=24699 max_load_reduction_pct=0.753754 moved_to_echo=19451 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 3598, 0, 37, 0, 0, 0, 348, 125, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 7, 262, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 1, 320, 0, 0, 0, 3844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 4096, 0, 0, 0, 0, 48, 1349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 1944, 0, 0, 0, 0, 0, 0, 0, 0, 126, 3, 0, 500, 1550, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1120, 0, 0, 2256, 0, 0, 0, 1457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 37, 0, 2, 0, 0, 499, 632, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 8346, 4325, 5586, 2026, 2364, 4833, 1178] avg=4096.0000 max=8346 max_rank=1 min=1178 min_rank=7 imbalance_ratio=2.037598 max_load_reduction=24422 max_load_reduction_pct=0.745300 moved_to_echo=10401 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 18, 1, 21, 1049, 0, 1, 1609, 293, 126, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 0, 222, 3565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 3, 3885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 62, 0, 0, 1, 253, 1138, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2805, 14, 5, 150, 358, 2642, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 681, 0, 0, 2413, 0, 0, 0, 1230, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 19, 4048, 0, 0, 679, 533, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3118, 7848, 3964, 1999, 5975, 258, 4324, 5282] avg=4096.0000 max=7848 max_rank=1 min=258 min_rank=5 imbalance_ratio=1.916016 max_load_reduction=24920 max_load_reduction_pct=0.760498 moved_to_echo=15839 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[220, 4093, 4084, 535, 22, 0, 51, 0, 350, 9, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 250, 2198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 258, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1117, 0, 0, 0, 586, 10, 0, 1352, 0, 0, 0, 0, 0, 0, 0, 4, 0, 339, 0, 0, 7, 0, 0, 1672, 0, 0, 0, 0, 0, 0, 0, 808, 0, 0, 4081, 4, 3638, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3166, 0, 1461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 659, 0, 0, 0, 0, 303, 433, 636, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9364, 2467, 509, 3065, 2022, 8682, 4628, 2031] avg=4096.0000 max=9364 max_rank=0 min=509 min_rank=2 imbalance_ratio=2.286133 max_load_reduction=23404 max_load_reduction_pct=0.714233 moved_to_echo=17363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3200, 4224, 768, 256, 1920, 8320, 0, 6912, 4480, 3200, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[640, 1920, 0, 0, 5760, 8064, 9600, 128, 7168, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 0, 128, 1280, 15232, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 2048, 640, 3968, 640, 768, 7168, 17792, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[256, 14080, 7808, 640, 1280, 1024, 1280, 4736, 1792, 640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[256, 128, 512, 7296, 5120, 128, 256, 2432, 17280, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[384, 3200, 14976, 256, 384, 3072, 4096, 384, 6656, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1024, 256, 2048, 4864, 512, 20352, 1664, 2560, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 13, 0, 0, 121, 69, 4096, 0, 0, 0, 0, 4, 0, 0, 0, 42, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4022, 0, 4088, 0, 0, 3971, 0, 3286, 3836, 0, 1, 0, 0, 0, 3823, 0, 0, 1054, 0, 0, 0, 0, 245, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 156, 4, 0, 61, 29, 2755, 0, 342, 0, 0, 184, 0, 0, 4, 0, 0, 0, 49, 1, 0, 502, 0, 3838, 0, 0, 0, 16, 12, 0, 0, 3802, 0, 0, 0, 1913, 4091, 4089, 0, 17, 0, 5, 468, 2, 0, 2227, 5, 2, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 4071, 19, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 17, 0, 0, 3342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3578, 0, 0, 1, 0, 0, 3681, 536, 0, 0, 0, 0, 0, 0, 4031, 0, 0, 191, 0, 13, 4048, 0, 25, 0, 3766, 0, 3065, 0, 0, 184, 924, 0, 0, 537, 896, 0, 0, 0, 3903, 0, 0, 30, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 2947, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3958, 4080, 0, 0, 0, 1, 0, 1521, 0, 0, 0, 0, 63, 2703, 2773, 0, 0, 46, 21, 0, 2319, 0, 30, 0, 0, 0, 4024, 1, 0, 0, 0, 0, 4096, 0, 0, 29, 0, 0, 7, 0, 0, 0, 0, 0, 0, 24, 4096, 16] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 154, 0, 0, 451, 3639, 0, 7, 0, 0, 0, 0, 0, 0, 1, 3748, 0, 269, 1822, 0, 0, 4011, 38, 844, 0, 0, 0, 4, 0, 239, 3371, 4096, 0, 0, 0, 2311, 2301, 0, 0, 0, 0, 13, 991, 0, 0, 0, 4, 4044, 1, 0, 12, 0, 0, 0, 0, 0, 397, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[22, 0, 0, 0, 4021, 0, 0, 0, 0, 0, 4094, 5, 0, 0, 376, 0, 9, 131, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 0, 0, 376, 13, 4090, 0, 118, 3920, 0, 0, 4052, 0, 4096, 17, 0, 394, 0, 0, 0, 3, 82, 0, 0, 0, 0, 0, 0, 0, 0, 54, 2779, 32] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 1219, 0, 0, 0, 2077, 3864, 0, 0, 13, 0, 421, 0, 0, 0, 4020, 0, 0, 0, 754, 0, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 316, 0, 0, 4003, 0, 4096, 0, 0, 0, 0, 57, 0, 0, 0, 5, 252, 0, 0, 39, 0, 0, 0, 3240, 0, 0, 0, 0, 4047, 1174, 0, 3069, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 3827, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2691, 0, 0, 7, 0, 64, 707, 0, 0, 0, 0, 0, 1457, 0, 0, 0, 0, 13, 3311, 116, 19, 85, 0, 45, 0, 0, 410, 0, 0, 0, 3953, 2, 0, 0, 0, 120, 0, 0, 4096, 0, 0, 0, 0, 0, 1692, 1017, 0, 0, 4060, 3400, 1647, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 156, 4, 0, 61, 29, 2728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 342, 0, 0, 184, 0, 0, 4, 0, 266, 2486, 0, 0, 0, 0, 0, 0, 0, 0, 49, 1, 0, 502, 0, 3838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 12, 0, 0, 3802, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 1913, 1605, 4089, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 202, 2, 0, 2227, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 4071, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2978, 3282, 4390, 3850, 7624, 2443, 4089, 4112] avg=4096.0000 max=7624 max_rank=4 min=2443 min_rank=5 imbalance_ratio=1.861328 max_load_reduction=25144 max_load_reduction_pct=0.767334 moved_to_echo=18268 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 17, 0, 0, 3310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1745, 2457, 727, 0, 0, 0, 0, 0, 1521, 0, 0, 1, 0, 0, 3681, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4031, 0, 1330, 0, 24, 0, 0, 0, 0, 0, 0, 191, 0, 13, 1591, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3766, 0, 1320, 0, 0, 184, 924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 896, 0, 0, 0, 3903, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3327, 4929, 5739, 5385, 1820, 6194, 5344, 30] avg=4096.0000 max=6194 max_rank=5 min=30 min_rank=7 imbalance_ratio=1.512207 max_load_reduction=26574 max_load_reduction_pct=0.810974 moved_to_echo=13388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 13, 0, 0, 121, 69, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 42, 2327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 30, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 0, 1761, 0, 0, 3971, 0, 3286, 0, 0, 0, 0, 0, 0, 0, 0, 3836, 0, 1, 0, 0, 0, 3823, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 0, 0, 0, 0, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4259, 2373, 4094, 34, 2, 13040, 7670, 1296] avg=4096.0000 max=13040 max_rank=5 min=2 min_rank=4 imbalance_ratio=3.183594 max_load_reduction=19728 max_load_reduction_pct=0.602051 moved_to_echo=22008 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3827, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2691, 0, 0, 7, 2250, 0, 0, 0, 0, 0, 0, 0, 0, 64, 707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1457, 0, 0, 0, 0, 13, 3311, 116, 0, 24, 0, 0, 0, 0, 0, 0, 19, 85, 0, 45, 0, 0, 410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1703, 2, 0, 0, 0, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1692, 1017, 0, 0, 4060, 3400, 1623, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3856, 4948, 771, 4921, 559, 1825, 4096, 11792] avg=4096.0000 max=11792 max_rank=7 min=559 min_rank=4 imbalance_ratio=2.878906 max_load_reduction=20976 max_load_reduction_pct=0.640137 moved_to_echo=18272 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 154, 0, 0, 451, 3604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 1, 7, 1403, 761, 0, 0, 0, 0, 0, 1594, 0, 269, 1822, 0, 0, 4011, 38, 0, 0, 0, 0, 0, 0, 0, 0, 844, 0, 0, 0, 4, 0, 239, 3371, 1393, 0, 26, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 908, 2301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 991, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 1, 0, 12, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4209, 2179, 7734, 5877, 7305, 1001, 4066, 397] avg=4096.0000 max=7734 max_rank=2 min=397 min_rank=7 imbalance_ratio=1.888184 max_load_reduction=25034 max_load_reduction_pct=0.763977 moved_to_echo=12769 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2947, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2291, 1407, 804, 0, 0, 0, 0, 0, 1683, 4080, 0, 0, 0, 1, 0, 1521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 2703, 2773, 0, 1471, 61, 0, 0, 0, 0, 0, 0, 0, 46, 21, 0, 912, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1733, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 29, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 4035, 16, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2960, 4502, 7285, 7071, 1009, 1734, 4132, 4075] avg=4096.0000 max=7285 max_rank=2 min=1009 min_rank=4 imbalance_ratio=1.778564 max_load_reduction=25483 max_load_reduction_pct=0.777679 moved_to_echo=10950 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[22, 0, 0, 0, 4021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 5, 0, 0, 376, 0, 2307, 2483, 1, 0, 0, 0, 0, 0, 5, 131, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 3, 41, 0, 0, 0, 0, 0, 0, 0, 0, 376, 13, 1607, 0, 118, 3920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1745, 0, 4096, 17, 0, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 2738, 32, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4043, 9266, 195, 4069, 6034, 6252, 85, 2824] avg=4096.0000 max=9266 max_rank=1 min=85 min_rank=6 imbalance_ratio=2.262207 max_load_reduction=23502 max_load_reduction_pct=0.717224 moved_to_echo=15195 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1219, 0, 0, 0, 2053, 3864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 421, 0, 0, 0, 35, 0, 820, 0, 0, 0, 0, 0, 1704, 0, 0, 0, 754, 0, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1496, 48, 17, 0, 0, 0, 0, 0, 316, 0, 0, 4003, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 5, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 3240, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 1174, 0, 3021, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7136, 1289, 2560, 1561, 8415, 279, 3286, 8242] avg=4096.0000 max=8415 max_rank=4 min=279 min_rank=5 imbalance_ratio=2.054443 max_load_reduction=24353 max_load_reduction_pct=0.743195 moved_to_echo=20222 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4480, 384, 512, 6016, 6656, 10496, 640, 3968, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[7808, 128, 8576, 1024, 4096, 6400, 1024, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[384, 0, 4224, 256, 3200, 0, 384, 128, 11264, 10240, 3200, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1792, 2560, 0, 4096, 5248, 7552, 11776, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[6528, 4352, 1152, 1920, 4992, 512, 7808, 6016, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[16128, 640, 5120, 128, 128, 0, 11008, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[2304, 0, 0, 128, 128, 2816, 10368, 11392, 5760, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 7040, 1536, 128, 4224, 640, 15872, 3968, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.rerun_state_machine:Result validation enabled +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +Number of parameters in transformer block in billions: 6.71 +Number of parameters in embedding layers in billions: 0.21 +Total number of parameters in billions: 6.92 +Number of parameters in most loaded shard in billions: 6.9171 +Theoretical memory footprints: weight and optimizer=49474.72 MB + [2026-09-21 08:24:23] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 39572.1 | throughput per GPU (TFLOP/s/GPU): 0.8 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.118499E+01 | loss scale: 1.0 | grad norm: 35.478 | number of skipped iterations: 0 | number of nan iterations: 0 | +[Rank 0] (after 1 iterations) memory (MB) | allocated: 19764.76171875 | max allocated: 26783.26318359375 | reserved: 27996.0 | max reserved: 27996.0 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[2035, 671, 119, 272, 309, 363, 56, 17, 354, 289, 40, 1, 62, 327, 195, 1370, 97, 446, 1471, 1809, 122, 759, 1142, 270, 859, 95, 128, 1916, 533, 273, 37, 216, 480, 931, 644, 115, 1185, 91, 440, 451, 479, 301, 462, 8, 1622, 686, 60, 540, 972, 716, 637, 49, 2053, 377, 155, 214, 563, 29, 674, 219, 55, 522, 260, 125] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[2340, 723, 75, 281, 298, 379, 64, 57, 650, 368, 84, 4, 172, 288, 285, 1633, 153, 572, 1275, 1179, 204, 560, 1126, 146, 989, 235, 49, 1897, 826, 368, 48, 210, 331, 895, 812, 47, 807, 270, 672, 462, 228, 535, 285, 33, 1544, 498, 152, 340, 306, 513, 523, 68, 2310, 321, 124, 216, 703, 25, 543, 215, 16, 541, 817, 78] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1958, 780, 94, 285, 284, 525, 151, 27, 441, 117, 62, 5, 85, 408, 370, 1483, 183, 497, 1341, 1794, 182, 662, 1317, 173, 811, 269, 153, 1620, 602, 342, 30, 291, 358, 595, 812, 130, 1289, 128, 264, 389, 819, 333, 651, 11, 1447, 515, 82, 391, 353, 671, 643, 117, 1914, 158, 211, 317, 657, 37, 697, 387, 145, 524, 260, 121] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[2452, 688, 150, 280, 249, 300, 133, 31, 280, 275, 119, 12, 80, 556, 161, 1473, 138, 269, 1491, 1503, 277, 438, 1435, 240, 943, 275, 181, 1531, 671, 486, 39, 305, 271, 653, 679, 187, 1032, 287, 549, 645, 364, 472, 380, 18, 1603, 499, 69, 457, 505, 733, 657, 126, 1798, 141, 115, 389, 1007, 32, 414, 283, 16, 347, 480, 99] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[2329, 913, 156, 359, 307, 408, 81, 40, 415, 332, 197, 6, 45, 401, 229, 1476, 120, 514, 1171, 1677, 343, 693, 1417, 206, 1188, 140, 149, 1034, 447, 386, 67, 255, 217, 718, 556, 93, 1312, 222, 318, 410, 406, 244, 214, 3, 1657, 519, 23, 719, 476, 408, 407, 76, 1970, 353, 152, 667, 743, 28, 815, 584, 61, 445, 276, 175] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1562, 1025, 61, 466, 142, 327, 171, 9, 435, 231, 94, 0, 32, 435, 137, 1860, 72, 418, 1165, 1795, 95, 995, 1023, 653, 866, 212, 140, 1319, 703, 407, 92, 183, 240, 886, 824, 86, 610, 107, 616, 343, 428, 475, 203, 6, 1944, 697, 144, 728, 621, 393, 882, 19, 1973, 207, 136, 975, 629, 47, 611, 251, 18, 281, 147, 116] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1575, 847, 207, 428, 221, 560, 216, 198, 368, 352, 90, 0, 72, 188, 362, 1658, 186, 731, 1186, 1944, 58, 557, 870, 42, 487, 235, 208, 1051, 575, 644, 79, 193, 273, 643, 628, 27, 1038, 135, 156, 305, 1031, 573, 690, 52, 1419, 261, 162, 390, 430, 562, 693, 135, 1853, 203, 23, 355, 1092, 25, 1612, 510, 93, 344, 576, 91] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1700, 690, 122, 298, 213, 635, 173, 60, 298, 365, 68, 2, 136, 302, 216, 1440, 67, 693, 910, 2074, 104, 831, 994, 289, 1063, 195, 151, 1623, 493, 633, 79, 188, 306, 825, 724, 100, 1548, 167, 435, 511, 396, 338, 225, 26, 1426, 825, 73, 653, 801, 486, 441, 127, 1880, 279, 111, 448, 592, 22, 914, 215, 65, 230, 313, 161] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2013, 671, 119, 272, 309, 363, 56, 17, 0, 0, 0, 0, 0, 0, 0, 0, 354, 289, 40, 1, 62, 327, 195, 1370, 544, 522, 47, 0, 0, 0, 0, 0, 97, 446, 1380, 0, 122, 759, 1142, 270, 0, 0, 0, 0, 0, 0, 0, 0, 859, 95, 128, 1916, 533, 273, 37, 216, 7, 0, 0, 0, 0, 0, 0, 0, 480, 931, 644, 115, 1185, 91, 440, 451, 44, 22, 1, 0, 0, 0, 0, 0, 479, 301, 462, 8, 1614, 686, 60, 540, 0, 0, 0, 0, 0, 0, 0, 0, 972, 716, 637, 49, 1509, 377, 155, 214, 0, 0, 0, 0, 0, 0, 0, 0, 563, 29, 674, 219, 55, 522, 260, 125, 1287, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3820, 3751, 4216, 4064, 4404, 4150, 4629, 3734] avg=4096.0000 max=4629 max_rank=6 min=3734 min_rank=7 imbalance_ratio=1.130127 max_load_reduction=28139 max_load_reduction_pct=0.858734 moved_to_echo=16917 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1937, 780, 94, 285, 284, 525, 151, 27, 0, 0, 0, 0, 0, 0, 0, 0, 441, 117, 62, 5, 85, 408, 370, 1483, 507, 518, 43, 0, 0, 0, 0, 0, 183, 497, 1258, 0, 182, 662, 1317, 173, 0, 0, 0, 0, 0, 0, 0, 0, 811, 269, 153, 1620, 602, 342, 30, 291, 6, 0, 0, 0, 0, 0, 0, 0, 358, 595, 812, 130, 1289, 128, 264, 389, 40, 21, 1, 0, 0, 0, 0, 0, 819, 333, 651, 11, 1440, 515, 82, 391, 0, 0, 0, 0, 0, 0, 0, 0, 353, 671, 643, 117, 1407, 158, 211, 317, 0, 0, 0, 0, 0, 0, 0, 0, 657, 37, 697, 387, 145, 524, 260, 121, 1276, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4039, 4272, 4124, 4027, 4242, 3877, 4104] avg=4096.0000 max=4272 max_rank=2 min=3877 min_rank=6 imbalance_ratio=1.042969 max_load_reduction=28496 max_load_reduction_pct=0.869629 moved_to_echo=16250 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2315, 723, 75, 281, 298, 379, 64, 57, 0, 0, 0, 0, 0, 0, 0, 0, 650, 368, 84, 4, 172, 288, 285, 1633, 612, 341, 40, 0, 0, 0, 0, 0, 153, 572, 1197, 0, 204, 560, 1126, 146, 0, 0, 0, 0, 0, 0, 0, 0, 989, 235, 49, 1897, 826, 368, 48, 210, 6, 0, 0, 0, 0, 0, 0, 0, 331, 895, 812, 47, 807, 270, 672, 462, 38, 25, 1, 0, 0, 0, 0, 0, 228, 535, 285, 33, 1537, 498, 152, 340, 0, 0, 0, 0, 0, 0, 0, 0, 306, 513, 523, 68, 1698, 321, 124, 216, 0, 0, 0, 0, 0, 0, 0, 0, 703, 25, 543, 215, 16, 541, 817, 78, 838, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4192, 4477, 3958, 4628, 4360, 3608, 3769, 3776] avg=4096.0000 max=4628 max_rank=3 min=3608 min_rank=5 imbalance_ratio=1.129883 max_load_reduction=28140 max_load_reduction_pct=0.858765 moved_to_echo=15513 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2304, 913, 156, 359, 307, 408, 81, 40, 0, 0, 0, 0, 0, 0, 0, 0, 415, 332, 197, 6, 45, 401, 229, 1476, 522, 484, 37, 0, 0, 0, 0, 0, 120, 514, 1099, 0, 343, 693, 1417, 206, 0, 0, 0, 0, 0, 0, 0, 0, 1188, 140, 149, 1034, 447, 386, 67, 255, 7, 0, 0, 0, 0, 0, 0, 0, 217, 718, 556, 93, 1312, 222, 318, 410, 35, 25, 1, 0, 0, 0, 0, 0, 406, 244, 214, 3, 1649, 519, 23, 719, 0, 0, 0, 0, 0, 0, 0, 0, 476, 408, 407, 76, 1448, 353, 152, 667, 0, 0, 0, 0, 0, 0, 0, 0, 743, 28, 815, 584, 61, 445, 276, 175, 1193, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4568, 4144, 4392, 3673, 3907, 3777, 3987, 4320] avg=4096.0000 max=4568 max_rank=0 min=3673 min_rank=3 imbalance_ratio=1.115234 max_load_reduction=28200 max_load_reduction_pct=0.860596 moved_to_echo=15991 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2425, 688, 150, 280, 249, 300, 133, 31, 0, 0, 0, 0, 0, 0, 0, 0, 280, 275, 119, 12, 80, 556, 161, 1473, 476, 435, 47, 0, 0, 0, 0, 0, 138, 269, 1400, 0, 277, 438, 1435, 240, 0, 0, 0, 0, 0, 0, 0, 0, 943, 275, 181, 1531, 671, 486, 39, 305, 6, 0, 0, 0, 0, 0, 0, 0, 271, 653, 679, 187, 1032, 287, 549, 645, 44, 27, 1, 0, 0, 0, 0, 0, 364, 472, 380, 18, 1596, 499, 69, 457, 0, 0, 0, 0, 0, 0, 0, 0, 505, 733, 657, 126, 1322, 141, 115, 389, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 32, 414, 283, 16, 347, 480, 99, 1068, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4256, 3914, 4197, 4437, 4375, 3855, 3988, 3746] avg=4096.0000 max=4437 max_rank=3 min=3746 min_rank=7 imbalance_ratio=1.083252 max_load_reduction=28331 max_load_reduction_pct=0.864594 moved_to_echo=15964 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1545, 1025, 61, 466, 142, 327, 171, 9, 0, 0, 0, 0, 0, 0, 0, 0, 435, 231, 94, 0, 32, 435, 137, 1860, 523, 519, 37, 0, 0, 0, 0, 0, 72, 418, 1094, 0, 95, 995, 1023, 653, 0, 0, 0, 0, 0, 0, 0, 0, 866, 212, 140, 1319, 703, 407, 92, 183, 8, 0, 0, 0, 0, 0, 0, 0, 240, 886, 824, 86, 610, 107, 616, 343, 34, 17, 1, 0, 0, 0, 0, 0, 428, 475, 203, 6, 1935, 697, 144, 728, 0, 0, 0, 0, 0, 0, 0, 0, 621, 393, 882, 19, 1450, 207, 136, 975, 0, 0, 0, 0, 0, 0, 0, 0, 629, 47, 611, 251, 18, 281, 147, 116, 1276, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3746, 4303, 4350, 3930, 3764, 4616, 4683, 3376] avg=4096.0000 max=4683 max_rank=6 min=3376 min_rank=7 imbalance_ratio=1.143311 max_load_reduction=28085 max_load_reduction_pct=0.857086 moved_to_echo=16439 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1553, 847, 207, 428, 221, 560, 216, 198, 0, 0, 0, 0, 0, 0, 0, 0, 368, 352, 90, 0, 72, 188, 362, 1658, 494, 562, 41, 0, 0, 0, 0, 0, 186, 731, 1107, 0, 58, 557, 870, 42, 0, 0, 0, 0, 0, 0, 0, 0, 487, 235, 208, 1051, 575, 644, 79, 193, 9, 0, 0, 0, 0, 0, 0, 0, 273, 643, 628, 27, 1038, 135, 156, 305, 38, 22, 4, 0, 0, 0, 0, 0, 1031, 573, 690, 52, 1406, 261, 162, 390, 0, 0, 0, 0, 0, 0, 0, 0, 430, 562, 693, 135, 1359, 203, 23, 355, 0, 0, 0, 0, 0, 0, 0, 0, 1092, 25, 1612, 510, 93, 344, 576, 91, 1382, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4230, 4187, 3551, 3481, 3269, 4565, 3760, 5725] avg=4096.0000 max=5725 max_rank=7 min=3269 min_rank=4 imbalance_ratio=1.397705 max_load_reduction=27043 max_load_reduction_pct=0.825287 moved_to_echo=17319 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1682, 690, 122, 298, 213, 635, 173, 60, 0, 0, 0, 0, 0, 0, 0, 0, 298, 365, 68, 2, 136, 302, 216, 1440, 498, 599, 29, 0, 0, 0, 0, 0, 67, 693, 854, 0, 104, 831, 994, 289, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 195, 151, 1623, 493, 633, 79, 188, 6, 0, 0, 0, 0, 0, 0, 0, 306, 825, 724, 100, 1548, 167, 435, 511, 27, 18, 1, 0, 0, 0, 0, 0, 396, 338, 225, 26, 1419, 825, 73, 653, 0, 0, 0, 0, 0, 0, 0, 0, 801, 486, 441, 127, 1382, 279, 111, 448, 0, 0, 0, 0, 0, 0, 0, 0, 592, 22, 914, 215, 65, 230, 313, 161, 1475, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3873, 3953, 3832, 4431, 4662, 3955, 4075, 3987] avg=4096.0000 max=4662 max_rank=4 min=3832 min_rank=2 imbalance_ratio=1.138184 max_load_reduction=28106 max_load_reduction_pct=0.857727 moved_to_echo=16679 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[6016, 256, 6400, 2688, 512, 3328, 3200, 1024, 9856, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4480, 4608, 4992, 768, 11648, 2048, 1152, 3584, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[2560, 6272, 5760, 896, 8832, 1408, 3456, 3584, 384, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 3328, 3200, 256, 12672, 4608, 768, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[3328, 2432, 768, 128, 768, 2944, 2048, 12416, 4224, 4096, 384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[7296, 1664, 1280, 12032, 4864, 3584, 512, 1920, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1024, 4224, 9472, 0, 1408, 5504, 9344, 2048, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[15872, 6400, 1024, 2688, 2048, 3584, 1152, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 1068, 5, 0, 4, 0, 107, 1, 3221, 0, 5, 0, 32, 0, 881, 1, 0, 0, 0, 0, 3, 2563, 0, 207, 3237, 0, 0, 1303, 86, 2042, 1821, 4080, 0, 0, 0, 1063, 10, 2, 0, 3757, 13, 605, 0, 0, 0, 3, 15, 3425, 3, 34, 1, 62, 1, 0, 13, 155, 782, 476, 71, 0, 653, 940, 17, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[19, 2113, 1452, 20, 0, 1, 812, 1, 815, 1, 6, 0, 835, 7, 67, 4, 55, 0, 1, 3, 0, 1295, 2, 220, 1812, 24, 0, 3443, 1367, 579, 43, 2710, 13, 0, 89, 1232, 1, 3, 8, 3037, 8, 2789, 1660, 0, 1, 0, 165, 3110, 4, 159, 3, 906, 6, 105, 1, 50, 39, 7, 82, 1388, 51, 135, 8, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1, 1244, 547, 8, 15, 0, 3465, 2, 691, 7, 16, 213, 1, 1, 37, 3, 0, 0, 7, 0, 0, 3976, 0, 883, 27, 5, 0, 1211, 327, 1064, 203, 2270, 2, 0, 2, 3719, 211, 0, 0, 3327, 155, 19, 220, 251, 0, 0, 388, 3631, 0, 1, 0, 1840, 6, 106, 795, 765, 7, 4, 410, 12, 12, 657, 4, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[2, 16, 927, 0, 0, 1, 820, 0, 2827, 0, 0, 0, 1227, 0, 197, 1, 0, 0, 0, 1, 0, 2646, 16, 61, 1, 1536, 3, 2419, 703, 1611, 13, 3397, 17, 11, 2, 28, 71, 1, 357, 4042, 419, 383, 0, 6, 0, 6, 460, 3065, 2, 56, 14, 0, 0, 1090, 8, 62, 1180, 61, 511, 195, 1286, 1010, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[619, 1, 331, 0, 6, 0, 174, 7, 574, 0, 2, 0, 1881, 1, 165, 5, 0, 4, 2, 1, 0, 2962, 0, 813, 0, 1090, 0, 894, 21, 558, 32, 3692, 13, 3, 4, 2844, 5, 0, 3, 3949, 239, 46, 10, 0, 0, 37, 374, 2405, 291, 554, 0, 189, 0, 317, 15, 0, 1568, 0, 28, 27, 2723, 3285, 1, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[592, 676, 592, 0, 0, 0, 715, 7, 438, 0, 733, 14, 34, 0, 378, 187, 7, 0, 0, 5, 5, 3156, 1, 1959, 42, 74, 0, 3693, 344, 2819, 133, 3791, 11, 0, 0, 1245, 12, 0, 0, 4062, 58, 14, 10, 0, 0, 0, 44, 1452, 0, 21, 1, 707, 0, 0, 2, 383, 19, 1358, 384, 5, 246, 2248, 91, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 20, 0, 0, 2, 181, 593, 1, 1003, 0, 366, 1, 205, 84, 4, 90, 1, 0, 16, 0, 0, 3981, 2, 2827, 1, 11, 0, 3090, 3130, 993, 350, 3214, 0, 25, 0, 1205, 13, 3, 33, 443, 6, 215, 1, 3565, 0, 15, 820, 1852, 64, 1, 8, 33, 36, 262, 0, 169, 2, 3, 8, 255, 391, 3174, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[21, 114, 738, 0, 55, 0, 2781, 127, 876, 27, 13, 11, 880, 0, 1517, 0, 1, 3, 89, 7, 0, 1497, 0, 2903, 0, 2, 0, 3838, 111, 44, 11, 3643, 0, 97, 127, 1032, 204, 1, 0, 2958, 2144, 25, 199, 7, 0, 1, 13, 1857, 0, 222, 0, 1, 85, 210, 0, 160, 390, 0, 56, 10, 1730, 1928, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1068, 5, 0, 4, 0, 107, 1, 959, 341, 2, 0, 0, 0, 0, 0, 3221, 0, 5, 0, 32, 0, 881, 1, 535, 610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2563, 0, 207, 96, 0, 0, 0, 0, 0, 0, 0, 3237, 0, 0, 427, 86, 2042, 1821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 10, 2, 0, 2798, 0, 0, 0, 0, 0, 0, 0, 0, 13, 605, 0, 0, 0, 3, 15, 2861, 0, 0, 0, 0, 0, 0, 0, 0, 3, 34, 1, 62, 1, 0, 13, 155, 3470, 0, 0, 0, 0, 0, 0, 0, 782, 476, 71, 0, 653, 940, 17, 0, 466, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2487, 5285, 2869, 7613, 3873, 3497, 3739, 3405] avg=4096.0000 max=7613 max_rank=3 min=2487 min_rank=0 imbalance_ratio=1.858643 max_load_reduction=25155 max_load_reduction_pct=0.767670 moved_to_echo=14514 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[619, 1, 331, 0, 6, 0, 174, 7, 1008, 234, 1, 0, 0, 0, 0, 0, 574, 0, 2, 0, 1881, 1, 165, 5, 367, 552, 0, 0, 0, 0, 0, 0, 0, 4, 2, 1, 0, 2962, 0, 813, 67, 0, 0, 0, 0, 0, 0, 0, 0, 1090, 0, 293, 21, 558, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 3, 4, 2844, 5, 0, 3, 2941, 0, 0, 0, 0, 0, 0, 0, 0, 239, 46, 10, 0, 0, 37, 374, 2010, 0, 0, 0, 0, 0, 0, 0, 0, 291, 554, 0, 189, 0, 317, 15, 0, 3140, 0, 0, 0, 0, 0, 0, 0, 1568, 0, 28, 27, 2723, 3285, 1, 3, 327, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2381, 3547, 3849, 1994, 5813, 2716, 4506, 7962] avg=4096.0000 max=7962 max_rank=7 min=1994 min_rank=3 imbalance_ratio=1.943848 max_load_reduction=24806 max_load_reduction_pct=0.757019 moved_to_echo=20997 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 16, 927, 0, 0, 1, 820, 0, 1031, 634, 2, 0, 0, 0, 0, 0, 2827, 0, 0, 0, 1227, 0, 197, 1, 994, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2646, 16, 61, 86, 0, 0, 0, 0, 0, 0, 0, 1, 1536, 3, 791, 703, 1611, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 11, 2, 28, 71, 1, 357, 3011, 0, 0, 0, 0, 0, 0, 0, 0, 419, 383, 0, 6, 0, 6, 460, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 2, 56, 14, 0, 0, 1090, 8, 62, 2889, 0, 0, 0, 0, 0, 0, 0, 1180, 61, 511, 195, 1286, 1010, 0, 0, 417, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3433, 5754, 2810, 4658, 3498, 3834, 4121, 4660] avg=4096.0000 max=5754 max_rank=1 min=2810 min_rank=2 imbalance_ratio=1.404785 max_load_reduction=27014 max_load_reduction_pct=0.824402 moved_to_echo=16113 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[19, 2113, 1452, 20, 0, 1, 812, 1, 775, 903, 2, 0, 0, 0, 0, 0, 815, 1, 6, 0, 835, 7, 67, 4, 1415, 406, 0, 0, 0, 0, 0, 0, 55, 0, 1, 3, 0, 1295, 2, 220, 87, 0, 0, 0, 0, 0, 0, 0, 1812, 24, 0, 1125, 1367, 579, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 89, 1232, 1, 3, 8, 2262, 0, 0, 0, 0, 0, 0, 0, 0, 8, 2789, 1660, 0, 1, 0, 165, 2598, 0, 0, 0, 0, 0, 0, 0, 0, 4, 159, 3, 906, 6, 105, 1, 50, 2304, 0, 0, 0, 0, 0, 0, 0, 39, 7, 82, 1388, 51, 135, 8, 1, 423, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6098, 3556, 1663, 4950, 3608, 7221, 3538, 2134] avg=4096.0000 max=7221 max_rank=5 min=1663 min_rank=2 imbalance_ratio=1.762939 max_load_reduction=25547 max_load_reduction_pct=0.779633 moved_to_echo=16501 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[592, 676, 592, 0, 0, 0, 715, 7, 1036, 968, 1, 0, 0, 0, 0, 0, 438, 0, 733, 14, 34, 0, 378, 187, 1517, 567, 0, 0, 0, 0, 0, 0, 7, 0, 0, 5, 5, 3156, 1, 1959, 40, 0, 0, 0, 0, 0, 0, 0, 42, 74, 0, 1208, 344, 2819, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 1245, 12, 0, 0, 3026, 0, 0, 0, 0, 0, 0, 0, 0, 58, 14, 10, 0, 0, 0, 44, 1214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 1, 707, 0, 0, 2, 383, 3224, 0, 0, 0, 0, 0, 0, 0, 19, 1358, 384, 5, 246, 2248, 91, 0, 197, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4587, 3868, 5173, 4620, 4294, 1340, 4338, 4548] avg=4096.0000 max=5173 max_rank=2 min=1340 min_rank=5 imbalance_ratio=1.262939 max_load_reduction=27595 max_load_reduction_pct=0.842133 moved_to_echo=14520 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[21, 114, 738, 0, 55, 0, 2781, 127, 755, 1006, 1, 0, 0, 0, 0, 0, 876, 27, 13, 11, 880, 0, 1517, 0, 1577, 545, 0, 0, 0, 0, 0, 0, 1, 3, 89, 7, 0, 1497, 0, 2903, 52, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1255, 111, 44, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 127, 1032, 204, 1, 0, 2203, 0, 0, 0, 0, 0, 0, 0, 0, 2144, 25, 199, 7, 0, 1, 13, 1551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 222, 0, 1, 85, 210, 0, 160, 3098, 0, 0, 0, 0, 0, 0, 0, 390, 0, 56, 10, 1730, 1928, 2, 0, 253, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5598, 5446, 4552, 1423, 3664, 3940, 3776, 4369] avg=4096.0000 max=5598 max_rank=0 min=1423 min_rank=3 imbalance_ratio=1.366699 max_load_reduction=27170 max_load_reduction_pct=0.829163 moved_to_echo=15749 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 1244, 547, 8, 15, 0, 3465, 2, 849, 317, 2, 0, 0, 0, 0, 0, 691, 7, 16, 213, 1, 1, 37, 3, 497, 339, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 3976, 0, 883, 102, 0, 0, 0, 0, 0, 0, 0, 27, 5, 0, 397, 327, 1064, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 3719, 211, 0, 0, 2478, 0, 0, 0, 0, 0, 0, 0, 0, 155, 19, 220, 251, 0, 0, 388, 3033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1840, 6, 106, 795, 765, 1931, 0, 0, 0, 0, 0, 0, 0, 7, 4, 410, 12, 12, 657, 4, 0, 494, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6450, 1805, 4968, 2023, 6412, 4066, 5444, 1600] avg=4096.0000 max=6450 max_rank=0 min=1600 min_rank=7 imbalance_ratio=1.574707 max_load_reduction=26318 max_load_reduction_pct=0.803162 moved_to_echo=17522 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 20, 0, 0, 2, 181, 593, 1, 116, 815, 6, 0, 0, 0, 0, 0, 1003, 0, 366, 1, 205, 84, 4, 90, 1273, 481, 0, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 3981, 2, 2827, 57, 0, 0, 0, 0, 0, 0, 0, 1, 11, 0, 1002, 3130, 993, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 1205, 13, 3, 33, 327, 0, 0, 0, 0, 0, 0, 0, 0, 6, 215, 1, 3565, 0, 15, 820, 1532, 0, 0, 0, 0, 0, 0, 0, 0, 64, 1, 8, 33, 36, 262, 0, 169, 2733, 0, 0, 0, 0, 0, 0, 0, 2, 3, 8, 255, 391, 3174, 0, 0, 257, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1734, 3507, 6884, 5487, 1606, 6154, 3306, 4090] avg=4096.0000 max=6884 max_rank=2 min=1606 min_rank=4 imbalance_ratio=1.680664 max_load_reduction=25884 max_load_reduction_pct=0.789917 moved_to_echo=15156 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 1920, 1664, 1920, 7168, 13440, 128, 128, 2944, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[5120, 2816, 128, 6528, 6144, 9728, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[10496, 128, 1152, 256, 5120, 128, 3328, 384, 8192, 4096, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3072, 4096, 2176, 3840, 128, 128, 2304, 17408, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 256, 256, 12416, 640, 128, 512, 19072, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 22144, 128, 9984, 640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[384, 1152, 128, 3840, 256, 2176, 896, 1792, 22912, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[1280, 5376, 4608, 128, 128, 256, 9472, 256, 6656, 5248, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 119, 0, 0, 1, 0, 0, 2582, 9, 0, 0, 0, 0, 0, 0, 0, 0, 214, 0, 0, 2, 602, 0, 4, 0, 4067, 0, 830, 648, 0, 4023, 0, 2753, 0, 0, 0, 3, 1, 14, 488, 281, 0, 3650, 5, 558, 3961, 2, 44, 1705, 0, 4066, 8, 0, 0, 1, 0, 0, 0, 11, 2102, 1, 13] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[405, 1, 0, 571, 0, 0, 1530, 0, 0, 3669, 3, 0, 22, 1, 0, 10, 0, 21, 1, 282, 0, 0, 0, 898, 489, 176, 0, 4038, 1, 0, 0, 10, 184, 3932, 4, 1, 0, 3285, 907, 1, 0, 951, 1270, 59, 428, 0, 2, 1424, 665, 15, 34, 12, 3602, 0, 0, 2, 1050, 1, 0, 84, 2518, 0, 209, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 8, 0, 0, 0, 0, 2, 12, 1220, 124, 1, 1405, 7, 113, 1, 3, 0, 251, 0, 4, 20, 0, 0, 3855, 10, 10, 1, 3054, 202, 0, 730, 0, 1987, 1684, 0, 33, 0, 176, 1178, 0, 0, 0, 95, 48, 3414, 27, 0, 0, 588, 0, 0, 0, 4022, 303, 0, 0, 93, 0, 1, 1, 4077, 4007, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 4, 0, 239, 4, 0, 2050, 0, 0, 2162, 0, 64, 3, 3, 0, 0, 9, 2301, 2, 0, 46, 9, 0, 228, 30, 803, 50, 4018, 20, 0, 3270, 8, 1032, 3917, 6, 6, 0, 427, 312, 81, 0, 1, 3979, 0, 2712, 2, 573, 688, 120, 0, 1723, 0, 0, 12, 0, 25, 0, 0, 0, 3, 1741, 6, 0, 79] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[89, 619, 1, 3, 1, 0, 95, 0, 0, 3193, 0, 20, 15, 0, 5, 0, 1, 8, 0, 66, 0, 0, 0, 2148, 0, 1, 0, 2922, 0, 4, 4012, 329, 116, 3988, 1, 0, 0, 0, 3418, 0, 0, 1, 185, 13, 3864, 0, 4, 921, 314, 272, 4, 0, 3731, 0, 0, 5, 141, 715, 0, 3, 125, 1381, 0, 34] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[103, 1, 3, 0, 0, 1, 769, 0, 510, 2078, 0, 3, 0, 0, 0, 0, 82, 756, 1, 0, 0, 0, 0, 3264, 0, 0, 0, 3280, 5, 61, 456, 0, 2264, 369, 0, 183, 0, 4016, 4095, 0, 0, 0, 444, 13, 99, 1, 0, 0, 1932, 996, 1, 0, 4000, 0, 12, 0, 9, 5, 0, 0, 2560, 393, 2, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 7, 6, 3064, 3, 133, 221, 3, 2594, 0, 0, 0, 0, 1, 0, 3, 1, 0, 0, 0, 750, 0, 2, 0, 3404, 3, 41, 1789, 0, 3652, 878, 0, 252, 27, 175, 0, 0, 0, 0, 855, 1, 645, 2905, 0, 848, 243, 704, 2629, 23, 4012, 0, 3, 0, 0, 1, 0, 0, 2886, 0, 0, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[131, 0, 0, 100, 0, 67, 65, 1, 0, 2758, 0, 0, 0, 0, 0, 0, 893, 2, 0, 35, 0, 61, 7, 2259, 0, 21, 4, 1357, 19, 317, 0, 0, 3703, 183, 528, 2, 0, 5, 3973, 0, 1, 0, 3300, 29, 4079, 7, 2588, 889, 1, 0, 95, 0, 455, 0, 0, 94, 39, 145, 0, 0, 3542, 297, 0, 716] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 119, 0, 0, 1, 0, 4023, 1107, 0, 0, 0, 0, 0, 0, 0, 2582, 9, 0, 0, 0, 0, 0, 1192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 214, 0, 0, 2, 602, 1511, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2865, 0, 830, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2753, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 14, 488, 281, 0, 1032, 5, 558, 3961, 0, 0, 0, 0, 0, 0, 0, 0, 2, 44, 1705, 0, 3436, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 11, 2102, 1, 13, 630, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5250, 3783, 2329, 4347, 2757, 6339, 5195, 2768] avg=4096.0000 max=6339 max_rank=5 min=2329 min_rank=2 imbalance_ratio=1.547607 max_load_reduction=26429 max_load_reduction_pct=0.806549 moved_to_echo=17059 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 0, 0, 0, 0, 2, 12, 1987, 1035, 0, 0, 0, 0, 0, 0, 1220, 124, 1, 1405, 7, 113, 1, 3, 895, 244, 0, 0, 0, 0, 0, 0, 0, 251, 0, 4, 20, 0, 0, 3855, 1413, 660, 0, 0, 0, 0, 0, 0, 10, 10, 1, 2152, 202, 0, 730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 780, 0, 33, 0, 176, 1178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 48, 966, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 588, 0, 0, 0, 3399, 303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 1, 1, 4077, 4007, 0, 1, 623, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3044, 4013, 6203, 3105, 2167, 1136, 4290, 8810] avg=4096.0000 max=8810 max_rank=7 min=1136 min_rank=5 imbalance_ratio=2.150879 max_load_reduction=23958 max_load_reduction_pct=0.731140 moved_to_echo=16403 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 239, 4, 0, 2050, 0, 1032, 822, 0, 0, 0, 0, 0, 0, 0, 2162, 0, 64, 3, 3, 0, 0, 1177, 568, 0, 0, 0, 0, 0, 0, 9, 2301, 2, 0, 46, 9, 0, 228, 1123, 1535, 0, 0, 0, 0, 0, 0, 30, 803, 50, 2832, 20, 0, 3270, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1814, 6, 6, 0, 427, 312, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3979, 0, 767, 2, 573, 688, 0, 0, 0, 0, 0, 0, 0, 0, 120, 0, 1723, 0, 0, 12, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1741, 6, 0, 79, 0, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4151, 3977, 5253, 7013, 2646, 6010, 1880, 1838] avg=4096.0000 max=7013 max_rank=3 min=1838 min_rank=7 imbalance_ratio=1.712158 max_load_reduction=25755 max_load_reduction_pct=0.785980 moved_to_echo=12374 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[405, 1, 0, 571, 0, 0, 1530, 0, 184, 129, 0, 0, 0, 0, 0, 0, 0, 3669, 3, 0, 22, 1, 0, 10, 1183, 570, 0, 0, 0, 0, 0, 0, 0, 21, 1, 282, 0, 0, 0, 898, 177, 1541, 0, 0, 0, 0, 0, 0, 489, 176, 0, 2845, 1, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1821, 4, 1, 0, 3285, 907, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 951, 1270, 59, 122, 0, 2, 1424, 0, 0, 0, 0, 0, 0, 0, 0, 665, 15, 34, 12, 3044, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1, 0, 84, 2518, 0, 209, 0, 558, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2820, 5458, 2920, 3521, 6019, 3828, 3772, 4430] avg=4096.0000 max=6019 max_rank=4 min=2820 min_rank=0 imbalance_ratio=1.469482 max_load_reduction=26749 max_load_reduction_pct=0.816315 moved_to_echo=18049 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 7, 6, 3064, 3, 3652, 195, 0, 0, 0, 0, 0, 0, 133, 221, 3, 2594, 0, 0, 0, 0, 997, 127, 0, 0, 0, 0, 0, 0, 1, 0, 3, 1, 0, 0, 0, 750, 267, 344, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2399, 3, 41, 1789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 407, 0, 252, 27, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 855, 1, 183, 2905, 0, 848, 0, 0, 0, 0, 0, 0, 0, 0, 243, 704, 2629, 23, 3390, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2886, 0, 0, 3, 622, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6928, 4075, 1366, 4234, 861, 4792, 6992, 3520] avg=4096.0000 max=6992 max_rank=6 min=861 min_rank=4 imbalance_ratio=1.707031 max_load_reduction=25776 max_load_reduction_pct=0.786621 moved_to_echo=16165 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[89, 619, 1, 3, 1, 0, 95, 0, 116, 1172, 0, 0, 0, 0, 0, 0, 0, 3193, 0, 20, 15, 0, 5, 0, 856, 578, 0, 0, 0, 0, 0, 0, 1, 8, 0, 66, 0, 0, 0, 2148, 1600, 1563, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2059, 0, 4, 4012, 329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1847, 1, 0, 0, 0, 3418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 185, 13, 1092, 0, 4, 921, 0, 0, 0, 0, 0, 0, 0, 0, 314, 272, 4, 0, 3153, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 141, 715, 0, 3, 125, 1381, 0, 34, 578, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2096, 4667, 5386, 6405, 5266, 2216, 3748, 2984] avg=4096.0000 max=6405 max_rank=3 min=2096 min_rank=0 imbalance_ratio=1.563721 max_load_reduction=26363 max_load_reduction_pct=0.804535 moved_to_echo=14214 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[131, 0, 0, 100, 0, 67, 65, 1, 3703, 1237, 0, 0, 0, 0, 0, 0, 0, 2758, 0, 0, 0, 0, 0, 0, 397, 26, 0, 0, 0, 0, 0, 0, 893, 2, 0, 35, 0, 61, 7, 2259, 1689, 71, 0, 0, 0, 0, 0, 0, 0, 21, 4, 957, 19, 317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 528, 2, 0, 5, 3973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3300, 29, 1153, 7, 2588, 889, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 95, 0, 385, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 39, 145, 0, 0, 3542, 297, 0, 716, 70, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5304, 3181, 5017, 1318, 4594, 7967, 575, 4812] avg=4096.0000 max=7967 max_rank=5 min=575 min_rank=6 imbalance_ratio=1.945068 max_load_reduction=24801 max_load_reduction_pct=0.756866 moved_to_echo=17948 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[103, 1, 3, 0, 0, 1, 769, 0, 2264, 34, 0, 0, 0, 0, 0, 0, 510, 2078, 0, 3, 0, 0, 0, 0, 966, 57, 0, 0, 0, 0, 0, 0, 82, 756, 1, 0, 0, 0, 0, 3264, 43, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2303, 5, 61, 456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 0, 183, 0, 4016, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 444, 13, 22, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1932, 996, 1, 0, 3375, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 5, 0, 0, 2560, 393, 2, 1, 625, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3175, 3614, 4294, 2825, 8458, 480, 6316, 3606] avg=4096.0000 max=8458 max_rank=4 min=480 min_rank=5 imbalance_ratio=2.064941 max_load_reduction=24310 max_load_reduction_pct=0.741882 moved_to_echo=18860 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 1536, 10496, 256, 5376, 3072, 3840, 8832, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 7040, 3328, 512, 128, 8192, 13952, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[768, 640, 128, 1152, 128, 128, 7680, 128, 17024, 5760, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[640, 1024, 128, 18432, 256, 1280, 11008, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3968, 2048, 6272, 128, 20224, 384, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[1024, 3456, 128, 640, 128, 128, 128, 14080, 7936, 5888, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1408, 896, 128, 128, 17536, 8192, 256, 896, 3712, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1920, 16896, 128, 4096, 128, 128, 128, 128, 7680, 2176, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 3979, 4067, 73, 0, 3703, 148, 6, 0, 3501, 13, 4, 1419, 1, 0, 0, 0, 4093, 107, 0, 0, 91, 13, 4, 82, 0, 11, 0, 1, 135, 0, 0, 0, 35, 3881, 0, 0, 0, 0, 0, 60, 0, 703, 0, 1, 19, 1, 0, 0, 0, 15, 15, 2732, 1, 3503, 1, 0, 1, 0, 300, 0, 12, 23, 14] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 4043, 663, 2, 330, 97, 0, 578, 0, 3701, 0, 0, 520, 0, 0, 0, 0, 4096, 0, 3, 0, 618, 0, 0, 0, 0, 0, 1048, 0, 195, 0, 0, 0, 20, 3761, 0, 0, 8, 0, 0, 1, 0, 4051, 0, 126, 0, 4021, 1, 70, 0, 378, 1, 3432, 1, 58, 0, 591, 1, 0, 160, 160, 0, 29, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[275, 3492, 116, 1, 0, 0, 0, 0, 0, 4066, 0, 58, 458, 1, 0, 306, 0, 3984, 0, 0, 0, 85, 0, 1, 0, 8, 156, 169, 0, 0, 0, 35, 3, 0, 3230, 2, 0, 0, 0, 0, 1030, 62, 3578, 1, 2, 0, 3242, 0, 46, 32, 16, 0, 4084, 0, 3710, 48, 0, 0, 0, 0, 471, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 3627, 182, 7, 50, 210, 0, 0, 1, 1588, 0, 0, 0, 2, 0, 8, 4, 4080, 0, 0, 0, 0, 0, 2, 1, 0, 4, 0, 1, 313, 0, 15, 80, 3, 2282, 0, 2, 90, 2, 0, 0, 7, 2318, 1474, 2, 0, 3728, 0, 3599, 0, 2, 59, 3762, 0, 0, 0, 1551, 0, 0, 114, 0, 61, 1, 3536] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 212, 3650, 88, 0, 0, 110, 0, 589, 7, 0, 3875, 0, 46, 505, 1, 1714, 1, 0, 0, 9, 0, 0, 0, 0, 4, 1040, 0, 0, 269, 0, 1129, 3465, 1296, 238, 0, 0, 0, 313, 3947, 1, 75, 0, 0, 2, 1392, 1, 18, 8, 0, 522, 3547, 1, 4, 5, 120, 2, 0, 57, 0, 0, 484, 4021] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 917, 235, 1105, 1524, 2401, 0, 0, 0, 4037, 1, 0, 0, 0, 2, 454, 124, 4085, 0, 0, 0, 3184, 0, 0, 1, 0, 72, 0, 0, 44, 3, 0, 533, 26, 3569, 0, 0, 0, 0, 8, 0, 2, 1498, 0, 2, 0, 72, 0, 77, 2, 860, 877, 3743, 0, 1, 0, 2091, 4, 0, 959, 241, 1, 0, 13] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 22, 0, 147, 0, 0, 0, 2620, 917, 3723, 37, 0, 341, 0, 0, 0, 0, 4090, 48, 0, 0, 810, 33, 0, 0, 0, 0, 1896, 218, 0, 230, 0, 0, 0, 2438, 0, 0, 0, 0, 163, 0, 0, 62, 362, 2320, 1, 3781, 383, 253, 4, 0, 3676, 3977, 0, 0, 0, 0, 0, 0, 0, 176, 0, 0, 40] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 1180, 3740, 22, 46, 3673, 0, 0, 144, 835, 181, 0, 0, 0, 288, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 573, 198, 3, 351, 18, 18, 1351, 4033, 0, 0, 0, 0, 0, 0, 0, 2761, 0, 2313, 1, 106, 61, 0, 44, 1190, 0, 4066, 0, 48, 0, 1, 0, 40, 347, 543, 0, 0, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[275, 545, 116, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 58, 458, 1, 0, 306, 150, 0, 0, 0, 0, 0, 0, 0, 0, 3641, 0, 0, 0, 85, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 156, 169, 0, 0, 0, 35, 2187, 2013, 0, 0, 0, 0, 0, 0, 3, 0, 3230, 2, 0, 0, 0, 0, 103, 0, 0, 0, 0, 0, 0, 0, 1030, 62, 3578, 1, 2, 0, 1099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 32, 16, 0, 1897, 0, 3710, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 0, 0, 0, 2143, 934, 90, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[937, 5039, 3727, 4568, 3338, 5772, 5749, 3638] avg=4096.0000 max=5772 max_rank=5 min=937 min_rank=0 imbalance_ratio=1.409180 max_load_reduction=26996 max_load_reduction_pct=0.823853 moved_to_echo=18497 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 631, 663, 2, 330, 97, 0, 578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3701, 0, 0, 520, 0, 0, 0, 154, 0, 0, 0, 0, 0, 0, 0, 0, 3744, 0, 3, 0, 618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 0, 195, 0, 0, 1838, 2331, 0, 0, 0, 0, 0, 0, 0, 20, 3761, 0, 0, 8, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4051, 0, 126, 0, 1364, 1, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 378, 1, 1594, 1, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 591, 1, 0, 160, 160, 0, 29, 4, 2657, 1081, 92, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2301, 4375, 4365, 5412, 3895, 5543, 2102, 4775] avg=4096.0000 max=5543 max_rank=5 min=2102 min_rank=6 imbalance_ratio=1.353271 max_load_reduction=27225 max_load_reduction_pct=0.830841 moved_to_echo=16315 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 621, 4067, 73, 0, 3703, 148, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3501, 13, 4, 1419, 1, 0, 0, 154, 0, 0, 0, 0, 0, 0, 0, 0, 3741, 107, 0, 0, 91, 13, 4, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 11, 0, 1, 135, 0, 0, 1463, 2294, 0, 0, 0, 0, 0, 0, 0, 35, 3881, 0, 0, 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 60, 0, 703, 0, 1, 19, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 15, 1269, 1, 3503, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 300, 0, 12, 23, 14, 0, 1064, 92, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8618, 5092, 3956, 3986, 4022, 784, 4804, 1506] avg=4096.0000 max=8618 max_rank=0 min=784 min_rank=5 imbalance_ratio=2.104004 max_load_reduction=24150 max_load_reduction_pct=0.737000 moved_to_echo=11116 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 566, 182, 7, 50, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1588, 0, 0, 0, 2, 0, 8, 153, 0, 0, 0, 0, 0, 0, 0, 4, 3729, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 1, 313, 0, 15, 2015, 2091, 0, 0, 0, 0, 0, 0, 80, 3, 2282, 0, 2, 90, 2, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2318, 1474, 2, 0, 1264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3599, 0, 2, 59, 1747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1551, 0, 0, 114, 0, 61, 1, 3536, 2464, 970, 92, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1015, 1752, 3735, 4440, 2565, 5065, 5407, 8789] avg=4096.0000 max=8789 max_rank=7 min=1015 min_rank=0 imbalance_ratio=2.145752 max_load_reduction=23979 max_load_reduction_pct=0.731781 moved_to_echo=21826 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 212, 3650, 88, 0, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 589, 7, 0, 3875, 0, 46, 505, 64, 0, 0, 0, 0, 0, 0, 0, 1, 1568, 1, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1040, 0, 0, 269, 0, 1900, 0, 0, 0, 0, 0, 0, 0, 1129, 3465, 1296, 238, 0, 0, 0, 313, 44, 0, 0, 0, 0, 0, 0, 0, 3947, 1, 75, 0, 0, 2, 472, 1, 0, 0, 0, 0, 0, 0, 0, 0, 18, 8, 0, 522, 1647, 1, 4, 5, 0, 0, 0, 0, 0, 0, 0, 0, 120, 2, 0, 57, 0, 0, 484, 4021, 920, 0, 38, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 5086, 1579, 3213, 6485, 4498, 2205, 5642] avg=4096.0000 max=6485 max_rank=4 min=1579 min_rank=2 imbalance_ratio=1.583252 max_load_reduction=26283 max_load_reduction_pct=0.802094 moved_to_echo=18830 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 144, 235, 1105, 1524, 2401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 1, 0, 0, 0, 2, 454, 153, 0, 0, 0, 0, 0, 0, 0, 124, 3734, 0, 0, 0, 3184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 72, 0, 0, 44, 3, 0, 2005, 528, 0, 0, 0, 0, 0, 0, 533, 26, 3569, 0, 0, 0, 0, 8, 106, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1498, 0, 2, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 2, 860, 877, 1738, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2091, 4, 0, 959, 241, 1, 0, 13, 47, 245, 92, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5409, 4647, 7042, 2653, 4242, 1527, 3555, 3693] avg=4096.0000 max=7042 max_rank=2 min=1527 min_rank=5 imbalance_ratio=1.719238 max_load_reduction=25726 max_load_reduction_pct=0.785095 moved_to_echo=13017 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 180, 3740, 22, 46, 3673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 835, 181, 0, 0, 0, 288, 0, 154, 0, 0, 0, 0, 0, 0, 0, 0, 3744, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 573, 198, 3, 351, 18, 2178, 680, 0, 0, 0, 0, 0, 0, 18, 1351, 4033, 0, 0, 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2761, 0, 2313, 1, 36, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 1190, 0, 1888, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 40, 347, 543, 0, 0, 493, 70, 320, 92, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7661, 1602, 3748, 4001, 5508, 5172, 3170, 1906] avg=4096.0000 max=7661 max_rank=0 min=1602 min_rank=1 imbalance_ratio=1.870361 max_load_reduction=25107 max_load_reduction_pct=0.766205 moved_to_echo=15756 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 147, 0, 0, 0, 2620, 0, 0, 0, 0, 0, 0, 0, 0, 917, 3723, 37, 0, 341, 0, 0, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 3725, 48, 0, 0, 810, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1896, 218, 0, 230, 0, 2134, 17, 0, 0, 0, 0, 0, 0, 0, 0, 2438, 0, 0, 0, 0, 163, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 362, 2320, 1, 1279, 383, 0, 0, 0, 0, 0, 0, 0, 0, 253, 4, 0, 3676, 1843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 0, 40, 2502, 5, 96, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2767, 5175, 4616, 4495, 2713, 4407, 5776, 2819] avg=4096.0000 max=5776 max_rank=6 min=2713 min_rank=4 imbalance_ratio=1.410156 max_load_reduction=26992 max_load_reduction_pct=0.823730 moved_to_echo=15715 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[384, 2688, 9216, 5120, 2048, 10112, 256, 3328, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 256, 4736, 512, 768, 896, 128, 15744, 9984, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[4480, 128, 128, 2048, 1664, 128, 640, 8192, 10880, 4736, 768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[5120, 128, 15104, 1920, 4864, 128, 5632, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 128, 2560, 5248, 13696, 128, 7424, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[256, 27648, 256, 128, 0, 4864, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1792, 4992, 24576, 256, 128, 128, 128, 512, 896, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1152, 22144, 256, 128, 6656, 128, 384, 1280, 1152, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33792 slack_slots=31744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 3221, 124, 0, 101, 0, 0, 0, 3519, 20, 1, 0, 8, 1392, 1572, 311, 1325, 0, 0, 1945, 1, 0, 0, 0, 0, 0, 0, 4066, 3385, 0, 0, 490, 2, 0, 0, 0, 0, 0, 0, 2446, 0, 14, 3728, 8, 0, 28, 0, 0, 0, 0, 32, 0, 0, 11, 0, 405, 520, 1, 4091, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[490, 0, 1, 2, 1467, 566, 108, 821, 4027, 1, 0, 1281, 0, 0, 0, 4, 1441, 3490, 239, 0, 0, 0, 2761, 10, 0, 15, 0, 0, 0, 5, 846, 3990, 0, 0, 0, 0, 1, 88, 3, 0, 3028, 0, 0, 2, 0, 0, 4, 1781, 0, 0, 0, 3300, 683, 41, 0, 0, 0, 0, 1857, 19, 0, 326, 0, 70] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 216, 164, 543, 0, 0, 620, 0, 0, 1, 0, 0, 0, 51, 0, 4089, 79, 3818, 0, 0, 3731, 12, 0, 0, 0, 0, 0, 0, 0, 3744, 0, 4, 0, 3378, 0, 108, 0, 0, 0, 0, 0, 0, 0, 2527, 3937, 389, 0, 211, 0, 509, 2670, 0, 0, 0, 0, 0, 0, 130, 19, 1792, 0, 26] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[22, 0, 190, 3753, 12, 0, 1, 0, 2564, 0, 0, 398, 0, 392, 6, 0, 449, 0, 1, 4016, 0, 94, 272, 1837, 0, 51, 0, 822, 0, 37, 4076, 3147, 0, 0, 1, 2859, 0, 2, 0, 0, 746, 2837, 743, 145, 0, 0, 889, 254, 5, 0, 0, 1537, 100, 0, 2, 1, 0, 0, 20, 0, 0, 2, 476, 9] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[2, 0, 0, 3487, 380, 710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2894, 81, 0, 3698, 768, 3, 1197, 0, 41, 0, 0, 0, 0, 0, 2560, 3854, 0, 2, 2, 1, 0, 1568, 0, 0, 2, 256, 0, 1, 725, 0, 3376, 18, 0, 0, 0, 0, 15, 109, 0, 3, 0, 0, 25, 2791, 113, 4086, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 3855, 8, 8, 234, 4, 0, 0, 0, 1479, 0, 0, 0, 0, 0, 1680, 1337, 33, 8, 0, 16, 0, 0, 0, 0, 0, 2104, 3603, 2674, 3, 0, 7, 0, 4052, 251, 10, 0, 0, 2239, 3817, 0, 0, 2466, 4, 0, 0, 421, 0, 0, 0, 0, 6, 1, 0, 8, 82, 0, 862, 0, 1403, 0, 93] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[173, 0, 4, 36, 3694, 1135, 0, 0, 23, 1100, 0, 0, 101, 0, 54, 0, 3307, 0, 0, 35, 6, 189, 3659, 0, 0, 0, 776, 1, 0, 1770, 0, 3623, 0, 4, 0, 12, 4, 4, 0, 0, 0, 0, 3974, 42, 0, 221, 4033, 0, 207, 3932, 4, 141, 0, 0, 2, 0, 0, 0, 0, 483, 0, 9, 0, 10] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[13, 0, 0, 3699, 2398, 1741, 393, 0, 1, 35, 0, 4, 0, 0, 12, 0, 3736, 3653, 0, 0, 0, 232, 19, 0, 0, 1, 0, 2, 0, 0, 149, 3074, 0, 0, 0, 164, 11, 3, 0, 163, 1361, 0, 656, 2095, 736, 3178, 3850, 4, 67, 60, 7, 0, 43, 972, 1, 0, 0, 0, 63, 0, 1, 0, 66, 105] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 3221, 124, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3519, 20, 1, 0, 8, 981, 1141, 0, 0, 0, 0, 0, 0, 1392, 0, 311, 1325, 0, 0, 216, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 1251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 2, 0, 0, 0, 0, 8, 474, 0, 0, 0, 0, 0, 0, 0, 0, 2446, 0, 14, 769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 32, 0, 0, 1660, 591, 0, 0, 0, 0, 0, 0, 11, 0, 405, 520, 1, 4091, 0, 0, 2959, 588, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3447, 5670, 3245, 5317, 974, 3229, 2311, 8575] avg=4096.0000 max=8575 max_rank=7 min=974 min_rank=4 imbalance_ratio=2.093506 max_load_reduction=24193 max_load_reduction_pct=0.738312 moved_to_echo=15089 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[490, 0, 1, 2, 1467, 566, 108, 821, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 1, 0, 1281, 0, 0, 0, 4, 2175, 1621, 0, 0, 0, 0, 0, 0, 1441, 0, 239, 0, 0, 0, 304, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 5, 846, 1475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 88, 3, 0, 4, 558, 0, 0, 0, 0, 0, 0, 3028, 0, 0, 2, 0, 0, 0, 1781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3300, 683, 41, 0, 0, 1957, 1315, 0, 0, 0, 0, 0, 0, 0, 0, 1857, 19, 0, 326, 0, 70, 0, 836, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3455, 9109, 1994, 2341, 654, 4811, 7296, 3108] avg=4096.0000 max=9109 max_rank=1 min=654 min_rank=4 imbalance_ratio=2.223877 max_load_reduction=23659 max_load_reduction_pct=0.722015 moved_to_echo=15869 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3597, 8, 8, 234, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1479, 0, 0, 0, 0, 1047, 9, 0, 0, 0, 0, 0, 0, 0, 0, 1337, 33, 8, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2104, 3603, 2674, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4052, 251, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2239, 3817, 0, 0, 2466, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 421, 0, 0, 0, 0, 6, 1, 0, 1, 633, 0, 0, 0, 0, 0, 0, 8, 82, 0, 862, 0, 1403, 0, 93, 3, 4, 258, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3851, 2535, 1381, 8383, 4320, 8523, 1062, 2713] avg=4096.0000 max=8523 max_rank=5 min=1062 min_rank=6 imbalance_ratio=2.080811 max_load_reduction=24245 max_load_reduction_pct=0.739899 moved_to_echo=16618 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 202, 164, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 620, 0, 0, 1, 0, 0, 0, 51, 2550, 2190, 0, 0, 0, 0, 0, 0, 0, 0, 79, 3818, 0, 0, 412, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3378, 0, 108, 0, 0, 3937, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 0, 389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 211, 0, 509, 2670, 0, 0, 0, 1836, 1539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 19, 1792, 0, 26, 2006, 1129, 14, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[909, 5412, 4321, 1384, 7951, 910, 6765, 5116] avg=4096.0000 max=7951 max_rank=4 min=909 min_rank=0 imbalance_ratio=1.941162 max_load_reduction=24817 max_load_reduction_pct=0.757355 moved_to_echo=20742 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[22, 0, 190, 3502, 12, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2564, 0, 0, 398, 0, 392, 6, 0, 0, 159, 0, 0, 0, 0, 0, 0, 449, 0, 1, 4016, 0, 94, 31, 1837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 822, 0, 37, 4076, 1164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2859, 0, 2, 0, 0, 889, 440, 0, 0, 0, 0, 0, 0, 746, 2837, 743, 145, 0, 0, 0, 254, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 1537, 100, 0, 2, 1, 1543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 2, 476, 9, 0, 82, 251, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3727, 3519, 6428, 6150, 4191, 4725, 3188, 840] avg=4096.0000 max=6428 max_rank=2 min=840 min_rank=7 imbalance_ratio=1.569336 max_load_reduction=26340 max_load_reduction_pct=0.803833 moved_to_echo=12944 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 3254, 380, 710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 702, 0, 0, 0, 0, 0, 0, 2894, 0, 0, 3698, 768, 3, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 2560, 1425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 1, 0, 1568, 0, 0, 3376, 539, 0, 0, 0, 0, 0, 0, 2, 256, 0, 1, 725, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 109, 0, 3, 1890, 31, 0, 0, 0, 0, 0, 0, 0, 0, 25, 2791, 113, 4086, 0, 0, 0, 362, 233, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4346, 752, 7496, 4026, 5488, 1002, 2048, 7610] avg=4096.0000 max=7610 max_rank=7 min=752 min_rank=1 imbalance_ratio=1.857910 max_load_reduction=25158 max_load_reduction_pct=0.767761 moved_to_echo=16148 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[173, 0, 4, 31, 3694, 1135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 1100, 0, 0, 101, 0, 54, 0, 0, 2152, 0, 0, 0, 0, 0, 0, 3307, 0, 0, 35, 6, 189, 395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 776, 1, 0, 1770, 0, 1331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 12, 4, 4, 0, 0, 4033, 512, 0, 0, 0, 0, 0, 0, 0, 0, 3974, 42, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 3932, 4, 141, 0, 0, 2, 0, 1780, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 0, 9, 0, 10, 177, 1112, 5, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5037, 3430, 3932, 3878, 4569, 4060, 6066, 1796] avg=4096.0000 max=6066 max_rank=6 min=1796 min_rank=7 imbalance_ratio=1.480957 max_load_reduction=26702 max_load_reduction_pct=0.814880 moved_to_echo=16491 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[13, 0, 0, 3451, 2398, 1741, 393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 35, 0, 4, 0, 0, 12, 0, 2278, 11, 0, 0, 0, 0, 0, 0, 3736, 0, 0, 0, 0, 232, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 149, 1137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 11, 3, 0, 163, 3850, 430, 0, 0, 0, 0, 0, 0, 1361, 0, 656, 2095, 736, 656, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 67, 60, 7, 0, 43, 972, 1, 0, 1507, 1375, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 1, 0, 66, 105, 2522, 5, 248, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7996, 2341, 3971, 1289, 4621, 5508, 4032, 3010] avg=4096.0000 max=7996 max_rank=0 min=1289 min_rank=3 imbalance_ratio=1.952148 max_load_reduction=24772 max_load_reduction_pct=0.755981 moved_to_echo=17171 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 2432, 4864, 256, 11776, 640, 384, 7680, 4224, 1024, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[768, 4224, 128, 5504, 3584, 1280, 128, 128, 12288, 5504, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[7296, 1152, 0, 6784, 128, 512, 128, 128, 9088, 8064, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 896, 896, 2176, 5504, 14464, 9216, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[7424, 6912, 7936, 2304, 3968, 2048, 0, 2560, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[768, 128, 256, 14080, 11392, 4864, 768, 1024, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[13312, 0, 2048, 12928, 896, 640, 1536, 1920, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 512, 10496, 384, 1792, 128, 256, 16128, 3584, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[267, 0, 4066, 3435, 0, 0, 0, 210, 0, 0, 3947, 0, 2, 0, 0, 0, 0, 0, 0, 169, 50, 5, 0, 0, 0, 1, 1975, 0, 0, 34, 71, 0, 0, 0, 0, 3920, 294, 2741, 0, 5, 0, 0, 0, 0, 1343, 0, 0, 0, 0, 8, 3915, 0, 0, 1294, 210, 4094, 0, 614, 0, 0, 7, 1, 0, 90] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[3, 21, 3645, 0, 0, 0, 0, 1403, 20, 1190, 3848, 502, 0, 0, 118, 1, 0, 0, 0, 203, 0, 314, 100, 0, 0, 3402, 4036, 1, 0, 4, 37, 0, 0, 0, 1, 52, 1543, 3382, 0, 0, 0, 0, 0, 235, 1370, 0, 0, 0, 208, 0, 2352, 0, 0, 0, 0, 3744, 638, 34, 0, 0, 0, 0, 0, 361] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 2, 0, 3995, 0, 0, 0, 7, 0, 69, 94, 22, 0, 0, 9, 374, 0, 4072, 0, 847, 0, 0, 0, 0, 0, 617, 0, 2, 0, 12, 3, 60, 0, 20, 0, 2, 4081, 4031, 0, 814, 0, 0, 0, 0, 2320, 0, 1, 658, 0, 0, 4096, 0, 3, 2692, 0, 156, 76, 2, 0, 0, 2, 0, 0, 3629] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[1, 0, 3, 3758, 0, 0, 0, 0, 0, 40, 3780, 62, 0, 0, 2204, 123, 0, 2201, 5, 3227, 0, 19, 32, 0, 10, 0, 3839, 1, 0, 922, 0, 218, 0, 4, 0, 0, 1111, 3945, 0, 0, 0, 1, 0, 1, 1684, 0, 6, 295, 0, 10, 3484, 0, 0, 0, 0, 1782, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 12, 41, 0, 0, 0, 0, 0, 1823, 0, 0, 0, 0, 0, 3864, 0, 0, 7, 11, 0, 0, 302, 557, 0, 0, 0, 3573, 96, 0, 0, 0, 0, 0, 3985, 2349, 0, 150, 4, 2, 0, 66, 3267, 632, 0, 0, 0, 1, 3704, 0, 0, 0, 0, 976, 0, 4085, 0, 19, 0, 1, 1304, 1937] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 3956, 0, 0, 0, 0, 0, 0, 3313, 0, 0, 0, 0, 0, 0, 1395, 143, 114, 2890, 5, 22, 0, 5, 1, 0, 0, 0, 0, 0, 0, 5, 1, 0, 2, 4095, 2990, 49, 8, 0, 0, 3, 0, 4082, 535, 0, 0, 0, 4, 4079, 0, 0, 0, 0, 3855, 0, 9, 432, 771, 0, 4, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 84, 514, 0, 0, 44, 10, 0, 3561, 0, 0, 0, 0, 888, 0, 2418, 74, 94, 4, 0, 0, 0, 0, 579, 0, 0, 3, 3808, 0, 6, 1, 0, 0, 3831, 3708, 2076, 1, 0, 29, 0, 1763, 0, 4052, 0, 0, 0, 0, 0, 2620, 14, 0, 0, 0, 2567, 0, 9, 4, 0, 0, 0, 1, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 24, 20, 3, 0, 0, 0, 0, 0, 3990, 1, 0, 7, 7, 0, 0, 0, 246, 0, 87, 0, 2, 0, 3099, 3680, 0, 1087, 0, 0, 0, 28, 9, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4065, 0, 3, 1, 9, 0, 4096, 0, 0, 0, 0, 4086, 7, 52, 0, 0, 3, 0, 1, 4065] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[267, 0, 4066, 3435, 0, 0, 0, 210, 776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3947, 0, 2, 0, 0, 0, 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 50, 5, 0, 0, 923, 370, 0, 0, 0, 0, 0, 0, 0, 1, 1975, 0, 0, 34, 71, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3920, 294, 547, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1343, 0, 0, 0, 679, 0, 0, 0, 0, 0, 0, 0, 0, 8, 975, 0, 0, 1294, 210, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 614, 0, 0, 7, 1, 0, 90, 2017, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8754, 4244, 1517, 2155, 4766, 2022, 6581, 2729] avg=4096.0000 max=8754 max_rank=0 min=1517 min_rank=2 imbalance_ratio=2.137207 max_load_reduction=24014 max_load_reduction_pct=0.732849 moved_to_echo=16098 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 3995, 0, 0, 0, 7, 1142, 0, 0, 0, 0, 0, 0, 0, 0, 69, 94, 22, 0, 0, 9, 374, 434, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 847, 0, 0, 0, 0, 966, 545, 0, 0, 0, 0, 0, 0, 0, 617, 0, 2, 0, 12, 3, 60, 110, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 2, 4081, 801, 0, 814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2320, 0, 1, 658, 999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 0, 3, 2692, 0, 156, 0, 0, 0, 0, 0, 0, 0, 0, 76, 2, 0, 0, 2, 0, 0, 3629, 2110, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5146, 1002, 6430, 804, 5718, 3978, 3871, 5819] avg=4096.0000 max=6430 max_rank=2 min=804 min_rank=3 imbalance_ratio=1.569824 max_load_reduction=26338 max_load_reduction_pct=0.803772 moved_to_echo=19386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 3, 3758, 0, 0, 0, 0, 1117, 0, 0, 0, 0, 0, 0, 0, 0, 40, 3780, 62, 0, 0, 2204, 123, 425, 0, 0, 0, 0, 0, 0, 0, 0, 2201, 5, 3227, 0, 19, 32, 0, 821, 533, 0, 0, 0, 0, 0, 0, 10, 0, 3839, 1, 0, 922, 0, 218, 107, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1111, 785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1684, 0, 6, 295, 978, 0, 0, 0, 0, 0, 0, 0, 0, 10, 868, 0, 0, 0, 0, 1782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1795, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4879, 6634, 6838, 5097, 1900, 2965, 2660, 1795] avg=4096.0000 max=6838 max_rank=2 min=1795 min_rank=7 imbalance_ratio=1.669434 max_load_reduction=25930 max_load_reduction_pct=0.791321 moved_to_echo=9320 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[3, 21, 3645, 0, 0, 0, 0, 1403, 958, 0, 0, 0, 0, 0, 0, 0, 20, 1190, 3848, 502, 0, 0, 118, 1, 364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 203, 0, 314, 100, 0, 554, 457, 0, 0, 0, 0, 0, 0, 0, 3402, 4036, 1, 0, 4, 37, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 52, 1543, 673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 235, 1370, 0, 0, 0, 838, 0, 0, 0, 0, 0, 0, 0, 208, 0, 587, 0, 0, 0, 0, 3744, 0, 0, 0, 0, 0, 0, 0, 0, 638, 34, 0, 0, 0, 0, 0, 361, 1211, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6030, 6043, 1628, 7572, 2269, 2443, 4539, 2244] avg=4096.0000 max=7572 max_rank=3 min=1628 min_rank=2 imbalance_ratio=1.848633 max_load_reduction=25196 max_load_reduction_pct=0.768921 moved_to_echo=11495 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 12, 41, 0, 0, 665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1823, 0, 0, 0, 0, 253, 0, 0, 0, 0, 0, 0, 0, 0, 3864, 0, 0, 7, 11, 0, 0, 873, 317, 0, 0, 0, 0, 0, 0, 302, 557, 0, 0, 0, 3573, 96, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3985, 468, 0, 150, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 66, 3267, 632, 0, 0, 582, 0, 0, 0, 0, 0, 0, 0, 0, 1, 923, 0, 0, 0, 0, 976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 19, 0, 1, 1304, 1937, 1908, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[718, 2076, 5072, 4592, 4603, 4553, 1900, 9254] avg=4096.0000 max=9254 max_rank=7 min=718 min_rank=0 imbalance_ratio=2.259277 max_load_reduction=23514 max_load_reduction_pct=0.717590 moved_to_echo=20310 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3956, 0, 0, 0, 0, 847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3313, 0, 0, 0, 0, 0, 322, 0, 0, 0, 0, 0, 0, 0, 0, 1395, 143, 114, 2890, 5, 22, 0, 962, 404, 0, 0, 0, 0, 0, 0, 5, 1, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 2, 4095, 595, 49, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4082, 535, 0, 0, 741, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1016, 0, 0, 0, 0, 3855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 432, 771, 0, 4, 0, 0, 2101, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4803, 3635, 5935, 87, 4755, 5361, 4875, 3317] avg=4096.0000 max=5935 max_rank=2 min=87 min_rank=3 imbalance_ratio=1.448975 max_load_reduction=26833 max_load_reduction_pct=0.818878 moved_to_echo=18308 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 84, 514, 0, 0, 44, 588, 0, 0, 0, 0, 0, 0, 0, 10, 0, 3561, 0, 0, 0, 0, 888, 224, 0, 0, 0, 0, 0, 0, 0, 0, 2418, 74, 94, 4, 0, 0, 0, 617, 280, 0, 0, 0, 0, 0, 0, 0, 579, 0, 0, 3, 3808, 0, 6, 56, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3831, 3708, 414, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 1763, 0, 4052, 0, 0, 0, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 654, 14, 0, 0, 0, 2567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4, 0, 0, 0, 1, 5, 1349, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1230, 4683, 3487, 4452, 7955, 6358, 3235, 1368] avg=4096.0000 max=7955 max_rank=4 min=1230 min_rank=0 imbalance_ratio=1.942139 max_load_reduction=24813 max_load_reduction_pct=0.757233 moved_to_echo=18916 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 24, 20, 3, 0, 0, 0, 1161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3990, 1, 0, 7, 7, 0, 446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 0, 87, 0, 2, 0, 970, 556, 0, 0, 0, 0, 0, 0, 3099, 3680, 0, 1087, 0, 0, 0, 28, 115, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 3, 1, 1019, 0, 0, 0, 0, 0, 0, 0, 9, 0, 1012, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 7, 52, 0, 0, 3, 0, 1, 4065, 2114, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1208, 4451, 1861, 8009, 802, 5088, 5107, 6242] avg=4096.0000 max=8009 max_rank=3 min=802 min_rank=4 imbalance_ratio=1.955322 max_load_reduction=24759 max_load_reduction_pct=0.755585 moved_to_echo=17239 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[768, 4864, 512, 896, 128, 128, 1408, 10112, 14720, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 13952, 512, 4736, 3072, 384, 256, 0, 6784, 3584, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[384, 128, 7808, 15360, 640, 128, 0, 1664, 7296, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 7808, 18944, 5120, 128, 1024, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3456, 8960, 9856, 1152, 128, 8448, 256, 384, 768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 1792, 384, 22272, 1280, 128, 1024, 6400, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[256, 128, 7168, 128, 128, 4096, 256, 21376, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 1408, 22656, 2432, 128, 128, 2432, 1408, 2816, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1297, 3627, 0, 0, 0, 33, 0, 1130, 0, 30, 0, 1, 0, 2360, 987, 0, 0, 0, 0, 969, 0, 0, 0, 0, 4, 0, 374, 346, 0, 2714, 0, 0, 1413, 0, 0, 0, 0, 4054, 0, 0, 0, 10, 2978, 10, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4061, 1, 0, 76, 446, 1978, 0, 0, 3863, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[3, 40, 0, 0, 0, 0, 35, 2254, 0, 633, 0, 0, 0, 1, 0, 0, 484, 0, 0, 119, 19, 0, 0, 0, 219, 8, 3874, 3850, 0, 4032, 0, 0, 525, 0, 0, 0, 0, 1, 0, 0, 0, 256, 3047, 3819, 0, 0, 0, 0, 0, 0, 0, 1, 0, 11, 0, 0, 46, 105, 3388, 0, 1934, 4064, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 5, 0, 35, 2, 0, 2759, 0, 0, 0, 339, 0, 531, 0, 0, 0, 0, 0, 4079, 29, 0, 1, 87, 0, 3859, 18, 1929, 0, 0, 3344, 0, 0, 0, 8, 925, 0, 0, 0, 0, 2741, 0, 0, 0, 39, 0, 0, 0, 0, 3673, 0, 0, 0, 0, 171, 3361, 140, 0, 0, 4085, 0, 604] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[222, 0, 1, 0, 0, 3842, 1488, 4, 0, 0, 3, 29, 0, 0, 0, 23, 1, 0, 0, 0, 0, 2655, 292, 0, 4085, 0, 0, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3778, 307, 0, 0, 562, 3320, 249, 0, 0, 0, 0, 0, 0, 2989, 2, 214, 1545, 3, 0, 0, 938, 4013, 0, 1, 2034, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[19, 2854, 0, 0, 0, 0, 386, 0, 0, 2, 0, 0, 40, 0, 186, 0, 0, 2, 0, 1580, 0, 3895, 0, 0, 0, 0, 350, 4057, 1, 3879, 68, 0, 4091, 0, 52, 0, 1795, 3225, 0, 0, 468, 0, 1906, 0, 3, 7, 0, 0, 0, 0, 28, 0, 0, 1, 0, 1, 9, 193, 257, 22, 0, 3391, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[126, 4055, 11, 691, 0, 161, 459, 0, 34, 0, 18, 0, 5, 0, 1, 0, 756, 11, 8, 0, 49, 203, 0, 0, 2654, 0, 0, 3806, 0, 4035, 0, 191, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 20, 12, 260, 0, 0, 0, 0, 0, 0, 107, 0, 6, 1, 5, 1, 37, 3589, 3900, 0, 159, 3300, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 2, 0, 17, 0, 0, 0, 0, 0, 0, 1143, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 2860, 0, 2594, 3614, 0, 2769, 7, 1, 0, 3497, 0, 0, 0, 0, 4, 1, 273, 1, 3994, 21, 0, 302, 1, 568, 0, 1, 0, 0, 1, 1911, 6, 0, 1, 0, 4094, 0, 1025, 0, 3, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 693, 0, 0, 3, 0, 140, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 4, 235, 125, 0, 0, 0, 0, 4030, 1, 2, 0, 0, 4036, 0, 0, 0, 4096, 3513, 0, 0, 0, 0, 2564, 0, 296, 4, 0, 0, 10, 0, 4043, 1, 0, 0, 690, 0, 12, 0, 0, 0, 434, 3766, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3, 40, 0, 0, 0, 0, 35, 2254, 268, 29, 0, 0, 0, 0, 0, 0, 0, 633, 0, 0, 0, 1, 0, 0, 3179, 0, 0, 0, 0, 0, 0, 0, 484, 0, 0, 119, 19, 0, 0, 0, 2186, 0, 0, 0, 0, 0, 0, 0, 219, 8, 3874, 0, 0, 3202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 226, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 3047, 3819, 0, 0, 0, 0, 830, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 11, 0, 0, 1544, 671, 0, 0, 0, 0, 0, 0, 46, 105, 3388, 0, 1934, 305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2629, 3813, 2808, 7303, 227, 7983, 2227, 5778] avg=4096.0000 max=7983 max_rank=5 min=227 min_rank=4 imbalance_ratio=1.948975 max_load_reduction=24785 max_load_reduction_pct=0.756378 moved_to_echo=16215 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[222, 0, 1, 0, 0, 3842, 1488, 4, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 3, 29, 0, 0, 0, 23, 138, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2655, 292, 0, 1094, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3778, 307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 562, 3320, 249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2989, 2, 214, 1545, 3, 0, 773, 30, 0, 0, 0, 0, 0, 0, 0, 938, 4013, 0, 1, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5571, 193, 4042, 4085, 4085, 4131, 5556, 5105] avg=4096.0000 max=5571 max_rank=0 min=193 min_rank=1 imbalance_ratio=1.360107 max_load_reduction=27197 max_load_reduction_pct=0.829987 moved_to_echo=18877 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 5, 0, 35, 2, 1709, 29, 0, 0, 0, 0, 0, 0, 0, 2759, 0, 0, 0, 339, 0, 531, 3186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 29, 0, 2197, 0, 0, 0, 0, 0, 0, 0, 1, 87, 0, 0, 18, 1531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1436, 0, 0, 0, 8, 925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2741, 0, 0, 0, 39, 0, 398, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3673, 0, 0, 0, 0, 1552, 673, 0, 0, 0, 0, 0, 0, 171, 3361, 140, 0, 0, 307, 0, 604, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1784, 6815, 6305, 1637, 2369, 3377, 5898, 4583] avg=4096.0000 max=6815 max_rank=1 min=1637 min_rank=3 imbalance_ratio=1.663818 max_load_reduction=25953 max_load_reduction_pct=0.792023 moved_to_echo=16227 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[126, 4055, 11, 691, 0, 161, 459, 0, 2094, 23, 0, 0, 0, 0, 0, 0, 34, 0, 18, 0, 5, 0, 1, 0, 3141, 0, 0, 0, 0, 0, 0, 0, 756, 11, 8, 0, 49, 203, 0, 0, 1775, 0, 0, 0, 0, 0, 0, 0, 2654, 0, 0, 0, 0, 3204, 0, 191, 0, 0, 0, 0, 0, 0, 0, 0, 1758, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 12, 260, 0, 0, 0, 0, 831, 244, 0, 0, 0, 0, 0, 0, 0, 0, 107, 0, 6, 1, 5, 1, 1254, 665, 0, 0, 0, 0, 0, 0, 37, 3589, 3900, 0, 159, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7620, 3199, 2802, 6049, 1759, 1367, 2039, 7933] avg=4096.0000 max=7933 max_rank=7 min=1367 min_rank=5 imbalance_ratio=1.936768 max_load_reduction=24835 max_load_reduction_pct=0.757904 moved_to_echo=13098 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1297, 3627, 0, 0, 0, 33, 0, 1130, 722, 28, 0, 0, 0, 0, 0, 0, 0, 30, 0, 1, 0, 2360, 987, 0, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 969, 0, 0, 0, 0, 2078, 0, 0, 0, 0, 0, 0, 0, 4, 0, 374, 0, 0, 2155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 607, 0, 0, 0, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 2978, 10, 0, 0, 4, 0, 559, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 1, 0, 1468, 61, 0, 0, 0, 0, 0, 0, 76, 446, 1978, 0, 0, 289, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6837, 3663, 3047, 2533, 4661, 3645, 5591, 2791] avg=4096.0000 max=6837 max_rank=0 min=2533 min_rank=3 imbalance_ratio=1.669189 max_load_reduction=25931 max_load_reduction_pct=0.791351 moved_to_echo=16688 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[19, 2854, 0, 0, 0, 0, 386, 0, 2091, 24, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 40, 0, 186, 0, 3350, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1580, 0, 3895, 0, 0, 1824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 350, 0, 1, 3080, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1757, 0, 52, 0, 1795, 3225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 0, 1906, 0, 3, 7, 0, 0, 799, 243, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 1, 0, 1, 1289, 707, 0, 0, 0, 0, 0, 0, 9, 193, 257, 22, 0, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5374, 3578, 7301, 3499, 6829, 3426, 2026, 735] avg=4096.0000 max=7301 max_rank=2 min=735 min_rank=7 imbalance_ratio=1.782471 max_load_reduction=25467 max_load_reduction_pct=0.777191 moved_to_echo=13016 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 2, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1143, 4052, 2984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2860, 0, 2594, 0, 0, 2199, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3497, 0, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 273, 1, 3994, 21, 0, 302, 1, 568, 570, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1911, 6, 0, 0, 630, 0, 0, 0, 0, 0, 0, 1, 0, 4094, 0, 1025, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20, 8179, 0, 7661, 3502, 5730, 2549, 5127] avg=4096.0000 max=8179 max_rank=1 min=0 min_rank=2 imbalance_ratio=1.996826 max_load_reduction=24589 max_load_reduction_pct=0.750397 moved_to_echo=16908 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 693, 0, 0, 3, 0, 140, 0, 2066, 31, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 4, 235, 125, 0, 2030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1727, 0, 0, 0, 4096, 3513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2564, 0, 296, 4, 0, 0, 2, 243, 0, 0, 0, 0, 0, 0, 10, 0, 4043, 1, 0, 0, 690, 0, 1435, 703, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 434, 270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2933, 3328, 6463, 1, 9336, 3109, 6882, 716] avg=4096.0000 max=9336 max_rank=4 min=1 min_rank=3 imbalance_ratio=2.279297 max_load_reduction=23432 max_load_reduction_pct=0.715088 moved_to_echo=20043 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 7168, 3712, 256, 7552, 768, 128, 9344, 4224, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[1792, 11392, 128, 768, 128, 4096, 2560, 3456, 8960, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[384, 8704, 17792, 128, 3584, 1920, 128, 640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[7552, 3584, 128, 0, 6016, 15616, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 3456, 128, 128, 128, 2816, 2432, 4608, 19712, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[768, 896, 20608, 4480, 384, 384, 128, 640, 4096, 1152, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1280, 128, 128, 6784, 128, 11136, 512, 0, 13184, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[9856, 128, 7296, 0, 128, 15488, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 25, 3840, 0, 4071, 6, 3501, 33, 0, 0, 0, 1, 67, 0, 0, 0, 0, 1719, 0, 0, 0, 0, 0, 482, 7, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3448, 1832, 2022, 0, 16, 2491, 0, 0, 0, 0, 28, 74, 0, 0, 2547, 0, 0, 0, 68, 260, 0, 1516, 0, 354, 0, 0, 5, 1, 0, 258] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 5, 1421, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4051, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 57, 2, 0, 4088, 0, 0, 9, 4078, 0, 4052, 0, 48, 0, 0, 0, 0, 0, 6, 0, 36, 1, 4084, 0, 0, 0, 0, 0, 3398, 0, 0, 0, 150, 0, 3176, 0, 0, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 3910, 813, 0, 3555, 4019, 8, 0, 275, 0, 0, 0, 0, 55, 0, 55, 3334, 0, 2, 0, 0, 0, 75, 0, 13, 0, 0, 296, 0, 3874, 0, 5, 0, 3313, 364, 67, 230, 5, 153, 0, 0, 6, 0, 131, 10, 0, 0, 0, 482, 0, 0, 0, 0, 0, 0, 4093, 0, 299, 0, 2, 3324, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 3605, 0, 0, 588, 2006, 0, 0, 0, 10, 0, 0, 0, 0, 0, 15, 4076, 0, 0, 11, 147, 0, 20, 4, 0, 0, 0, 0, 0, 4095, 1, 0, 4062, 0, 0, 4030, 0, 34, 65, 0, 0, 0, 0, 0, 1, 0, 0, 3801, 0, 0, 0, 0, 0, 39, 1, 213, 0, 0, 501, 4092, 13, 1338, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[2, 0, 4045, 3224, 0, 1775, 170, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 3591, 0, 0, 62, 0, 0, 0, 0, 0, 0, 15, 0, 0, 140, 0, 783, 607, 4085, 0, 4089, 0, 22, 0, 12, 0, 0, 0, 242, 0, 0, 1, 4094, 0, 0, 0, 2, 0, 0, 2, 2311, 1, 10, 168, 2799, 471, 16, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[3, 0, 0, 8, 2941, 3186, 0, 255, 0, 0, 7, 4, 913, 0, 321, 0, 0, 1287, 1, 53, 0, 0, 32, 3287, 0, 0, 1, 0, 760, 101, 3799, 31, 5, 796, 3025, 0, 2189, 0, 15, 1, 0, 0, 91, 0, 5, 691, 0, 0, 3456, 633, 56, 0, 0, 0, 3716, 1091, 0, 2, 6, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 2, 0, 65, 0, 3504, 0, 0, 0, 0, 7, 0, 0, 2, 0, 0, 1, 4095, 0, 951, 0, 2, 0, 0, 3955, 0, 0, 0, 0, 0, 840, 4, 0, 3, 4051, 0, 4096, 0, 0, 7, 0, 0, 0, 0, 0, 1, 0, 0, 4039, 0, 0, 178, 0, 0, 0, 6, 0, 0, 0, 0, 2695, 759, 3505, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8, 3957, 579, 758, 0, 0, 0, 1, 0, 0, 0, 4, 0, 4040, 0, 0, 37, 0, 51, 3197, 0, 0, 0, 0, 486, 23, 506, 46, 0, 0, 3503, 0, 3281, 0, 0, 0, 0, 0, 0, 14, 204, 0, 0, 0, 3049, 1362, 0, 142, 10, 0, 3947, 0, 25, 0, 0, 0, 3532, 5, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 25, 2209, 0, 4071, 0, 3501, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 67, 0, 0, 0, 3, 935, 1631, 291, 0, 0, 0, 0, 0, 1327, 0, 0, 0, 0, 0, 482, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4096, 0, 0, 221, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1832, 1801, 0, 16, 2491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 74, 0, 0, 3448, 3, 0, 0, 0, 0, 0, 0, 1612, 0, 0, 0, 68, 260, 0, 1516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 354, 0, 0, 5, 1, 0, 258, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9839, 2928, 1809, 4425, 6140, 3553, 3456, 618] avg=4096.0000 max=9839 max_rank=0 min=618 min_rank=7 imbalance_ratio=2.402100 max_load_reduction=22929 max_load_reduction_pct=0.699738 moved_to_echo=13767 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 3, 1500, 0, 687, 0, 0, 0, 0, 0, 3124, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 2, 0, 4088, 0, 444, 240, 265, 0, 0, 0, 0, 0, 0, 9, 0, 0, 3608, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 36, 1, 4078, 2, 0, 0, 0, 0, 0, 0, 2584, 0, 0, 0, 0, 0, 3398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 0, 2911, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1421, 2199, 7213, 5096, 3665, 4123, 5982, 3069] avg=4096.0000 max=7213 max_rank=2 min=1421 min_rank=0 imbalance_ratio=1.760986 max_load_reduction=25555 max_load_reduction_pct=0.779877 moved_to_echo=16839 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 8, 2941, 0, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4, 913, 0, 321, 0, 2088, 1269, 0, 218, 0, 0, 0, 0, 0, 993, 1, 53, 0, 0, 32, 3287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 760, 101, 3799, 31, 239, 76, 0, 0, 0, 0, 0, 0, 5, 796, 0, 0, 1950, 0, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 5, 691, 0, 0, 3025, 1098, 0, 0, 0, 0, 0, 0, 2187, 633, 56, 0, 0, 0, 3716, 1091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3207, 4820, 4366, 5007, 2767, 4910, 7683, 8] avg=4096.0000 max=7683 max_rank=6 min=8 min_rank=7 imbalance_ratio=1.875732 max_load_reduction=25085 max_load_reduction_pct=0.765533 moved_to_echo=15368 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2071, 0, 0, 0, 2006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 386, 1396, 1534, 691, 0, 0, 0, 0, 15, 3144, 0, 0, 11, 147, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4095, 1, 441, 241, 342, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 3589, 0, 34, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 202, 0, 0, 0, 0, 0, 0, 2405, 0, 0, 0, 0, 0, 39, 1, 0, 0, 0, 0, 0, 0, 0, 0, 213, 0, 0, 501, 3750, 13, 1338, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4017, 3337, 5124, 7750, 203, 2445, 5815] avg=4096.0000 max=7750 max_rank=4 min=203 min_rank=5 imbalance_ratio=1.892090 max_load_reduction=25018 max_load_reduction_pct=0.763489 moved_to_echo=16213 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2249, 813, 0, 0, 4019, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 275, 0, 0, 0, 0, 55, 0, 2331, 0, 1661, 565, 0, 0, 0, 0, 55, 2572, 0, 2, 0, 0, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 296, 0, 3874, 0, 7, 197, 0, 0, 0, 0, 0, 0, 5, 0, 0, 364, 60, 230, 5, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 131, 10, 0, 0, 3313, 1224, 0, 0, 0, 0, 0, 0, 0, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 299, 0, 2, 3324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7089, 4887, 2704, 4387, 817, 4684, 482, 7718] avg=4096.0000 max=7718 max_rank=7 min=482 min_rank=6 imbalance_ratio=1.884277 max_load_reduction=25050 max_load_reduction_pct=0.764465 moved_to_echo=13701 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 2327, 3224, 0, 0, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 1164, 1504, 1718, 609, 0, 0, 0, 0, 0, 2770, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 140, 0, 448, 212, 234, 0, 0, 0, 0, 0, 783, 607, 0, 0, 3641, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 242, 0, 0, 1, 4085, 611, 0, 0, 0, 0, 0, 0, 2590, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2311, 1, 10, 168, 2565, 471, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5723, 5024, 2832, 1049, 5053, 4951, 2594, 5542] avg=4096.0000 max=5723 max_rank=0 min=1049 min_rank=3 imbalance_ratio=1.397217 max_load_reduction=27045 max_load_reduction_pct=0.825348 moved_to_echo=18140 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8, 0, 579, 758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 2593, 1120, 0, 685, 0, 0, 0, 0, 0, 3116, 0, 0, 37, 0, 51, 3197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 23, 506, 46, 359, 239, 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 204, 0, 0, 0, 3503, 1364, 0, 0, 0, 0, 0, 0, 1929, 1362, 0, 142, 10, 0, 3947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 3237, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1345, 4403, 6401, 1954, 2922, 5085, 7390, 3268] avg=4096.0000 max=7390 max_rank=6 min=1345 min_rank=0 imbalance_ratio=1.804199 max_load_reduction=25378 max_load_reduction_pct=0.774475 moved_to_echo=18665 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 2, 0, 0, 2297, 1487, 0, 697, 0, 0, 0, 0, 1, 3152, 0, 951, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3955, 0, 0, 0, 0, 0, 840, 4, 452, 246, 229, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3644, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4051, 1207, 0, 0, 0, 0, 0, 0, 2552, 0, 0, 178, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2466, 759, 3505, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[67, 4490, 4106, 5726, 3654, 5259, 2736, 6730] avg=4096.0000 max=6730 max_rank=7 min=67 min_rank=0 imbalance_ratio=1.643066 max_load_reduction=26038 max_load_reduction_pct=0.794617 moved_to_echo=18379 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 8960, 4224, 7040, 0, 11776, 1152, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[0, 384, 128, 128, 1024, 128, 512, 128, 10880, 9216, 6656, 4480, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[6656, 384, 512, 768, 14976, 4608, 4864, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 20224, 128, 1024, 128, 256, 128, 11264, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[3968, 128, 128, 128, 1664, 4224, 17408, 128, 2688, 1664, 1408, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[15872, 2560, 128, 384, 128, 384, 11136, 2688, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[896, 5504, 0, 2304, 21248, 256, 256, 2816, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 640, 896, 128, 128, 25600, 5760, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 7, 0, 4034, 3682, 0, 13, 474, 0, 3, 0, 4036, 0, 0, 0, 0, 14, 0, 35, 0, 0, 0, 0, 0, 0, 0, 1, 3884, 1078, 0, 0, 194, 100, 0, 12, 4074, 0, 8, 17, 4079, 0, 0, 20, 0, 0, 0, 3921, 0, 295, 0, 2786, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3415, 3891, 2681, 1, 889, 0, 0, 0, 218, 0, 0, 0, 868, 9, 59, 0, 0, 1340, 552, 0, 5, 1, 0, 0, 51, 0, 52, 1, 0, 3264, 4078, 48, 364, 0, 0, 0, 4072, 0, 1049, 0, 0, 5, 0, 0, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2740, 9, 0, 2938] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4, 4051, 2231, 83, 330, 0, 0, 0, 0, 0, 4, 117, 3282, 0, 0, 61, 0, 0, 0, 0, 0, 3, 2, 0, 0, 4096, 627, 0, 0, 0, 3958, 0, 3676, 0, 0, 0, 3941, 1, 0, 0, 32, 0, 0, 4, 0, 0, 217, 439, 0, 0, 0, 0, 1, 9, 0, 0, 4014, 30, 1554, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 50, 1, 0, 244, 2162, 0, 4094, 0, 87, 0, 4096, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 246, 0, 0, 0, 0, 4045, 0, 0, 0, 1002, 14, 0, 2910, 0, 0, 0, 3929, 3083, 0, 0, 30, 0, 0, 83, 110, 3377, 0, 1, 24, 145, 0, 0, 12, 26, 10, 7, 462, 2514, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 1119, 4079, 4040, 0, 119, 0, 0, 4, 0, 42, 2, 4089, 0, 46, 0, 184, 0, 4049, 0, 0, 0, 37, 0, 1, 0, 0, 3195, 0, 0, 0, 1, 0, 2482, 1475, 58, 0, 0, 92, 0, 0, 829, 0, 0, 0, 3961, 0, 2557, 0, 0, 0, 0, 0, 0, 40, 232, 0, 0, 0, 28, 0, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 8, 13, 1220, 3355, 0, 37, 3676, 0, 15, 0, 0, 2, 0, 0, 0, 0, 215, 0, 0, 6, 93, 0, 9, 116, 0, 0, 0, 0, 36, 0, 0, 26, 335, 3957, 0, 3428, 0, 2048, 2590, 4096, 0, 0, 0, 0, 0, 0, 129, 0, 149, 0, 68, 0, 0, 0, 3071, 0, 0, 0, 0, 4070, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 1, 0, 1372, 3907, 941, 4079, 35, 0, 0, 0, 11, 3802, 0, 0, 0, 4091, 0, 4, 0, 0, 0, 10, 11, 0, 0, 635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2794, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 2846, 7, 1, 0, 0, 0, 0, 46, 6, 0, 0, 0, 0, 4079, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 82, 50, 810, 0, 0, 0, 0, 1, 95, 908, 2, 0, 3599, 0, 1, 0, 3463, 0, 4, 30, 0, 0, 0, 0, 0, 0, 0, 24, 3012, 0, 1, 0, 4096, 0, 0, 0, 4013, 0, 0, 0, 0, 0, 0, 1412, 2328, 631, 0, 0, 0, 0, 18, 4090, 0, 0, 0, 0, 4045, 0, 51, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 7, 0, 0, 2843, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 474, 0, 3, 0, 4036, 0, 0, 0, 1361, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 35, 0, 0, 0, 0, 1654, 211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3884, 1078, 0, 0, 3127, 0, 0, 0, 0, 0, 0, 0, 194, 100, 0, 12, 994, 0, 8, 17, 0, 0, 0, 0, 0, 0, 0, 0, 2214, 0, 0, 20, 0, 0, 0, 3921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 295, 0, 2786, 0, 0, 0, 0, 1426, 907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 839, 293, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2863, 5874, 1914, 8090, 1325, 6155, 5414, 1133] avg=4096.0000 max=8090 max_rank=3 min=1133 min_rank=7 imbalance_ratio=1.975098 max_load_reduction=24678 max_load_reduction_pct=0.753113 moved_to_echo=14027 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 3128, 2231, 83, 0, 0, 0, 0, 0, 0, 0, 0, 330, 0, 0, 0, 0, 0, 4, 117, 1315, 0, 0, 0, 0, 0, 0, 0, 3282, 0, 0, 61, 0, 0, 0, 0, 1492, 203, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 4096, 627, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3958, 0, 897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2140, 1, 0, 0, 32, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 217, 439, 0, 0, 0, 0, 1287, 1, 0, 0, 0, 0, 0, 0, 1, 9, 0, 0, 4014, 30, 1554, 0, 923, 283, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5443, 1766, 5038, 4731, 4855, 2177, 1944, 6814] avg=4096.0000 max=6814 max_rank=7 min=1766 min_rank=1 imbalance_ratio=1.663574 max_load_reduction=25954 max_load_reduction_pct=0.792053 moved_to_echo=15790 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1119, 0, 3120, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 42, 2, 4089, 0, 30, 0, 0, 0, 0, 0, 0, 0, 46, 0, 184, 0, 4049, 0, 0, 0, 598, 4, 0, 0, 0, 0, 0, 0, 37, 0, 1, 0, 0, 3195, 0, 0, 3161, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2482, 361, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 829, 0, 0, 0, 3961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2557, 0, 0, 0, 0, 0, 0, 516, 918, 0, 0, 0, 0, 0, 0, 40, 232, 0, 0, 0, 28, 0, 7, 920, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4358, 4167, 4881, 6394, 2902, 4842, 3991, 1233] avg=4096.0000 max=6394 max_rank=3 min=1233 min_rank=7 imbalance_ratio=1.561035 max_load_reduction=26374 max_load_reduction_pct=0.804871 moved_to_echo=12968 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3005, 2681, 1, 0, 0, 0, 0, 0, 0, 0, 0, 889, 0, 0, 0, 218, 0, 0, 0, 1358, 0, 0, 0, 0, 0, 0, 0, 868, 9, 59, 0, 0, 1340, 552, 0, 147, 210, 0, 0, 0, 0, 0, 0, 5, 1, 0, 0, 51, 0, 52, 1, 2648, 0, 0, 0, 0, 0, 0, 0, 0, 3264, 4078, 48, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2212, 0, 1049, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 0, 0, 0, 0, 0, 0, 0, 127, 767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2740, 9, 0, 2938, 886, 292, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5687, 2465, 3185, 2758, 7480, 3266, 1062, 6865] avg=4096.0000 max=7480 max_rank=4 min=1062 min_rank=6 imbalance_ratio=1.826172 max_load_reduction=25288 max_load_reduction_pct=0.771729 moved_to_echo=18673 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 50, 1, 0, 189, 2162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 87, 0, 4096, 0, 0, 0, 1311, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 1181, 203, 0, 0, 0, 0, 0, 0, 246, 0, 0, 0, 0, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 14, 0, 710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2133, 3083, 0, 0, 30, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 110, 3377, 0, 1, 24, 145, 0, 0, 1019, 0, 0, 0, 0, 0, 0, 0, 12, 26, 10, 7, 462, 2514, 0, 0, 55, 282, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2402, 9588, 1388, 4291, 1726, 5329, 4676, 3368] avg=4096.0000 max=9588 max_rank=1 min=1388 min_rank=2 imbalance_ratio=2.340820 max_load_reduction=23180 max_load_reduction_pct=0.707397 moved_to_echo=15099 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 1372, 0, 727, 4079, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 3802, 0, 0, 0, 1364, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4, 0, 0, 0, 10, 11, 1134, 211, 0, 0, 0, 0, 0, 0, 0, 0, 635, 0, 0, 0, 0, 0, 3029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 682, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2221, 0, 0, 0, 0, 0, 0, 2846, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1, 0, 0, 0, 0, 46, 6, 978, 878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 214, 294, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6214, 5177, 5461, 3664, 682, 5067, 1916, 4587] avg=4096.0000 max=6214 max_rank=0 min=682 min_rank=4 imbalance_ratio=1.517090 max_load_reduction=26554 max_load_reduction_pct=0.810364 moved_to_echo=12252 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 13, 1220, 0, 0, 37, 3676, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 2, 0, 0, 0, 1366, 0, 0, 0, 0, 0, 0, 0, 0, 215, 0, 0, 6, 93, 0, 9, 1391, 211, 0, 0, 0, 0, 0, 0, 116, 0, 0, 0, 0, 36, 0, 0, 2600, 0, 0, 0, 0, 0, 0, 0, 26, 335, 3957, 0, 837, 0, 2048, 2590, 0, 0, 0, 0, 0, 0, 0, 0, 2225, 0, 0, 0, 0, 0, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 0, 68, 0, 0, 0, 3071, 1200, 755, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 294, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4954, 1383, 1925, 2752, 9793, 2354, 5243, 4364] avg=4096.0000 max=9793 max_rank=4 min=1383 min_rank=1 imbalance_ratio=2.390869 max_load_reduction=22975 max_load_reduction_pct=0.701141 moved_to_echo=21754 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 0, 35, 810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 95, 908, 2, 0, 1342, 0, 0, 0, 0, 0, 0, 0, 3599, 0, 1, 0, 3463, 0, 4, 30, 1667, 212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 64, 0, 0, 0, 0, 0, 0, 0, 3012, 0, 1, 0, 992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2166, 0, 0, 0, 0, 0, 0, 1412, 0, 0, 0, 0, 0, 0, 0, 0, 2328, 631, 0, 0, 0, 0, 18, 4090, 1437, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 51, 0, 15, 293, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[847, 2348, 8976, 88, 4005, 3578, 8522, 4404] avg=4096.0000 max=8976 max_rank=2 min=88 min_rank=3 imbalance_ratio=2.191406 max_load_reduction=23792 max_load_reduction_pct=0.726074 moved_to_echo=20509 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3328, 4736, 12032, 2560, 5632, 128, 2176, 2688, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[2688, 7040, 256, 3328, 128, 256, 128, 7168, 8064, 4352, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[11904, 256, 256, 128, 7552, 1536, 640, 128, 9344, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[5888, 128, 128, 128, 12416, 1024, 4096, 128, 9472, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 3840, 0, 13056, 12032, 3968, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[15488, 3200, 1152, 896, 128, 128, 0, 12416, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[512, 128, 640, 128, 3968, 12544, 768, 128, 14720, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 384, 128, 128, 19456, 2688, 1664, 3072, 3968, 2048, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[3076, 0, 0, 0, 0, 0, 1, 162, 0, 4037, 0, 1289, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 1695, 3211, 624, 1, 89, 0, 4089, 0, 0, 3854, 0, 0, 0, 104, 3, 1188, 292, 0, 70, 70, 0, 0, 0, 70, 4096, 0, 0, 0, 0, 0, 0, 581, 0, 47, 4095, 0, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[78, 0, 0, 0, 0, 0, 0, 1242, 0, 0, 88, 0, 4096, 253, 0, 0, 0, 0, 0, 7, 0, 1525, 0, 0, 3, 0, 253, 0, 0, 43, 30, 282, 0, 0, 0, 2, 4096, 2602, 0, 0, 0, 1, 0, 1201, 0, 10, 315, 0, 0, 3944, 0, 0, 0, 0, 4075, 0, 0, 1545, 3097, 3980, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[2995, 0, 4037, 0, 4, 2, 0, 27, 308, 4088, 45, 0, 0, 0, 6, 0, 1953, 105, 0, 0, 0, 0, 0, 0, 1218, 366, 3660, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 1402, 0, 7, 111, 0, 0, 11, 3612, 0, 0, 79, 3717, 3347, 0, 2, 27, 0, 281, 930, 56, 111, 231, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 1, 1188, 0, 457, 0, 3361, 0, 0, 542, 33, 0, 0, 0, 1, 0, 7, 0, 3400, 2613, 0, 0, 0, 0, 3084, 0, 168, 0, 0, 0, 2391, 0, 2499, 0, 0, 0, 0, 3912, 21, 0, 0, 5, 0, 1452, 289, 0, 0, 0, 23, 0, 0, 0, 3318, 4002, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 548, 216, 0, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 3840, 0, 0, 4066, 0, 1114, 0, 0, 0, 2, 0, 3, 0, 0, 656, 1017, 4062, 0, 862, 0, 0, 4, 4095, 3997, 0, 0, 2035, 0, 0, 137, 1, 0, 0, 0, 0, 405, 0, 1760, 3539, 8, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 1991, 0, 0, 0, 1, 0, 0, 1337, 0, 355, 0, 693, 0, 0, 0, 0, 0, 50, 0, 0, 4095, 0, 0, 3639, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 1832, 0, 1, 0, 0, 0, 276, 3738, 0, 353, 677, 365, 130, 301, 0, 0, 0, 1338, 14, 3412, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[223, 74, 25, 1616, 967, 0, 450, 998, 0, 0, 440, 0, 2, 0, 0, 0, 174, 0, 0, 2, 0, 4089, 1, 24, 3584, 0, 4066, 75, 57, 0, 0, 0, 0, 4095, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 321, 0, 4001, 0, 436, 0, 0, 0, 0, 0, 0, 2914, 3, 4096, 9, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[6, 1483, 180, 0, 0, 0, 19, 0, 0, 0, 725, 0, 163, 666, 0, 4084, 2, 0, 0, 77, 0, 241, 0, 163, 3499, 0, 3822, 0, 11, 0, 4027, 1, 1340, 5, 434, 5, 574, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 216, 0, 3069, 254, 0, 0, 0, 12, 0, 0, 0, 0, 0, 3612, 3, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3076, 0, 0, 0, 0, 0, 1, 162, 1911, 386, 0, 0, 0, 0, 0, 0, 0, 4037, 0, 1289, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 1309, 0, 0, 0, 0, 0, 0, 0, 0, 3211, 523, 1, 89, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3854, 0, 0, 0, 104, 3, 1188, 0, 0, 0, 0, 0, 0, 0, 0, 292, 0, 70, 70, 0, 0, 0, 70, 101, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 2110, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5536, 5400, 1332, 7913, 5149, 603, 4677, 2158] avg=4096.0000 max=7913 max_rank=3 min=603 min_rank=5 imbalance_ratio=1.931885 max_load_reduction=24855 max_load_reduction_pct=0.758514 moved_to_echo=12587 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 1, 1548, 774, 0, 0, 0, 0, 0, 0, 1188, 0, 457, 0, 3361, 0, 0, 542, 839, 60, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 1, 0, 7, 0, 2626, 0, 0, 0, 0, 0, 0, 0, 0, 2613, 0, 0, 0, 0, 3084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 0, 0, 0, 1503, 0, 2499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3912, 21, 0, 0, 5, 0, 49, 4, 0, 0, 0, 0, 0, 0, 1448, 289, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1710, 4002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2324, 6447, 2667, 5697, 4170, 3991, 1760, 5712] avg=4096.0000 max=6447 max_rank=1 min=1760 min_rank=6 imbalance_ratio=1.573975 max_load_reduction=26321 max_load_reduction_pct=0.803253 moved_to_echo=15633 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[78, 0, 0, 0, 0, 0, 0, 1242, 1445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 4096, 253, 0, 0, 1437, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 1525, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 0, 0, 43, 30, 282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2575, 2602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1201, 0, 10, 315, 0, 41, 84, 13, 0, 0, 0, 0, 0, 0, 3931, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1545, 1596, 3980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2765, 5930, 1535, 567, 5179, 1665, 8006, 7121] avg=4096.0000 max=8006 max_rank=6 min=567 min_rank=3 imbalance_ratio=1.954590 max_load_reduction=24762 max_load_reduction_pct=0.755676 moved_to_echo=21971 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 548, 216, 0, 341, 821, 926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 356, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 3840, 0, 0, 3140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 933, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 656, 640, 4062, 0, 862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4095, 3997, 0, 0, 2035, 181, 21, 0, 0, 0, 0, 0, 0, 0, 0, 137, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 405, 0, 908, 3539, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2852, 387, 7040, 935, 6223, 10333, 138, 4860] avg=4096.0000 max=10333 max_rank=5 min=138 min_rank=6 imbalance_ratio=2.522705 max_load_reduction=22435 max_load_reduction_pct=0.684662 moved_to_echo=21554 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2995, 0, 4037, 0, 4, 2, 0, 27, 107, 278, 0, 0, 0, 0, 0, 0, 308, 4088, 45, 0, 0, 0, 6, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1953, 105, 0, 0, 0, 0, 0, 0, 940, 0, 0, 0, 0, 0, 0, 0, 0, 366, 3064, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1402, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 111, 0, 0, 11, 3612, 0, 0, 79, 596, 0, 11, 0, 0, 0, 0, 0, 3717, 3336, 0, 2, 27, 0, 281, 930, 0, 0, 0, 0, 0, 0, 0, 0, 56, 111, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7450, 4451, 2998, 3460, 1409, 4420, 8293, 287] avg=4096.0000 max=8293 max_rank=6 min=287 min_rank=7 imbalance_ratio=2.024658 max_load_reduction=24475 max_load_reduction_pct=0.746918 moved_to_echo=14409 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1991, 0, 0, 0, 1, 0, 0, 0, 829, 0, 0, 0, 0, 0, 0, 1337, 0, 355, 0, 693, 0, 0, 0, 1430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 4095, 0, 0, 2810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2562, 1832, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 276, 3738, 0, 353, 677, 667, 84, 0, 0, 0, 0, 0, 0, 365, 130, 301, 0, 0, 0, 1338, 14, 0, 0, 0, 0, 0, 0, 0, 0, 3412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2821, 3815, 6955, 3427, 4395, 5795, 2148, 3412] avg=4096.0000 max=6955 max_rank=2 min=2148 min_rank=6 imbalance_ratio=1.697998 max_load_reduction=25813 max_load_reduction_pct=0.787750 moved_to_echo=15750 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[6, 1483, 180, 0, 0, 0, 19, 0, 0, 798, 0, 0, 0, 0, 0, 0, 0, 0, 725, 0, 163, 666, 0, 4084, 203, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 77, 0, 241, 0, 163, 2701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3199, 0, 11, 0, 4027, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1340, 5, 434, 5, 358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 216, 623, 13, 10, 0, 0, 0, 0, 0, 0, 3059, 254, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3612, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2486, 5841, 3184, 7238, 2142, 4937, 3325, 3615] avg=4096.0000 max=7238 max_rank=3 min=2142 min_rank=4 imbalance_ratio=1.767090 max_load_reduction=25530 max_load_reduction_pct=0.779114 moved_to_echo=14019 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[223, 74, 25, 1616, 967, 0, 450, 998, 1364, 817, 0, 0, 0, 0, 0, 0, 0, 0, 440, 0, 2, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 174, 0, 0, 2, 0, 4089, 1, 24, 2767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3399, 75, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 321, 667, 0, 16, 0, 0, 0, 0, 0, 0, 3985, 0, 436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1495, 3, 4096, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6534, 497, 7057, 3531, 4101, 1024, 4421, 5603] avg=4096.0000 max=7057 max_rank=2 min=497 min_rank=1 imbalance_ratio=1.722900 max_load_reduction=25711 max_load_reduction_pct=0.784637 moved_to_echo=15149 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[6400, 3584, 4352, 1664, 1536, 256, 512, 2816, 7296, 4864, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 16000, 1024, 512, 128, 0, 5760, 1536, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1536, 8064, 512, 768, 7680, 10112, 2560, 2176, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 6272, 14848, 128, 256, 128, 11264, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[2944, 8192, 2176, 1408, 8320, 1024, 128, 4736, 4352, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[3968, 1792, 8064, 11648, 7808, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[2176, 128, 128, 256, 128, 13824, 128, 256, 16384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[512, 128, 128, 13696, 11392, 128, 768, 3456, 2944, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 9, 0, 0, 0, 159, 45, 4034, 0, 0, 1, 0, 0, 0, 42, 0, 0, 3799, 0, 0, 0, 3969, 4, 3622, 279, 0, 216, 4079, 0, 0, 0, 153, 0, 0, 0, 0, 3714, 0, 6, 0, 0, 3963, 0, 476, 0, 0, 0, 0, 0, 0, 0, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 3185, 3375, 0, 0, 145, 1, 2620, 1, 42, 505, 0, 0, 2, 3, 3799, 12, 0, 443, 0, 0, 281, 4060, 4086, 54, 0, 0, 2240, 0, 458, 0, 0, 0, 108, 0, 0, 0, 0, 0, 2, 8, 0, 582, 0, 0, 4077, 0, 0, 0, 1900, 0, 0, 0, 15, 1, 0, 719, 0, 0, 44, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[4056, 2029, 0, 265, 3653, 23, 0, 0, 0, 0, 122, 0, 0, 0, 476, 0, 0, 5, 0, 3728, 34, 0, 197, 0, 0, 0, 4096, 1, 0, 3, 4092, 3879, 0, 166, 0, 669, 0, 0, 0, 3864, 0, 91, 0, 13, 0, 0, 34, 7, 839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 426, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 26, 277, 3808, 674, 283, 0, 0, 0, 0, 1521, 31, 0, 6, 0, 0, 144, 1, 0, 0, 3809, 0, 0, 11, 115, 0, 4023, 0, 0, 2343, 3352, 455, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 4083, 0, 0, 1, 0, 68, 0, 3530, 0, 0, 0, 106, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[10, 1, 2, 2994, 182, 0, 544, 48, 284, 1135, 1, 0, 0, 6, 0, 0, 1, 0, 0, 4008, 0, 0, 0, 0, 237, 0, 3897, 0, 0, 0, 3771, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 3918, 0, 0, 6, 0, 796, 1, 0, 0, 0, 2678, 13, 0, 17, 30, 0, 0, 0, 10, 0, 3, 4091, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 266, 14, 4048, 0, 1354, 0, 0, 0, 0, 0, 5, 0, 36, 3400, 0, 45, 700, 0, 58, 4096, 0, 1912, 677, 0, 0, 845, 0, 63, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 8, 764, 0, 8, 3693, 0, 73, 0, 3519, 0, 126, 2682, 268, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[29, 0, 0, 3728, 0, 7, 0, 6, 14, 0, 1305, 30, 0, 293, 0, 0, 0, 1537, 0, 18, 3180, 0, 1093, 0, 1078, 2605, 2899, 0, 0, 0, 3657, 0, 0, 0, 0, 6, 0, 0, 0, 487, 0, 0, 0, 0, 1059, 1418, 37, 0, 3899, 145, 0, 0, 86, 0, 6, 50, 0, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[82, 0, 307, 0, 2465, 0, 0, 0, 0, 0, 0, 0, 264, 531, 1652, 0, 1, 0, 0, 0, 3412, 0, 16, 0, 21, 0, 665, 0, 26, 0, 0, 0, 0, 0, 2653, 0, 0, 0, 2954, 3673, 0, 56, 572, 0, 1376, 0, 1340, 0, 0, 0, 0, 0, 0, 1713, 0, 21, 0, 723, 4070, 0, 3166, 0, 1009, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2389, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 159, 45, 4034, 0, 0, 1, 0, 0, 1838, 1028, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 2931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3969, 4, 0, 279, 0, 216, 1428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 0, 0, 0, 0, 3714, 414, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 3963, 0, 476, 0, 454, 367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 1340, 813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2594, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2398, 7105, 2973, 5896, 4281, 5266, 2255, 2594] avg=4096.0000 max=7105 max_rank=1 min=2255 min_rank=6 imbalance_ratio=1.734619 max_load_reduction=25663 max_load_reduction_pct=0.783173 moved_to_echo=14396 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4056, 2029, 0, 156, 3653, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 0, 0, 0, 476, 0, 1844, 1162, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3728, 27, 0, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 1432, 3879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 166, 0, 669, 0, 0, 0, 3864, 3, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 13, 0, 0, 34, 7, 4, 23, 0, 0, 0, 0, 0, 0, 839, 0, 0, 0, 0, 0, 0, 0, 86, 816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 426, 0, 2934, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9917, 3604, 3957, 5315, 4702, 172, 1741, 3360] avg=4096.0000 max=9917 max_rank=0 min=172 min_rank=5 imbalance_ratio=2.421143 max_load_reduction=22851 max_load_reduction_pct=0.697357 moved_to_echo=9975 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1858, 3375, 0, 0, 145, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2620, 1, 42, 505, 0, 0, 2, 0, 15, 0, 0, 0, 0, 0, 0, 3, 3799, 12, 0, 342, 0, 0, 281, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 4086, 0, 0, 0, 2240, 0, 458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8, 0, 582, 0, 0, 4077, 53, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1900, 0, 0, 0, 15, 1042, 0, 0, 0, 0, 0, 0, 0, 1, 0, 719, 0, 0, 44, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5378, 3186, 4437, 10844, 156, 5007, 2957, 803] avg=4096.0000 max=10844 max_rank=3 min=156 min_rank=4 imbalance_ratio=2.647461 max_load_reduction=21924 max_load_reduction_pct=0.669067 moved_to_echo=8923 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[10, 1, 2, 1747, 182, 0, 544, 48, 0, 0, 0, 0, 0, 0, 0, 0, 284, 1135, 1, 0, 0, 6, 0, 0, 1699, 1107, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 237, 0, 0, 0, 0, 0, 1320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3918, 0, 0, 6, 0, 796, 1, 0, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2678, 13, 0, 17, 30, 979, 752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 3, 4091, 8, 2790, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2534, 4232, 4009, 1557, 4076, 4989, 4469, 6902] avg=4096.0000 max=6902 max_rank=7 min=1557 min_rank=3 imbalance_ratio=1.685059 max_load_reduction=25866 max_load_reduction_pct=0.789368 moved_to_echo=20436 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 26, 277, 2222, 674, 283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1521, 31, 0, 6, 0, 0, 1512, 1142, 0, 0, 0, 0, 0, 0, 144, 1, 0, 0, 2938, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 0, 0, 2343, 1170, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 415, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 1, 0, 68, 0, 456, 341, 0, 0, 0, 0, 0, 0, 3530, 0, 0, 0, 106, 1, 0, 0, 1245, 670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2881, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3482, 4212, 3094, 4083, 4515, 4949, 5552, 2881] avg=4096.0000 max=5552 max_rank=6 min=2881 min_rank=7 imbalance_ratio=1.355469 max_load_reduction=27216 max_load_reduction_pct=0.830566 moved_to_echo=17897 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[29, 0, 0, 2175, 0, 7, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 1305, 30, 0, 293, 0, 0, 1648, 822, 0, 0, 0, 0, 0, 0, 0, 1537, 0, 18, 2454, 0, 1093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1078, 2605, 0, 0, 0, 0, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 487, 346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1418, 37, 0, 380, 334, 0, 0, 0, 0, 0, 0, 3899, 145, 0, 0, 86, 0, 6, 50, 1219, 729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2077, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2217, 4112, 5102, 4963, 839, 3228, 6134, 6173] avg=4096.0000 max=6173 max_rank=7 min=839 min_rank=4 imbalance_ratio=1.507080 max_load_reduction=26595 max_load_reduction_pct=0.811615 moved_to_echo=16374 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[82, 0, 307, 0, 2465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 264, 531, 1652, 0, 0, 189, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2633, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2653, 0, 0, 0, 2954, 3673, 371, 0, 0, 0, 0, 0, 0, 0, 0, 56, 572, 0, 1376, 0, 1340, 0, 408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1713, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 723, 4070, 0, 3166, 0, 1009, 0, 476, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2854, 2636, 2650, 47, 9651, 3752, 1734, 9444] avg=4096.0000 max=9651 max_rank=4 min=47 min_rank=3 imbalance_ratio=2.356201 max_load_reduction=23117 max_load_reduction_pct=0.705475 moved_to_echo=24581 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 266, 14, 2354, 0, 1354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 36, 3400, 0, 0, 240, 0, 0, 0, 0, 0, 0, 45, 700, 0, 58, 3154, 0, 1912, 677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 449, 0, 0, 0, 0, 0, 0, 0, 8, 764, 0, 8, 3693, 0, 73, 0, 493, 366, 0, 0, 0, 0, 0, 0, 3519, 0, 126, 2682, 268, 3, 0, 0, 1328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 605, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3988, 3681, 6546, 63, 4548, 5405, 7926, 611] avg=4096.0000 max=7926 max_rank=6 min=63 min_rank=3 imbalance_ratio=1.935059 max_load_reduction=24842 max_load_reduction_pct=0.758118 moved_to_echo=18490 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[9600, 6784, 0, 384, 128, 4864, 6656, 4864, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[512, 3840, 7040, 128, 896, 896, 5632, 128, 8576, 5760, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[11904, 256, 128, 7296, 512, 1792, 128, 128, 7296, 3840, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 768, 4864, 128, 3200, 128, 9728, 128, 14464, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 2432, 640, 12928, 10368, 1792, 640, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 256, 2816, 896, 128, 0, 3072, 24064, 2048, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 8960, 640, 128, 10752, 1536, 2944, 4096, 2304, 2048, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[256, 6144, 128, 7936, 14592, 0, 3328, 1024, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[18, 0, 27, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 3, 499, 0, 0, 111, 0, 1, 0, 0, 0, 4, 1837, 810, 1, 4096, 0, 0, 394, 4091, 3385, 0, 0, 0, 0, 0, 0, 3797, 0, 0, 0, 4010, 3884, 0, 1, 60, 9, 0, 3857, 0, 1189, 180, 0, 386, 0, 113, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 3980, 665, 0, 0, 4048, 0, 0, 0, 0, 3318, 4089, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2505, 0, 0, 0, 0, 0, 8, 0, 6, 4, 0, 185, 0, 1, 0, 3773, 6, 0, 55, 0, 0, 955, 4083, 0, 26, 87, 40, 0, 0, 0, 5, 165, 542, 0, 0, 0, 125, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 2963, 0, 0, 0, 3744, 1, 0, 0, 0, 0, 0, 0, 16, 0, 325, 0, 415, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2619, 0, 0, 0, 3980, 0, 1922, 0, 3233, 0, 0, 2291, 5, 0, 2402, 1234, 4030, 689, 920, 0, 3, 0, 0, 0, 7, 1900, 0, 0, 0, 0, 67, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 80, 0, 0, 0, 3887, 0, 10, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 4041, 0, 0, 0, 0, 3696, 0, 2182, 0, 0, 19, 0, 0, 0, 200, 3240, 0, 0, 163, 2468, 3, 0, 3886, 8, 66, 2, 0, 0, 0, 484, 2, 0, 0, 0, 0, 0, 0, 0, 0, 163] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 3998, 0, 0, 0, 3756, 0, 0, 0, 0, 0, 52, 0, 4024, 0, 0, 0, 0, 0, 3589, 0, 0, 0, 0, 0, 518, 2, 0, 3, 328, 911, 0, 0, 46, 92, 0, 2, 0, 352, 2428, 0, 331, 4, 49, 30, 4095, 0, 1, 0, 2877, 0, 0, 0, 0, 0, 1241, 0, 0, 2, 5, 0, 0, 4032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1250, 0, 4081, 0, 53, 0, 0, 0, 0, 14, 0, 0, 4003, 0, 3594, 20, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 2, 0, 4071, 856, 2357, 0, 1, 0, 2977, 0, 0, 1240, 19, 0, 0, 0, 149, 1178, 0, 0, 2207, 0, 2, 4095, 129, 12, 1, 0, 0, 0, 0, 0, 0, 138, 0, 313] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 1162, 209, 0, 0, 0, 0, 4, 0, 3870, 0, 568, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 1317, 0, 25, 0, 4030, 1770, 0, 0, 0, 97, 145, 0, 4050, 2077, 4, 0, 0, 128, 713, 3038, 2380, 0, 2, 0, 0, 0, 48, 1820, 1145, 0, 0, 4085, 0, 0, 0, 0, 72] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 1, 3901, 285, 393, 19, 0, 0, 0, 0, 0, 348, 65, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3809, 0, 0, 0, 0, 0, 2371, 2585, 119, 1, 0, 41, 349, 707, 0, 3, 1008, 6, 1, 3474, 0, 1117, 0, 60, 0, 0, 4028, 0, 0, 0, 0, 347, 0, 3613, 0, 0, 3, 0, 0, 17] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[18, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 3, 499, 0, 1098, 280, 0, 0, 0, 0, 0, 0, 0, 111, 0, 1, 0, 0, 0, 4, 3396, 2, 0, 0, 0, 0, 0, 0, 1837, 810, 1, 4096, 0, 0, 394, 695, 0, 0, 0, 0, 0, 0, 0, 0, 3385, 0, 0, 0, 0, 0, 0, 2771, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2912, 3884, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 3857, 0, 1189, 180, 0, 386, 5, 746, 0, 0, 0, 0, 0, 0, 0, 113, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[38, 1885, 3514, 7833, 6156, 6856, 6372, 114] avg=4096.0000 max=7833 max_rank=3 min=38 min_rank=0 imbalance_ratio=1.912354 max_load_reduction=24935 max_load_reduction_pct=0.760956 moved_to_echo=19498 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 56, 0, 0, 0, 3887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4081, 0, 0, 0, 0, 0, 675, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 1811, 7, 0, 0, 0, 0, 0, 0, 4041, 0, 0, 0, 0, 3696, 0, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 200, 2364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 163, 1793, 3, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 66, 2, 0, 0, 0, 484, 2, 0, 17, 637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 163, 3886, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3943, 5005, 5905, 8108, 2583, 1967, 1208, 4049] avg=4096.0000 max=8108 max_rank=3 min=1208 min_rank=6 imbalance_ratio=1.979492 max_load_reduction=24660 max_load_reduction_pct=0.752563 moved_to_echo=9807 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1250, 0, 2848, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4003, 0, 3594, 20, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 710, 362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4071, 146, 0, 0, 0, 0, 0, 0, 0, 0, 2357, 0, 1, 0, 2977, 0, 0, 906, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 149, 1178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2207, 0, 2, 4095, 129, 12, 1, 0, 871, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 0, 313, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4151, 7722, 1078, 4219, 6241, 1346, 7560, 451] avg=4096.0000 max=7722 max_rank=1 min=451 min_rank=7 imbalance_ratio=1.885254 max_load_reduction=25046 max_load_reduction_pct=0.764343 moved_to_echo=15598 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 1162, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3870, 0, 568, 0, 0, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3346, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1317, 0, 25, 0, 684, 0, 0, 0, 0, 0, 0, 0, 0, 1770, 0, 0, 0, 97, 145, 0, 2955, 0, 0, 0, 0, 0, 0, 0, 0, 2077, 4, 0, 0, 128, 713, 0, 2380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 48, 1820, 1145, 0, 796, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 72, 3038, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1376, 4741, 3349, 2027, 4967, 5302, 3811, 7195] avg=4096.0000 max=7195 max_rank=7 min=1376 min_rank=0 imbalance_ratio=1.756592 max_load_reduction=25573 max_load_reduction_pct=0.780426 moved_to_echo=21275 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2789, 0, 0, 0, 3756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 4024, 0, 1, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3589, 0, 0, 0, 756, 355, 0, 0, 0, 0, 0, 0, 0, 0, 518, 2, 0, 3, 328, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 92, 0, 2, 0, 257, 0, 0, 0, 0, 0, 0, 0, 0, 2428, 0, 331, 3, 49, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2877, 0, 0, 0, 0, 0, 854, 69, 0, 0, 0, 0, 0, 0, 1241, 0, 0, 2, 5, 0, 0, 4032, 4095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6545, 4103, 4700, 1006, 397, 2841, 3801, 9375] avg=4096.0000 max=9375 max_rank=7 min=397 min_rank=4 imbalance_ratio=2.288818 max_load_reduction=23393 max_load_reduction_pct=0.713898 moved_to_echo=16414 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2067, 0, 0, 0, 3744, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 1, 238, 0, 0, 0, 0, 0, 0, 325, 0, 415, 0, 0, 0, 0, 0, 2174, 263, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3980, 0, 1922, 0, 2360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2291, 4, 0, 2402, 0, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 689, 920, 0, 3, 0, 0, 0, 7, 633, 635, 0, 0, 0, 0, 0, 0, 1900, 0, 0, 0, 0, 67, 0, 0, 1234, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5812, 255, 3177, 447, 8262, 8727, 2887, 3201] avg=4096.0000 max=8727 max_rank=5 min=255 min_rank=1 imbalance_ratio=2.130615 max_load_reduction=24041 max_load_reduction_pct=0.733673 moved_to_echo=23077 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 2722, 285, 393, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 348, 65, 1, 0, 0, 953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2150, 346, 0, 0, 0, 0, 0, 0, 3809, 0, 0, 0, 0, 0, 2371, 435, 0, 0, 0, 0, 0, 0, 0, 0, 119, 1, 0, 41, 349, 707, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 6, 1, 2521, 0, 1117, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4028, 0, 0, 0, 0, 347, 833, 0, 0, 0, 0, 0, 0, 0, 0, 3613, 0, 0, 3, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3420, 1367, 6592, 6615, 1220, 4713, 5208, 3633] avg=4096.0000 max=6615 max_rank=3 min=1220 min_rank=4 imbalance_ratio=1.614990 max_load_reduction=26153 max_load_reduction_pct=0.798126 moved_to_echo=14774 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2770, 665, 0, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3318, 4089, 1, 0, 0, 0, 282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 357, 0, 0, 0, 0, 0, 0, 2505, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4, 0, 185, 0, 1, 0, 2746, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 55, 0, 0, 955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 87, 40, 0, 0, 0, 5, 165, 853, 745, 0, 0, 0, 0, 0, 0, 542, 0, 0, 0, 125, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7483, 7690, 4453, 2513, 2942, 1016, 1921, 4750] avg=4096.0000 max=7690 max_rank=1 min=1016 min_rank=5 imbalance_ratio=1.877441 max_load_reduction=25078 max_load_reduction_pct=0.765320 moved_to_echo=10629 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[12288, 896, 640, 5504, 128, 3840, 7296, 2944, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[7680, 128, 128, 4352, 3456, 2816, 256, 14464, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3712, 3840, 4096, 128, 256, 256, 0, 4608, 16384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 4096, 3712, 12160, 128, 8704, 128, 2816, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[5632, 128, 2944, 7296, 4224, 6400, 0, 6656, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[3072, 1024, 10880, 4224, 1408, 768, 1920, 2176, 4096, 3968, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1280, 128, 13312, 1024, 1664, 256, 15488, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[384, 128, 512, 128, 15872, 0, 128, 128, 14464, 1792, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[2953, 2, 0, 4019, 0, 4078, 0, 0, 2694, 0, 0, 885, 0, 1, 0, 0, 26, 0, 4, 0, 0, 4051, 0, 0, 0, 21, 0, 0, 358, 3, 115, 4095, 0, 155, 0, 2855, 0, 0, 0, 22, 146, 300, 0, 1, 0, 0, 0, 1954, 200, 3, 0, 0, 0, 2019, 0, 0, 0, 11, 0, 0, 0, 1796, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[3638, 0, 0, 0, 0, 659, 0, 0, 4016, 3320, 0, 0, 0, 0, 0, 0, 0, 3, 0, 95, 0, 4083, 992, 0, 0, 0, 0, 0, 1834, 0, 0, 3957, 41, 1689, 0, 0, 0, 16, 15, 0, 0, 0, 0, 0, 294, 0, 0, 4059, 0, 494, 0, 0, 0, 0, 0, 0, 2379, 20, 0, 0, 8, 1156, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 13, 0, 0, 2664, 0, 0, 3958, 0, 164, 892, 0, 0, 76, 0, 0, 0, 0, 48, 1, 4091, 0, 0, 0, 3481, 0, 0, 0, 0, 0, 1286, 0, 111, 1, 4006, 0, 233, 158, 0, 0, 0, 0, 0, 0, 161, 167, 0, 0, 0, 0, 1483, 0, 75, 79, 0, 0, 0, 1485, 0, 0, 4082, 4053, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[244, 268, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 2025, 142, 0, 42, 17, 21, 0, 0, 3586, 3417, 0, 0, 0, 0, 0, 0, 2, 939, 0, 2484, 0, 0, 5, 4095, 0, 104, 4007, 4, 0, 0, 0, 1020, 0, 347, 771, 1089, 0, 1, 0, 0, 0, 0, 11, 0, 4, 0, 4078, 764, 0, 3240, 0, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 304, 101, 3942, 0, 24, 0, 0, 0, 0, 237, 844, 0, 0, 0, 0, 0, 0, 0, 971, 3968, 3976, 0, 0, 110, 0, 0, 1, 0, 0, 0, 2100, 0, 0, 3, 2, 0, 0, 4036, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 197, 0, 0, 0, 0, 0, 3786, 4065, 4, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1305, 0, 0, 3705, 0, 0, 15, 0, 0, 0, 0, 37, 0, 0, 0, 0, 507, 0, 3, 1991, 71, 4090, 6, 13, 25, 0, 0, 0, 12, 4, 0, 0, 0, 3966, 0, 3405, 0, 247, 4067, 1563, 0, 0, 0, 234, 0, 4096, 0, 2586, 21, 0, 0, 0, 3, 175, 0, 612, 0, 0, 0, 0, 0, 2, 4, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 1, 3490, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 0, 0, 0, 3935, 1638, 0, 22, 11, 0, 0, 0, 0, 0, 0, 0, 2609, 0, 31, 0, 1, 3474, 0, 2, 12, 0, 4056, 0, 554, 386, 3675, 80, 0, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 0, 4092] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 3472, 1, 4042, 0, 13, 3583, 0, 0, 0, 0, 0, 150, 240, 779, 0, 0, 115, 0, 2638, 0, 0, 0, 0, 0, 24, 32, 0, 0, 0, 0, 365, 252, 3763, 0, 0, 749, 0, 0, 0, 0, 172, 3607, 52, 0, 3445, 0, 39, 1134, 0, 1, 0, 0, 0, 4084, 0, 0, 14, 2, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3638, 0, 0, 0, 0, 659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4016, 3320, 0, 0, 0, 0, 0, 0, 0, 177, 101, 0, 0, 0, 0, 0, 0, 3, 0, 95, 0, 2391, 992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1834, 0, 0, 3957, 477, 573, 0, 0, 0, 0, 0, 0, 41, 1689, 0, 0, 0, 16, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 294, 0, 0, 3958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 0, 0, 0, 0, 0, 0, 0, 1119, 0, 0, 0, 0, 0, 0, 2379, 20, 0, 0, 8, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4297, 7614, 3481, 6841, 1761, 4252, 1613, 2909] avg=4096.0000 max=7614 max_rank=1 min=1613 min_rank=6 imbalance_ratio=1.858887 max_load_reduction=25154 max_load_reduction_pct=0.767639 moved_to_echo=10535 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2953, 2, 0, 2685, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2694, 0, 0, 885, 0, 1, 0, 0, 1334, 275, 48, 0, 0, 0, 0, 0, 26, 0, 4, 0, 0, 2373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 358, 3, 115, 4095, 742, 568, 0, 0, 0, 0, 0, 0, 0, 155, 0, 767, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 146, 300, 0, 1, 0, 0, 0, 1906, 0, 0, 0, 0, 0, 0, 0, 0, 200, 3, 0, 0, 0, 2019, 0, 0, 2088, 1110, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 779, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9718, 5237, 2403, 5902, 944, 2353, 5420, 791] avg=4096.0000 max=9718 max_rank=0 min=791 min_rank=7 imbalance_ratio=2.372559 max_load_reduction=23050 max_load_reduction_pct=0.703430 moved_to_echo=9508 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 13, 0, 0, 2664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3958, 0, 164, 892, 0, 0, 76, 0, 0, 625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 1, 2396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3481, 0, 0, 0, 0, 0, 1286, 1686, 574, 0, 0, 0, 0, 0, 0, 0, 111, 1, 1073, 0, 233, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1483, 0, 75, 79, 0, 2933, 1121, 0, 0, 0, 0, 0, 0, 0, 0, 1485, 0, 0, 1771, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2677, 5715, 2445, 7027, 1576, 328, 5691, 7309] avg=4096.0000 max=7309 max_rank=7 min=328 min_rank=5 imbalance_ratio=1.784424 max_load_reduction=25459 max_load_reduction_pct=0.776947 moved_to_echo=14904 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1305, 0, 0, 2475, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 1230, 0, 64, 0, 0, 0, 0, 0, 507, 0, 3, 1991, 71, 2395, 6, 13, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 12, 4, 0, 0, 0, 574, 0, 0, 0, 0, 0, 0, 0, 3966, 0, 915, 0, 247, 4067, 1563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 234, 0, 4096, 0, 2522, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 3, 175, 0, 612, 2490, 1121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3795, 1331, 4986, 615, 10758, 6852, 4422, 9] avg=4096.0000 max=10758 max_rank=4 min=9 min_rank=7 imbalance_ratio=2.626465 max_load_reduction=22010 max_load_reduction_pct=0.671692 moved_to_echo=22041 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[244, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 2025, 142, 0, 42, 0, 496, 30, 0, 0, 0, 0, 0, 17, 21, 0, 0, 3586, 2002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 939, 0, 2484, 1338, 479, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1100, 0, 104, 4007, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 0, 347, 771, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 11, 0, 2995, 936, 0, 0, 0, 0, 0, 0, 4, 0, 4078, 764, 0, 1406, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[512, 2772, 5626, 5242, 5220, 3197, 3943, 6256] avg=4096.0000 max=6256 max_rank=7 min=512 min_rank=0 imbalance_ratio=1.527344 max_load_reduction=26512 max_load_reduction_pct=0.809082 moved_to_echo=18616 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2320, 1, 4042, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 3583, 0, 0, 0, 0, 0, 150, 240, 1152, 0, 85, 0, 0, 0, 0, 0, 779, 0, 0, 115, 0, 1545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 32, 0, 0, 0, 0, 370, 0, 0, 0, 0, 0, 0, 0, 365, 252, 1011, 0, 0, 749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 3607, 52, 0, 3360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 1134, 0, 1, 0, 0, 0, 2752, 723, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6376, 5210, 2439, 426, 2377, 7191, 4649, 4100] avg=4096.0000 max=7191 max_rank=5 min=426 min_rank=3 imbalance_ratio=1.755615 max_load_reduction=25577 max_load_reduction_pct=0.780548 moved_to_echo=18317 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 2332, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1158, 621, 91, 0, 0, 0, 0, 0, 0, 185, 0, 0, 0, 2305, 1638, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 11, 0, 0, 0, 0, 0, 0, 1673, 552, 0, 0, 0, 0, 0, 0, 0, 2609, 0, 9, 0, 1, 3474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 12, 0, 4056, 0, 554, 386, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 462, 0, 0, 0, 0, 0, 22, 1078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1757, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2334, 1870, 4128, 2258, 6093, 8594, 1642, 5849] avg=4096.0000 max=8594 max_rank=5 min=1642 min_rank=6 imbalance_ratio=2.098145 max_load_reduction=24174 max_load_reduction_pct=0.737732 moved_to_echo=22178 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 304, 101, 2630, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 237, 844, 0, 0, 0, 0, 1312, 626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 971, 3968, 2321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 0, 1, 0, 0, 0, 2100, 1684, 562, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 197, 0, 2, 1093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3786, 1755, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3059, 3019, 7260, 4457, 4039, 1, 5388, 5545] avg=4096.0000 max=7260 max_rank=2 min=1 min_rank=5 imbalance_ratio=1.772461 max_load_reduction=25508 max_load_reduction_pct=0.778442 moved_to_echo=14973 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[256, 384, 128, 5504, 3968, 5248, 1408, 16512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 8960, 384, 4992, 0, 640, 16512, 1664, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[256, 3584, 0, 128, 2304, 1024, 128, 13952, 7680, 4352, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8192, 640, 128, 12544, 128, 11520, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[6528, 128, 5632, 896, 3840, 8064, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[384, 640, 1664, 1536, 128, 6400, 384, 640, 13312, 8320, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1408, 256, 128, 3328, 7680, 17792, 2688, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[14336, 3456, 512, 2688, 2048, 256, 256, 384, 6272, 2944, 512, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[1651, 0, 0, 4006, 7, 0, 0, 0, 49, 0, 12, 1933, 924, 0, 1, 0, 6, 0, 2730, 3, 23, 2587, 0, 0, 0, 0, 2487, 163, 1736, 0, 0, 0, 649, 158, 0, 0, 0, 3531, 0, 3241, 0, 0, 0, 0, 0, 642, 51, 0, 0, 44, 0, 0, 0, 4, 1578, 844, 3708, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[664, 0, 0, 7, 238, 1117, 0, 0, 0, 0, 0, 1, 3488, 946, 0, 4069, 0, 0, 0, 3585, 0, 0, 1, 62, 434, 2, 852, 0, 3953, 1, 1, 23, 700, 65, 0, 0, 0, 160, 3, 910, 0, 367, 0, 3988, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 1, 0, 0, 19, 0, 3034, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 40, 116, 0, 0, 0, 0, 263, 7, 0, 0, 0, 0, 476, 0, 2261, 3828, 31, 3842, 28, 0, 4038, 4, 3030, 61, 4, 1, 40, 3611, 48, 0, 0, 1781, 0, 7, 0, 0, 3, 0, 3928, 474, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 974, 827, 0, 0, 1294, 467, 0, 574, 185, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 781, 1354, 0, 0, 0, 0, 84, 0, 0, 0, 3370, 1935, 278, 0, 0, 4073, 1, 0, 3600, 4095, 1, 0, 0, 0, 0, 0, 3085, 38, 1541, 3950, 0, 0, 2, 0, 0, 1, 0, 0, 3133, 35, 65, 0, 0, 0, 0, 535, 0, 11, 0, 0, 0, 0, 0, 0, 7, 455, 0, 5, 0, 0, 2, 331, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[549, 0, 0, 22, 0, 2724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 1494, 154, 0, 3938, 0, 0, 277, 14, 0, 0, 4007, 0, 0, 1, 0, 0, 2195, 0, 4, 0, 0, 4038, 0, 0, 0, 4080, 0, 0, 0, 0, 21, 1172, 3, 437, 0, 0, 0, 320, 4, 0, 0, 1, 3227, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[21, 0, 5, 3587, 2936, 3362, 0, 287, 0, 0, 0, 3571, 52, 29, 0, 505, 0, 1, 1, 0, 7, 13, 0, 0, 0, 1, 0, 0, 4022, 105, 0, 2, 5, 170, 0, 0, 48, 0, 114, 4086, 0, 0, 0, 8, 0, 0, 0, 2503, 26, 367, 0, 35, 0, 617, 0, 1330, 4093, 0, 0, 3, 0, 0, 856, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[304, 0, 0, 267, 2, 52, 100, 5, 0, 0, 6, 0, 0, 3328, 0, 1, 0, 2, 0, 3770, 3824, 1813, 0, 0, 0, 0, 0, 130, 3397, 0, 0, 6, 260, 0, 3478, 0, 0, 0, 0, 3811, 0, 0, 0, 0, 0, 1291, 246, 24, 0, 147, 0, 0, 0, 15, 0, 0, 2581, 102, 0, 0, 3806, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[1, 0, 0, 4096, 63, 0, 0, 0, 0, 0, 0, 2658, 0, 231, 13, 1221, 0, 0, 0, 0, 0, 2846, 23, 0, 0, 0, 3321, 623, 2767, 0, 1481, 0, 101, 517, 0, 25, 0, 50, 0, 3816, 1022, 3, 0, 0, 1, 332, 0, 118, 0, 0, 0, 35, 0, 9, 0, 3309, 4086, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1651, 0, 0, 4006, 7, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 49, 0, 12, 1933, 924, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2730, 2, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2487, 163, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 649, 158, 0, 0, 0, 3531, 0, 2253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 642, 51, 0, 988, 597, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 4, 1578, 844, 2587, 97, 0, 0, 0, 0, 0, 0, 3708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5665, 2919, 2761, 3692, 6591, 2278, 5154, 3708] avg=4096.0000 max=6591 max_rank=4 min=2278 min_rank=5 imbalance_ratio=1.609131 max_load_reduction=26177 max_load_reduction_pct=0.798859 moved_to_echo=17731 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[664, 0, 0, 7, 238, 1117, 0, 0, 1305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3488, 946, 0, 3170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1833, 0, 0, 1, 62, 0, 0, 0, 0, 0, 0, 0, 0, 434, 2, 852, 0, 2372, 1, 1, 23, 0, 0, 0, 0, 0, 0, 0, 0, 700, 65, 0, 0, 0, 160, 3, 633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 367, 0, 3988, 0, 0, 0, 0, 277, 1359, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 1, 0, 222, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 3034, 0, 1, 0, 899, 402, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3331, 7605, 1896, 3685, 1561, 6036, 4299, 4355] avg=4096.0000 max=7605 max_rank=1 min=1561 min_rank=4 imbalance_ratio=1.856689 max_load_reduction=25163 max_load_reduction_pct=0.767914 moved_to_echo=16251 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 781, 1354, 0, 0, 0, 0, 84, 1311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3370, 1935, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 1, 0, 1840, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3085, 23, 1541, 3950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 2178, 0, 0, 0, 0, 0, 0, 0, 0, 35, 65, 0, 0, 0, 0, 535, 0, 955, 13, 45, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 7, 1, 2, 0, 0, 0, 0, 0, 0, 455, 0, 5, 0, 0, 2, 331, 0, 0, 404, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3530, 5583, 10009, 8599, 2181, 1648, 21, 1197] avg=4096.0000 max=10009 max_rank=2 min=21 min_rank=6 imbalance_ratio=2.443604 max_load_reduction=22759 max_load_reduction_pct=0.694550 moved_to_echo=5047 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[21, 0, 5, 3587, 2936, 3362, 0, 287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3571, 52, 29, 0, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2413, 105, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 5, 170, 0, 0, 48, 0, 114, 2841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 2503, 1245, 1383, 0, 0, 0, 0, 0, 0, 26, 367, 0, 35, 0, 617, 0, 1330, 13, 226, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 3, 0, 0, 856, 0, 111, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10198, 4046, 9, 2521, 3178, 5139, 2614, 5063] avg=4096.0000 max=10198 max_rank=0 min=9 min_rank=2 imbalance_ratio=2.489746 max_load_reduction=22570 max_load_reduction_pct=0.688782 moved_to_echo=15994 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 40, 116, 0, 0, 0, 0, 263, 10, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 476, 0, 1762, 0, 0, 0, 0, 0, 0, 0, 0, 3828, 31, 3842, 15, 0, 0, 4, 3030, 0, 0, 0, 0, 0, 0, 0, 0, 61, 4, 1, 40, 2166, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1781, 0, 7, 0, 0, 3, 0, 2731, 0, 0, 0, 0, 0, 0, 0, 0, 474, 525, 0, 0, 0, 0, 0, 0, 1197, 1242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 974, 827, 0, 4038, 203, 0, 0, 0, 0, 0, 0, 0, 1294, 467, 0, 574, 185, 0, 0, 499, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[429, 2245, 10750, 2320, 4522, 3438, 6042, 3022] avg=4096.0000 max=10750 max_rank=2 min=429 min_rank=0 imbalance_ratio=2.624512 max_load_reduction=22018 max_load_reduction_pct=0.671936 moved_to_echo=17024 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[304, 0, 0, 267, 2, 52, 100, 5, 1372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 3328, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1927, 3824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 2039, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 260, 0, 3478, 0, 0, 0, 0, 2649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1291, 246, 24, 1162, 1168, 48, 0, 0, 0, 0, 0, 0, 147, 0, 0, 0, 15, 0, 0, 1813, 190, 0, 0, 0, 0, 0, 0, 2581, 102, 0, 0, 3806, 0, 0, 0, 0, 423, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2102, 3335, 5753, 2175, 6387, 3939, 2165, 6912] avg=4096.0000 max=6912 max_rank=7 min=2102 min_rank=0 imbalance_ratio=1.687500 max_load_reduction=25856 max_load_reduction_pct=0.789062 moved_to_echo=19403 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 4096, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2658, 0, 231, 13, 952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3321, 623, 1661, 0, 1481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 517, 0, 25, 0, 50, 0, 2653, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 3, 0, 0, 1, 332, 0, 118, 1163, 951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 9, 0, 3309, 2846, 155, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 269, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4160, 3854, 23, 7086, 3346, 3590, 6354, 4355] avg=4096.0000 max=7086 max_rank=3 min=23 min_rank=2 imbalance_ratio=1.729980 max_load_reduction=25682 max_load_reduction_pct=0.783752 moved_to_echo=17645 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[549, 0, 0, 22, 0, 2724, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1494, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 277, 14, 0, 0, 2398, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2195, 0, 4, 0, 0, 2803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 1235, 1381, 4, 0, 0, 0, 0, 0, 21, 1172, 3, 437, 0, 0, 0, 320, 3938, 228, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1, 3227, 0, 0, 0, 905, 19, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3353, 3181, 1567, 2690, 5002, 6700, 6119, 4156] avg=4096.0000 max=6700 max_rank=5 min=1567 min_rank=2 imbalance_ratio=1.635742 max_load_reduction=26068 max_load_reduction_pct=0.795532 moved_to_echo=21977 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[7936, 128, 8192, 5760, 8064, 0, 128, 3200, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3200, 896, 1536, 12032, 3328, 7296, 128, 640, 4096, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[1536, 1024, 0, 8192, 128, 2304, 896, 2688, 8320, 8192, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3584, 1024, 5760, 128, 128, 3840, 128, 18816, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[896, 128, 6784, 4096, 14208, 1792, 5504, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 1792, 128, 512, 4096, 1664, 2432, 5888, 15360, 1408, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 11648, 6400, 5376, 128, 9472, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[14976, 1408, 512, 128, 10752, 256, 1280, 0, 2688, 1280, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 2696, 1492, 0, 270, 0, 0, 0, 0, 0, 0, 0, 2377, 4096, 0, 41, 26, 0, 0, 2971, 108, 0, 11, 782, 0, 0, 208, 0, 0, 3168, 0, 0, 130, 0, 3, 62, 0, 604, 0, 25, 0, 3495, 8, 0, 0, 3988, 4093, 2110, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 19, 3971, 0, 719, 2, 1, 2, 3740, 0, 0, 0, 483, 1, 146, 4093, 0, 0, 3421, 0, 0, 22, 0, 0, 21, 0, 2431, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 3754, 999, 113, 122, 1949, 1, 0, 3416, 0, 0, 0, 0, 0, 0, 0, 0, 214, 0, 2788] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[78, 97, 0, 0, 0, 141, 1764, 3862, 0, 0, 16, 0, 0, 2653, 0, 0, 0, 0, 0, 3489, 4096, 0, 0, 117, 0, 5, 0, 0, 2, 0, 1, 2438, 0, 1, 0, 0, 15, 0, 0, 0, 0, 0, 7, 4090, 0, 3561, 3946, 0, 0, 1, 0, 0, 0, 0, 1099, 0, 0, 0, 0, 0, 0, 1289, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 81, 0, 251, 0, 10, 0, 4067, 2051, 0, 0, 0, 662, 1, 0, 0, 40, 3992, 7, 4096, 1240, 148, 3285, 15, 0, 0, 1, 0, 0, 11, 0, 0, 0, 0, 0, 0, 1110, 0, 1057, 0, 0, 1888, 0, 0, 0, 9, 4066, 0, 0, 0, 0, 9, 0, 0, 750, 0, 119, 0, 0, 0, 728, 3074, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2703, 0, 0, 1, 198, 53, 0, 113, 3331, 0, 10, 6, 1, 0, 753, 0, 13, 37, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 3775, 0, 0, 0, 0, 307, 0, 0, 0, 378, 2392, 0, 0, 0, 2941, 0, 0, 2254, 21, 3786, 0, 626, 0, 0, 4071, 0, 920, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 3144, 1709, 0, 17, 0, 0, 0, 3197, 0, 0, 0, 45, 0, 0, 3, 0, 0, 91, 0, 4013, 0, 876, 0, 21, 3902, 0, 1, 37, 68, 468, 0, 0, 0, 31, 0, 0, 507, 0, 1813, 0, 0, 9, 0, 0, 4064, 2927, 0, 0, 1138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 601, 0, 4085, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[67, 0, 1449, 0, 0, 0, 0, 3, 0, 0, 29, 0, 4011, 0, 0, 0, 0, 3934, 0, 4092, 0, 1, 0, 3708, 0, 0, 4088, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 33, 0, 0, 451, 0, 0, 0, 67, 0, 125, 7, 11, 0, 3, 2730, 0, 0, 481, 2228, 0, 37, 0, 0, 1273, 0, 3936] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[40, 3990, 0, 106, 2182, 0, 0, 0, 0, 1926, 9, 0, 0, 461, 0, 0, 0, 5, 0, 0, 0, 4095, 0, 1630, 731, 0, 4062, 0, 0, 1439, 0, 53, 0, 0, 0, 0, 257, 0, 0, 23, 0, 4095, 0, 0, 0, 3816, 134, 9, 2, 187, 0, 16, 0, 0, 0, 48, 0, 0, 0, 0, 195, 3257, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 2696, 1492, 0, 270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2377, 4096, 0, 41, 0, 8, 122, 0, 0, 0, 0, 0, 26, 0, 0, 0, 44, 0, 11, 782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 208, 0, 0, 3168, 0, 0, 30, 335, 0, 0, 0, 0, 0, 0, 130, 0, 3, 62, 0, 604, 0, 25, 3988, 268, 0, 0, 0, 0, 0, 0, 0, 3495, 8, 0, 0, 0, 3971, 2110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 2368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4458, 6644, 863, 3741, 5080, 9584, 2372, 26] avg=4096.0000 max=9584 max_rank=5 min=26 min_rank=7 imbalance_ratio=2.339844 max_load_reduction=23184 max_load_reduction_pct=0.707520 moved_to_echo=17062 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[78, 97, 0, 0, 0, 141, 1764, 3395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 2653, 0, 0, 467, 329, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1605, 0, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 2, 0, 1, 2438, 1158, 394, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 15, 0, 0, 0, 3561, 314, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4090, 0, 0, 3829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1099, 0, 2781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1289, 0, 0, 1004, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5475, 3582, 1722, 3998, 3891, 7926, 3881, 2293] avg=4096.0000 max=7926 max_rank=5 min=1722 min_rank=2 imbalance_ratio=1.935059 max_load_reduction=24842 max_load_reduction_pct=0.758118 moved_to_echo=17991 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 81, 0, 251, 0, 10, 0, 3575, 0, 0, 0, 0, 0, 0, 0, 0, 2051, 0, 0, 0, 662, 1, 0, 0, 492, 99, 121, 0, 0, 0, 0, 0, 40, 3992, 7, 0, 487, 148, 3285, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 11, 0, 0, 350, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1110, 0, 1057, 0, 9, 368, 0, 0, 0, 0, 0, 0, 0, 1888, 0, 0, 0, 0, 3945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 750, 0, 3266, 0, 0, 0, 0, 0, 0, 0, 119, 0, 0, 0, 728, 3074, 0, 0, 304, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3917, 3426, 7974, 824, 2544, 5833, 4025, 4225] avg=4096.0000 max=7974 max_rank=2 min=824 min_rank=3 imbalance_ratio=1.946777 max_load_reduction=24794 max_load_reduction_pct=0.756653 moved_to_echo=16627 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 19, 3490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 719, 2, 1, 2, 3740, 0, 0, 481, 329, 29, 0, 0, 0, 0, 0, 0, 483, 1, 0, 1601, 0, 0, 3421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 21, 0, 2431, 1158, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 340, 0, 0, 0, 3754, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 970, 113, 0, 0, 0, 0, 0, 0, 0, 0, 122, 1949, 1, 0, 3416, 0, 0, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 214, 0, 2788, 1005, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3509, 5303, 5506, 3649, 4107, 1083, 5604, 4007] avg=4096.0000 max=5604 max_rank=6 min=1083 min_rank=5 imbalance_ratio=1.368164 max_load_reduction=27164 max_load_reduction_pct=0.828979 moved_to_echo=14801 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 3144, 1709, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3197, 0, 0, 0, 45, 0, 0, 3, 0, 322, 87, 0, 0, 0, 0, 0, 0, 0, 91, 0, 1572, 0, 876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 3902, 0, 1, 37, 68, 468, 0, 1135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 507, 0, 1813, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 2840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 601, 0, 4085, 0, 1, 984, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4870, 3654, 2539, 5632, 6415, 2849, 1138, 5671] avg=4096.0000 max=6415 max_rank=4 min=1138 min_rank=6 imbalance_ratio=1.566162 max_load_reduction=26353 max_load_reduction_pct=0.804230 moved_to_echo=16073 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[40, 3990, 0, 106, 2182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1926, 9, 0, 0, 461, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4095, 0, 1630, 0, 0, 0, 0, 0, 0, 0, 0, 731, 0, 4062, 0, 0, 1439, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 257, 0, 0, 23, 3816, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 128, 9, 0, 0, 0, 0, 0, 0, 0, 0, 2, 187, 0, 16, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 195, 3257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6318, 2402, 5730, 6285, 4096, 4232, 253, 3452] avg=4096.0000 max=6318 max_rank=0 min=253 min_rank=6 imbalance_ratio=1.542480 max_load_reduction=26450 max_load_reduction_pct=0.807190 moved_to_echo=12033 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2703, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 198, 53, 0, 113, 3331, 0, 10, 6, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 753, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3775, 0, 0, 0, 0, 2392, 0, 0, 0, 0, 0, 0, 0, 307, 0, 0, 0, 378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2941, 0, 0, 2254, 21, 3786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 626, 0, 0, 4071, 0, 920, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2704, 3715, 791, 4084, 6167, 685, 9002, 5620] avg=4096.0000 max=9002 max_rank=6 min=685 min_rank=5 imbalance_ratio=2.197754 max_load_reduction=23766 max_load_reduction_pct=0.725281 moved_to_echo=21474 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[67, 0, 1449, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 4011, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3934, 0, 0, 0, 1, 0, 3708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 3, 0, 0, 0, 0, 0, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 33, 0, 67, 367, 0, 0, 0, 0, 0, 0, 0, 451, 0, 0, 0, 0, 0, 125, 0, 0, 0, 0, 0, 0, 0, 0, 7, 11, 0, 3, 2730, 0, 0, 481, 3261, 0, 0, 0, 0, 0, 0, 0, 2228, 0, 37, 0, 0, 1273, 0, 3936, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1517, 4042, 7643, 4555, 468, 576, 6493, 7474] avg=4096.0000 max=7643 max_rank=2 min=468 min_rank=4 imbalance_ratio=1.865967 max_load_reduction=25125 max_load_reduction_pct=0.766754 moved_to_echo=15011 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[384, 9984, 128, 4096, 384, 0, 15744, 2432, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 8448, 896, 0, 5376, 4352, 4224, 9728, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[256, 6272, 128, 128, 8448, 128, 5760, 640, 11904, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3072, 0, 128, 4736, 1024, 14208, 0, 6784, 3328, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[256, 128, 128, 3840, 1792, 1152, 1152, 1920, 21760, 1408, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 10112, 4736, 384, 5248, 256, 1792, 10496, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[768, 3968, 12544, 128, 128, 4736, 512, 4992, 3840, 1792, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[5504, 2816, 128, 128, 10496, 11008, 128, 128, 1536, 1152, 512, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 95, 6, 0, 0, 7, 4000, 0, 0, 262, 4093, 125, 2884, 2460, 0, 0, 0, 0, 0, 438, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 1020, 0, 4095, 0, 0, 2, 0, 34, 13, 41, 4, 0, 0, 0, 3515, 0, 0, 9, 0, 0, 0, 1480, 4081, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 1, 330, 0, 0, 0, 3615, 4047, 909, 0, 0, 3977, 1, 0, 1, 0, 1878, 0, 319, 202, 0, 0, 2420, 3172, 0, 0, 0, 2, 0, 1472, 0, 0, 0, 0, 0, 1, 0, 377, 565, 7, 0, 0, 1309, 0, 0, 0, 2227, 0, 4046, 0, 0, 390, 0, 0, 0, 0, 67, 0, 0, 0, 0, 0, 1433, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1, 1216, 62, 0, 3713, 0, 0, 38, 0, 5, 3117, 16, 0, 0, 0, 3493, 4095, 615, 1128, 0, 461, 0, 14, 0, 0, 6, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2514, 0, 0, 3783, 74, 0, 3429, 0, 536, 0, 0, 0, 0, 0, 335, 0, 21, 0, 0, 0, 0, 0, 0, 0, 3997, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 177, 0, 0, 0, 375, 3765, 0, 0, 42, 0, 0, 0, 1, 1808, 3337, 0, 0, 1987, 465, 3712, 0, 0, 0, 1, 267, 1521, 0, 10, 0, 0, 0, 0, 3190, 0, 6, 6, 0, 12, 0, 4064, 0, 154, 0, 4, 0, 0, 241, 0, 0, 7, 0, 3732, 0, 0, 0, 3476, 390, 0, 0, 18, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1, 17, 84, 1, 0, 1, 2, 8, 0, 46, 0, 0, 1367, 0, 1, 0, 246, 19, 0, 0, 0, 0, 1579, 4092, 0, 2, 0, 3, 2202, 0, 0, 0, 0, 0, 0, 0, 606, 3459, 502, 51, 287, 0, 4085, 0, 0, 7, 3899, 4086, 0, 0, 0, 3840, 0, 0, 0, 0, 2213, 0, 0, 0, 46, 0, 0, 16] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[8, 0, 0, 627, 3807, 0, 3918, 214, 1112, 186, 65, 2887, 2000, 0, 0, 0, 721, 0, 0, 0, 2168, 32, 4096, 1450, 0, 0, 1551, 0, 333, 8, 0, 0, 0, 18, 256, 2049, 0, 0, 0, 0, 0, 3, 1, 0, 0, 2, 0, 0, 1, 0, 0, 95, 1, 0, 1614, 0, 0, 344, 0, 2, 0, 341, 2127, 731] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 36, 0, 0, 0, 0, 0, 10, 3425, 0, 1885, 0, 0, 0, 2, 0, 0, 0, 0, 602, 60, 0, 4088, 143, 0, 0, 434, 0, 0, 0, 3830, 0, 194, 0, 0, 868, 0, 0, 0, 2, 3447, 0, 4096, 0, 64, 0, 1, 1815, 3447, 0, 0, 41, 0, 0, 0, 0, 4095, 0, 0, 0, 183, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4077, 2, 4, 0, 0, 2, 4094, 0, 0, 0, 4094, 0, 0, 0, 365, 977, 54, 45, 0, 0, 3072, 18, 0, 0, 0, 0, 0, 0, 1, 2837, 0, 0, 33, 1064, 1674, 0, 3818, 0, 0, 1, 0, 0, 0, 57, 2380, 0, 0, 7, 4089, 0, 0, 0, 1, 0, 0, 0, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 6, 0, 0, 7, 42, 0, 0, 0, 0, 0, 0, 0, 996, 0, 0, 262, 4093, 125, 0, 2460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 438, 0, 0, 2677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 2143, 207, 0, 0, 0, 0, 0, 0, 1020, 0, 2209, 0, 0, 2, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 13, 41, 4, 0, 0, 0, 3515, 0, 797, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 1480, 4081, 4, 672, 861, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4471, 150, 7936, 3115, 2354, 3265, 4370, 7107] avg=4096.0000 max=7936 max_rank=2 min=150 min_rank=1 imbalance_ratio=1.937500 max_load_reduction=24832 max_load_reduction_pct=0.757812 moved_to_echo=17096 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 330, 0, 0, 0, 3615, 4047, 120, 0, 0, 0, 0, 0, 0, 0, 909, 0, 0, 3977, 1, 0, 1, 0, 13, 0, 0, 0, 0, 0, 0, 0, 461, 0, 319, 202, 0, 0, 0, 3172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1472, 0, 0, 2247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 377, 565, 7, 1010, 173, 0, 0, 0, 0, 0, 0, 0, 0, 707, 0, 0, 0, 2227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 0, 0, 390, 0, 0, 0, 0, 255, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 0, 0, 1433, 0, 214, 407, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8113, 4901, 4154, 3721, 2133, 2934, 4691, 2121] avg=4096.0000 max=8113 max_rank=0 min=2121 min_rank=7 imbalance_ratio=1.980713 max_load_reduction=24655 max_load_reduction_pct=0.752411 moved_to_echo=11879 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 17, 84, 1, 0, 1, 2, 8, 374, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 1367, 0, 1, 0, 42, 0, 0, 0, 0, 0, 0, 0, 63, 19, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 2202, 0, 0, 0, 1466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 606, 3459, 502, 51, 131, 113, 0, 0, 0, 0, 0, 0, 287, 0, 2204, 0, 0, 7, 3899, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3840, 0, 0, 0, 0, 795, 0, 0, 0, 0, 0, 0, 0, 2213, 0, 0, 0, 46, 0, 0, 16, 670, 52, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[488, 1456, 4174, 3673, 4862, 10483, 4635, 2997] avg=4096.0000 max=10483 max_rank=5 min=488 min_rank=0 imbalance_ratio=2.559326 max_load_reduction=22285 max_load_reduction_pct=0.680084 moved_to_echo=22977 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 1216, 62, 0, 3713, 0, 0, 38, 314, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3117, 16, 0, 0, 0, 3493, 35, 0, 0, 0, 0, 0, 0, 0, 1020, 615, 1128, 0, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 99, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2514, 0, 0, 3783, 2194, 1, 0, 0, 0, 0, 0, 0, 74, 0, 1850, 0, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 335, 0, 21, 0, 0, 0, 668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3997, 0, 0, 0, 562, 881, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5344, 6666, 3224, 118, 8492, 2460, 1024, 5440] avg=4096.0000 max=8492 max_rank=4 min=118 min_rank=3 imbalance_ratio=2.073242 max_load_reduction=24276 max_load_reduction_pct=0.740845 moved_to_echo=17416 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 0, 627, 3807, 0, 3918, 214, 0, 0, 0, 0, 0, 0, 0, 0, 1112, 186, 65, 2887, 2000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 0, 0, 0, 2168, 32, 0, 1450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1551, 0, 333, 8, 0, 0, 3803, 0, 0, 0, 0, 0, 0, 0, 0, 18, 256, 2049, 0, 0, 0, 0, 386, 293, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 95, 1, 0, 1614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 344, 0, 2, 0, 341, 2127, 731, 0, 155, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8574, 6250, 3830, 5695, 3002, 6, 1711, 3700] avg=4096.0000 max=8574 max_rank=0 min=6 min_rank=5 imbalance_ratio=2.093262 max_load_reduction=24194 max_load_reduction_pct=0.738342 moved_to_echo=8419 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 177, 0, 0, 0, 375, 372, 0, 0, 0, 0, 0, 0, 0, 3765, 0, 0, 42, 0, 0, 0, 1, 41, 0, 0, 0, 0, 0, 0, 0, 450, 3337, 0, 0, 1987, 465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 267, 1521, 0, 10, 0, 3447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3190, 0, 6, 6, 0, 969, 265, 0, 0, 0, 0, 0, 0, 12, 0, 2194, 0, 154, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 241, 0, 0, 7, 0, 3732, 0, 791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3476, 390, 0, 0, 18, 0, 666, 389, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[924, 3849, 6239, 5246, 4436, 2364, 4771, 4939] avg=4096.0000 max=6239 max_rank=2 min=924 min_rank=0 imbalance_ratio=1.523193 max_load_reduction=26529 max_load_reduction_pct=0.809601 moved_to_echo=16510 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4077, 2, 350, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 2, 4094, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 1019, 0, 0, 0, 365, 977, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3072, 18, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2837, 0, 0, 33, 1064, 2194, 4, 0, 0, 0, 0, 0, 0, 1674, 0, 2060, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 2380, 0, 0, 7, 4089, 0, 743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 626, 881, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4429, 4139, 2406, 3140, 6133, 3735, 7276, 1510] avg=4096.0000 max=7276 max_rank=6 min=1510 min_rank=7 imbalance_ratio=1.776367 max_load_reduction=25492 max_load_reduction_pct=0.777954 moved_to_echo=18654 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 36, 0, 0, 0, 0, 0, 10, 379, 0, 0, 0, 0, 0, 0, 0, 3425, 0, 1885, 0, 0, 0, 2, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 602, 60, 0, 0, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 434, 0, 0, 0, 3830, 0, 3796, 0, 0, 0, 0, 0, 0, 0, 194, 0, 0, 868, 0, 0, 0, 2, 0, 292, 0, 0, 0, 0, 0, 0, 3447, 0, 2194, 0, 64, 0, 1, 1815, 0, 0, 0, 0, 0, 0, 0, 0, 3447, 0, 0, 41, 0, 0, 0, 0, 802, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 183, 0, 0, 0, 676, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[425, 5357, 805, 8060, 1356, 7521, 4290, 4954] avg=4096.0000 max=8060 max_rank=3 min=425 min_rank=0 imbalance_ratio=1.967773 max_load_reduction=24708 max_load_reduction_pct=0.754028 moved_to_echo=18121 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[6528, 128, 13440, 0, 768, 128, 6144, 6016, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 1280, 512, 896, 7552, 128, 15744, 4736, 2304, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 5248, 384, 4096, 1920, 3840, 0, 17536, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 1536, 1152, 9216, 1664, 0, 11392, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[256, 128, 384, 8960, 3200, 3968, 1152, 4992, 9088, 1408, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[9216, 256, 5120, 7040, 7552, 0, 128, 3584, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[6400, 384, 3584, 512, 4352, 1920, 7680, 768, 4096, 3712, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[7552, 384, 2816, 4480, 128, 128, 13056, 0, 4864, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True + [2026-09-21 08:24:24] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 351.3 | throughput per GPU (TFLOP/s/GPU): 91.7 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.120955E+01 | loss scale: 1.0 | grad norm: 36.283 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[2051, 812, 151, 306, 217, 488, 249, 79, 290, 427, 115, 4, 171, 282, 210, 1385, 112, 666, 804, 1488, 163, 408, 841, 259, 817, 140, 55, 993, 469, 564, 88, 433, 288, 597, 814, 158, 1311, 152, 451, 389, 623, 776, 857, 3, 1767, 426, 37, 511, 348, 1510, 500, 119, 1834, 318, 131, 226, 837, 29, 811, 361, 64, 379, 477, 127] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[2035, 819, 88, 193, 301, 378, 258, 102, 425, 204, 88, 1, 66, 329, 188, 1891, 189, 379, 928, 1688, 115, 874, 834, 513, 1244, 154, 95, 1180, 718, 484, 49, 221, 410, 654, 981, 133, 667, 200, 743, 507, 481, 224, 615, 16, 1508, 711, 29, 751, 723, 599, 851, 68, 1443, 567, 262, 269, 509, 21, 662, 212, 77, 307, 349, 188] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1964, 951, 59, 226, 167, 347, 85, 48, 433, 315, 63, 1, 28, 575, 219, 1901, 78, 518, 1294, 2021, 102, 730, 1561, 539, 1029, 55, 372, 1224, 649, 241, 125, 300, 154, 929, 654, 129, 628, 344, 527, 435, 261, 335, 417, 7, 1653, 632, 64, 839, 633, 502, 376, 35, 1687, 226, 202, 853, 436, 101, 578, 131, 7, 272, 272, 229] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[2684, 581, 150, 228, 345, 418, 89, 132, 426, 265, 65, 4, 74, 350, 302, 2016, 129, 1004, 1241, 1339, 138, 348, 503, 230, 725, 172, 189, 1427, 335, 594, 60, 251, 256, 469, 454, 57, 681, 186, 346, 389, 582, 486, 1131, 20, 1814, 852, 53, 417, 438, 903, 972, 32, 1917, 461, 85, 137, 771, 15, 721, 378, 19, 353, 447, 112] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[2113, 840, 154, 441, 255, 545, 128, 104, 345, 250, 48, 2, 20, 355, 139, 2041, 40, 557, 997, 1784, 123, 935, 940, 294, 1187, 123, 229, 1298, 429, 532, 81, 266, 221, 522, 672, 117, 586, 330, 473, 377, 519, 364, 433, 15, 1502, 764, 81, 545, 560, 727, 763, 57, 1856, 420, 261, 783, 521, 34, 627, 204, 65, 215, 428, 131] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[2091, 517, 170, 190, 208, 375, 140, 121, 261, 344, 45, 7, 82, 300, 236, 1280, 135, 772, 1344, 1786, 264, 712, 1080, 252, 755, 224, 247, 1348, 559, 414, 61, 257, 524, 742, 518, 119, 459, 280, 437, 571, 430, 636, 852, 20, 1488, 991, 85, 452, 345, 404, 577, 29, 2498, 445, 78, 479, 509, 18, 794, 200, 62, 499, 525, 125] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1529, 919, 104, 359, 113, 363, 25, 57, 141, 276, 79, 1, 23, 404, 86, 1583, 123, 348, 1408, 1639, 160, 942, 1193, 259, 931, 211, 134, 1573, 255, 495, 132, 221, 208, 821, 781, 158, 799, 311, 435, 515, 479, 641, 616, 32, 1663, 1005, 43, 445, 936, 711, 494, 105, 1417, 546, 206, 567, 626, 12, 1213, 264, 42, 203, 211, 177] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[2310, 693, 194, 170, 304, 395, 32, 75, 242, 286, 24, 0, 73, 224, 173, 1713, 133, 1080, 1453, 1966, 75, 445, 817, 145, 975, 169, 87, 1365, 450, 971, 135, 322, 297, 478, 993, 77, 670, 189, 334, 498, 290, 743, 775, 37, 1710, 670, 53, 287, 272, 471, 788, 48, 2218, 586, 95, 114, 575, 4, 530, 202, 4, 390, 797, 77] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2003, 819, 88, 193, 301, 378, 258, 102, 0, 0, 0, 0, 0, 0, 0, 0, 425, 204, 88, 1, 66, 329, 188, 1891, 612, 152, 117, 0, 0, 0, 0, 0, 189, 379, 928, 31, 115, 874, 834, 513, 0, 0, 0, 0, 0, 0, 0, 0, 1244, 154, 95, 1180, 718, 484, 49, 221, 70, 32, 0, 0, 0, 0, 0, 0, 410, 654, 981, 133, 667, 200, 743, 507, 363, 0, 0, 0, 0, 0, 0, 0, 481, 224, 615, 16, 958, 711, 29, 751, 0, 0, 0, 0, 0, 0, 0, 0, 723, 599, 851, 68, 831, 567, 262, 269, 0, 0, 0, 0, 0, 0, 0, 0, 509, 21, 662, 212, 77, 307, 349, 188, 1505, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4142, 4073, 3863, 4247, 4658, 3785, 4170, 3830] avg=4096.0000 max=4658 max_rank=4 min=3785 min_rank=5 imbalance_ratio=1.137207 max_load_reduction=28110 max_load_reduction_pct=0.857849 moved_to_echo=16443 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1933, 951, 59, 226, 167, 347, 85, 48, 0, 0, 0, 0, 0, 0, 0, 0, 433, 315, 63, 1, 28, 575, 219, 1901, 716, 182, 128, 0, 0, 0, 0, 0, 78, 518, 1294, 37, 102, 730, 1561, 539, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 55, 372, 1224, 649, 241, 125, 300, 77, 31, 0, 0, 0, 0, 0, 0, 154, 929, 654, 129, 628, 344, 527, 435, 398, 0, 0, 0, 0, 0, 0, 0, 261, 335, 417, 7, 1050, 632, 64, 839, 0, 0, 0, 0, 0, 0, 0, 0, 633, 502, 376, 35, 971, 226, 202, 853, 0, 0, 0, 0, 0, 0, 0, 0, 436, 101, 578, 131, 7, 272, 272, 229, 1802, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3816, 4561, 4859, 4103, 4198, 3605, 3798, 3828] avg=4096.0000 max=4859 max_rank=2 min=3605 min_rank=5 imbalance_ratio=1.186279 max_load_reduction=27909 max_load_reduction_pct=0.851715 moved_to_echo=15429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2019, 812, 151, 306, 217, 488, 249, 79, 0, 0, 0, 0, 0, 0, 0, 0, 290, 427, 115, 4, 171, 282, 210, 1385, 778, 134, 137, 0, 0, 0, 0, 0, 112, 666, 804, 27, 163, 408, 841, 259, 0, 0, 0, 0, 0, 0, 0, 0, 817, 140, 55, 993, 469, 564, 88, 433, 82, 32, 0, 0, 0, 0, 0, 0, 288, 597, 814, 158, 1311, 152, 451, 389, 426, 0, 0, 0, 0, 0, 0, 0, 623, 776, 857, 3, 1122, 426, 37, 511, 0, 0, 0, 0, 0, 0, 0, 0, 348, 1510, 500, 119, 1056, 318, 131, 226, 0, 0, 0, 0, 0, 0, 0, 0, 837, 29, 811, 361, 64, 379, 477, 127, 1327, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4321, 3933, 3280, 3673, 4586, 4355, 4208, 4412] avg=4096.0000 max=4586 max_rank=4 min=3280 min_rank=2 imbalance_ratio=1.119629 max_load_reduction=28182 max_load_reduction_pct=0.860046 moved_to_echo=17561 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2642, 581, 150, 228, 345, 418, 89, 132, 0, 0, 0, 0, 0, 0, 0, 0, 426, 265, 65, 4, 74, 350, 302, 2016, 814, 120, 141, 0, 0, 0, 0, 0, 129, 1004, 1241, 25, 138, 348, 503, 230, 0, 0, 0, 0, 0, 0, 0, 0, 725, 172, 189, 1427, 335, 594, 60, 251, 84, 42, 0, 0, 0, 0, 0, 0, 256, 469, 454, 57, 681, 186, 346, 389, 437, 0, 0, 0, 0, 0, 0, 0, 582, 486, 1131, 20, 1152, 852, 53, 417, 0, 0, 0, 0, 0, 0, 0, 0, 438, 903, 972, 32, 1103, 461, 85, 137, 0, 0, 0, 0, 0, 0, 0, 0, 771, 15, 721, 378, 19, 353, 447, 112, 1194, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4585, 4577, 3618, 3879, 3275, 4693, 4131, 4010] avg=4096.0000 max=4693 max_rank=5 min=3275 min_rank=4 imbalance_ratio=1.145752 max_load_reduction=28075 max_load_reduction_pct=0.856781 moved_to_echo=16109 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2080, 840, 154, 441, 255, 545, 128, 104, 0, 0, 0, 0, 0, 0, 0, 0, 345, 250, 48, 2, 20, 355, 139, 2041, 788, 160, 116, 0, 0, 0, 0, 0, 40, 557, 997, 33, 123, 935, 940, 294, 0, 0, 0, 0, 0, 0, 0, 0, 1187, 123, 229, 1298, 429, 532, 81, 266, 70, 33, 0, 0, 0, 0, 0, 0, 221, 522, 672, 117, 586, 330, 473, 377, 362, 0, 0, 0, 0, 0, 0, 0, 519, 364, 433, 15, 954, 764, 81, 545, 0, 0, 0, 0, 0, 0, 0, 0, 560, 727, 763, 57, 1068, 420, 261, 783, 0, 0, 0, 0, 0, 0, 0, 0, 521, 34, 627, 204, 65, 215, 428, 131, 1591, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4547, 4264, 3919, 4248, 3660, 3675, 4639, 3816] avg=4096.0000 max=4639 max_rank=6 min=3660 min_rank=4 imbalance_ratio=1.132568 max_load_reduction=28129 max_load_reduction_pct=0.858429 moved_to_echo=15790 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2058, 517, 170, 190, 208, 375, 140, 121, 0, 0, 0, 0, 0, 0, 0, 0, 261, 344, 45, 7, 82, 300, 236, 1280, 1060, 161, 115, 0, 0, 0, 0, 0, 135, 772, 1344, 32, 264, 712, 1080, 252, 0, 0, 0, 0, 0, 0, 0, 0, 755, 224, 247, 1348, 559, 414, 61, 257, 69, 33, 0, 0, 0, 0, 0, 0, 524, 742, 518, 119, 459, 280, 437, 571, 359, 0, 0, 0, 0, 0, 0, 0, 430, 636, 852, 20, 945, 991, 85, 452, 0, 0, 0, 0, 0, 0, 0, 0, 345, 404, 577, 29, 1438, 445, 78, 479, 0, 0, 0, 0, 0, 0, 0, 0, 509, 18, 794, 200, 62, 499, 525, 125, 1593, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3779, 3891, 4591, 3967, 4009, 4411, 3795, 4325] avg=4096.0000 max=4591 max_rank=2 min=3779 min_rank=0 imbalance_ratio=1.120850 max_load_reduction=28177 max_load_reduction_pct=0.859894 moved_to_echo=16540 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2274, 693, 194, 170, 304, 395, 32, 75, 0, 0, 0, 0, 0, 0, 0, 0, 242, 286, 24, 0, 73, 224, 173, 1713, 941, 177, 133, 0, 0, 0, 0, 0, 133, 1080, 1453, 36, 75, 445, 817, 145, 0, 0, 0, 0, 0, 0, 0, 0, 975, 169, 87, 1365, 450, 971, 135, 322, 79, 36, 0, 0, 0, 0, 0, 0, 297, 478, 993, 77, 670, 189, 334, 498, 412, 0, 0, 0, 0, 0, 0, 0, 290, 743, 775, 37, 1086, 670, 53, 287, 0, 0, 0, 0, 0, 0, 0, 0, 272, 471, 788, 48, 1277, 586, 95, 114, 0, 0, 0, 0, 0, 0, 0, 0, 575, 4, 530, 202, 4, 390, 797, 77, 1753, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4137, 3986, 4184, 4589, 3948, 3941, 3651, 4332] avg=4096.0000 max=4589 max_rank=3 min=3651 min_rank=6 imbalance_ratio=1.120361 max_load_reduction=28179 max_load_reduction_pct=0.859955 moved_to_echo=15872 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1501, 919, 104, 359, 113, 363, 25, 57, 0, 0, 0, 0, 0, 0, 0, 0, 141, 276, 79, 1, 23, 404, 86, 1583, 606, 151, 133, 0, 0, 0, 0, 0, 123, 348, 1408, 21, 160, 942, 1193, 259, 0, 0, 0, 0, 0, 0, 0, 0, 931, 211, 134, 1573, 255, 495, 132, 221, 82, 28, 0, 0, 0, 0, 0, 0, 208, 821, 781, 158, 799, 311, 435, 515, 406, 0, 0, 0, 0, 0, 0, 0, 479, 641, 616, 32, 1042, 1005, 43, 445, 0, 0, 0, 0, 0, 0, 0, 0, 936, 711, 494, 105, 811, 546, 206, 567, 0, 0, 0, 0, 0, 0, 0, 0, 626, 12, 1213, 264, 42, 203, 211, 177, 1467, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3441, 3483, 4454, 4062, 4434, 4303, 4376, 4215] avg=4096.0000 max=4454 max_rank=2 min=3441 min_rank=0 imbalance_ratio=1.087402 max_load_reduction=28314 max_load_reduction_pct=0.864075 moved_to_echo=17328 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4864, 256, 6016, 2048, 384, 2688, 3584, 1280, 12288, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[2688, 2432, 640, 128, 640, 2944, 1664, 13824, 6400, 1280, 1024, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[2432, 5248, 5888, 1024, 5888, 2048, 3840, 3712, 3200, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[7680, 1280, 1408, 10496, 3968, 4352, 768, 2304, 640, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1024, 5376, 9472, 256, 1152, 5504, 7808, 2560, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3712, 4224, 5760, 256, 8320, 6144, 512, 4352, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[16512, 6144, 1152, 2176, 1920, 3328, 1024, 768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4352, 5888, 5376, 512, 8576, 3584, 1408, 3456, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[63, 1520, 1630, 0, 0, 0, 647, 0, 495, 17, 3, 4, 22, 0, 681, 0, 3, 0, 0, 0, 0, 3029, 0, 2612, 0, 3, 1, 1636, 11, 1181, 22, 2390, 282, 0, 591, 24, 0, 2, 5, 3692, 260, 42, 2818, 2, 0, 0, 946, 1840, 338, 1526, 0, 707, 5, 0, 0, 1601, 986, 166, 61, 0, 893, 0, 11, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 595, 667, 0, 0, 1, 1489, 0, 214, 0, 30, 139, 434, 0, 1347, 0, 168, 1, 1, 0, 0, 1769, 1, 4039, 1, 1059, 0, 710, 1, 89, 8, 3738, 110, 0, 1, 11, 0, 0, 0, 1017, 24, 379, 903, 0, 0, 1, 30, 3186, 1780, 2512, 4, 82, 0, 24, 17, 16, 2106, 83, 2, 46, 3927, 0, 6, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[3, 2857, 362, 0, 7, 11, 401, 0, 2567, 0, 9, 97, 317, 18, 33, 5, 5, 25, 4, 1, 3, 3892, 0, 2116, 139, 29, 0, 4, 365, 28, 180, 446, 10, 0, 2, 0, 0, 4, 346, 3944, 2, 3793, 2457, 5, 234, 0, 1294, 1246, 803, 518, 0, 812, 0, 1, 9, 2705, 59, 186, 8, 0, 340, 0, 66, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[2, 1729, 103, 0, 0, 0, 1043, 0, 1172, 0, 336, 2, 4, 0, 58, 28, 7, 0, 0, 0, 0, 3489, 0, 3647, 24, 10, 0, 20, 702, 1067, 1, 651, 510, 10, 76, 20, 1, 0, 1, 4004, 16, 139, 1712, 7, 0, 1, 0, 2988, 117, 2818, 0, 544, 6, 0, 5, 2320, 2, 52, 1, 0, 3158, 165, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 3730, 148, 0, 0, 103, 1413, 1, 1683, 0, 744, 28, 71, 17, 182, 0, 0, 0, 1, 1, 1, 3941, 1, 3739, 1, 483, 0, 291, 192, 117, 101, 673, 33, 0, 2795, 1580, 0, 0, 0, 3469, 874, 104, 86, 184, 36, 0, 11, 1702, 0, 2884, 0, 58, 0, 5, 1, 982, 86, 2, 6, 5, 202, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[6, 2680, 125, 0, 0, 15, 164, 0, 2346, 0, 54, 32, 5, 1, 0, 0, 0, 1, 1, 0, 0, 3728, 1, 3574, 1, 34, 3, 1, 265, 2909, 743, 921, 10, 4, 56, 66, 25, 2, 7, 3942, 0, 769, 333, 0, 0, 0, 2, 1809, 1, 1894, 0, 892, 0, 6, 0, 2102, 2, 0, 14, 1, 2683, 160, 378, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 2385, 58, 1, 6, 1, 149, 8, 1963, 0, 10, 66, 8, 2, 63, 1, 0, 0, 1, 0, 1, 4091, 0, 3968, 51, 79, 0, 1932, 641, 221, 269, 1998, 1, 0, 457, 416, 9, 0, 20, 2074, 5, 711, 1376, 0, 8, 2, 43, 2573, 1462, 1228, 0, 1454, 0, 29, 63, 122, 2, 370, 74, 5, 2258, 29, 1, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[7, 1196, 69, 0, 0, 0, 3269, 2, 3856, 7, 2357, 62, 13, 6, 5, 1, 0, 67, 2, 0, 0, 4025, 0, 4043, 0, 429, 0, 349, 114, 35, 30, 94, 77, 0, 81, 26, 0, 0, 0, 961, 5, 164, 3157, 6, 0, 0, 110, 2985, 119, 394, 0, 135, 1, 0, 39, 2757, 5, 2, 3, 1, 1616, 57, 23, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[6, 2680, 125, 0, 0, 15, 164, 0, 292, 156, 0, 0, 0, 0, 0, 0, 2346, 0, 54, 32, 5, 1, 0, 0, 1437, 36, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 632, 1, 3574, 0, 0, 0, 0, 0, 0, 0, 0, 1, 34, 3, 1, 265, 2909, 743, 921, 520, 0, 0, 0, 0, 0, 0, 0, 10, 4, 56, 66, 25, 2, 7, 3942, 274, 0, 0, 0, 0, 0, 0, 0, 0, 769, 333, 0, 0, 0, 2, 961, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1464, 0, 892, 0, 6, 0, 2102, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 14, 1, 2683, 160, 378, 0, 1659, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3438, 3911, 4209, 5397, 4386, 2065, 4465, 4897] avg=4096.0000 max=5397 max_rank=3 min=2065 min_rank=5 imbalance_ratio=1.317627 max_load_reduction=27371 max_load_reduction_pct=0.835297 moved_to_echo=15813 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 2385, 58, 1, 6, 1, 149, 8, 416, 101, 0, 0, 0, 0, 0, 0, 1963, 0, 10, 66, 8, 2, 63, 1, 1577, 52, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 693, 0, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 51, 79, 0, 1932, 641, 221, 269, 1998, 740, 0, 0, 0, 0, 0, 0, 0, 1, 0, 457, 416, 9, 0, 20, 2074, 177, 0, 0, 0, 0, 0, 0, 0, 5, 711, 1376, 0, 8, 2, 43, 1365, 0, 0, 0, 0, 0, 0, 0, 0, 1462, 950, 0, 1454, 0, 29, 63, 122, 0, 0, 0, 0, 0, 0, 0, 0, 2, 370, 74, 5, 2258, 29, 1, 3, 1821, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3125, 3742, 4663, 5931, 3154, 3510, 4080, 4563] avg=4096.0000 max=5931 max_rank=3 min=3125 min_rank=0 imbalance_ratio=1.447998 max_load_reduction=26837 max_load_reduction_pct=0.819000 moved_to_echo=15307 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 595, 667, 0, 0, 1, 1489, 0, 515, 207, 0, 0, 0, 0, 0, 0, 214, 0, 30, 139, 434, 0, 1347, 0, 682, 64, 0, 0, 0, 0, 0, 0, 168, 1, 1, 0, 0, 300, 1, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1059, 0, 710, 1, 89, 8, 3738, 916, 0, 0, 0, 0, 0, 0, 0, 110, 0, 1, 11, 0, 0, 0, 1017, 363, 0, 0, 0, 0, 0, 0, 0, 24, 379, 903, 0, 0, 1, 30, 1691, 0, 0, 0, 0, 0, 0, 0, 0, 1780, 1942, 4, 82, 0, 24, 17, 16, 0, 0, 0, 0, 0, 0, 0, 0, 2106, 83, 2, 46, 3927, 0, 6, 0, 787, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3474, 2910, 4510, 6522, 1502, 3028, 3865, 6957] avg=4096.0000 max=6957 max_rank=7 min=1502 min_rank=4 imbalance_ratio=1.698486 max_load_reduction=25811 max_load_reduction_pct=0.787689 moved_to_echo=15352 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[7, 1196, 69, 0, 0, 0, 3269, 2, 483, 32, 0, 0, 0, 0, 0, 0, 3856, 7, 2357, 62, 13, 6, 5, 1, 1551, 60, 0, 0, 0, 0, 0, 0, 0, 67, 2, 0, 0, 683, 0, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 429, 0, 349, 114, 35, 30, 94, 859, 0, 0, 0, 0, 0, 0, 0, 77, 0, 81, 26, 0, 0, 0, 961, 57, 0, 0, 0, 0, 0, 0, 0, 5, 164, 3157, 6, 0, 0, 110, 1583, 0, 0, 0, 0, 0, 0, 0, 0, 119, 305, 0, 135, 1, 0, 39, 2757, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 3, 1, 1616, 57, 23, 6, 1791, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5058, 7918, 4795, 1910, 1202, 5025, 3356, 3504] avg=4096.0000 max=7918 max_rank=1 min=1202 min_rank=4 imbalance_ratio=1.933105 max_load_reduction=24850 max_load_reduction_pct=0.758362 moved_to_echo=13087 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[63, 1520, 1630, 0, 0, 0, 647, 0, 297, 125, 0, 0, 0, 0, 0, 0, 495, 17, 3, 4, 22, 0, 681, 0, 1167, 37, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 514, 0, 2612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 1636, 11, 1181, 22, 2390, 529, 0, 0, 0, 0, 0, 0, 0, 282, 0, 591, 24, 0, 2, 5, 3692, 220, 0, 0, 0, 0, 0, 0, 0, 260, 42, 2818, 2, 0, 0, 946, 977, 0, 0, 0, 0, 0, 0, 0, 0, 338, 1181, 0, 707, 5, 0, 0, 1601, 0, 0, 0, 0, 0, 0, 0, 0, 986, 166, 61, 0, 893, 0, 11, 0, 1348, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4282, 2426, 3129, 5773, 4816, 5045, 3832, 3465] avg=4096.0000 max=5773 max_rank=3 min=2426 min_rank=1 imbalance_ratio=1.409424 max_load_reduction=26995 max_load_reduction_pct=0.823822 moved_to_echo=17158 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 1729, 103, 0, 0, 0, 1043, 0, 483, 232, 0, 0, 0, 0, 0, 0, 1172, 0, 336, 2, 4, 0, 58, 28, 1345, 60, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 591, 0, 3647, 0, 0, 0, 0, 0, 0, 0, 0, 24, 10, 0, 20, 702, 1067, 1, 651, 859, 0, 0, 0, 0, 0, 0, 0, 510, 10, 76, 20, 1, 0, 1, 4004, 407, 0, 0, 0, 0, 0, 0, 0, 16, 139, 1712, 7, 0, 1, 0, 1586, 0, 0, 0, 0, 0, 0, 0, 0, 117, 2179, 0, 544, 6, 0, 5, 2320, 0, 0, 0, 0, 0, 0, 0, 0, 2, 52, 1, 0, 3158, 165, 0, 0, 1553, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3592, 3005, 4245, 3334, 5029, 3461, 5171, 4931] avg=4096.0000 max=5171 max_rank=6 min=3005 min_rank=1 imbalance_ratio=1.262451 max_load_reduction=27597 max_load_reduction_pct=0.842194 moved_to_echo=18592 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 3730, 148, 0, 0, 103, 1413, 1, 279, 241, 0, 0, 0, 0, 0, 0, 1683, 0, 744, 28, 71, 17, 182, 0, 1523, 37, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 660, 1, 3739, 0, 0, 0, 0, 0, 0, 0, 0, 1, 483, 0, 291, 192, 117, 101, 673, 494, 0, 0, 0, 0, 0, 0, 0, 33, 0, 2795, 1580, 0, 0, 0, 3469, 422, 0, 0, 0, 0, 0, 0, 0, 874, 104, 86, 184, 36, 0, 11, 892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2221, 0, 58, 0, 5, 1, 982, 0, 0, 0, 0, 0, 0, 0, 0, 86, 2, 6, 5, 202, 1, 0, 0, 1758, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5915, 4285, 4403, 2352, 8299, 2187, 3267, 2060] avg=4096.0000 max=8299 max_rank=4 min=2060 min_rank=7 imbalance_ratio=2.026123 max_load_reduction=24469 max_load_reduction_pct=0.746735 moved_to_echo=15813 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3, 2857, 362, 0, 7, 11, 401, 0, 201, 42, 0, 0, 0, 0, 0, 0, 2567, 0, 9, 97, 317, 18, 33, 5, 1500, 25, 0, 0, 0, 0, 0, 0, 5, 25, 4, 1, 3, 660, 0, 2116, 0, 0, 0, 0, 0, 0, 0, 0, 139, 29, 0, 4, 365, 28, 180, 446, 358, 0, 0, 0, 0, 0, 0, 0, 10, 0, 2, 0, 0, 4, 346, 3944, 74, 0, 0, 0, 0, 0, 0, 0, 2, 3793, 2457, 5, 234, 0, 1294, 662, 0, 0, 0, 0, 0, 0, 0, 0, 803, 402, 0, 812, 0, 1, 9, 2705, 0, 0, 0, 0, 0, 0, 0, 0, 59, 186, 8, 0, 340, 0, 66, 0, 1732, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3884, 4571, 2814, 1549, 4380, 8447, 4732, 2391] avg=4096.0000 max=8447 max_rank=5 min=1549 min_rank=3 imbalance_ratio=2.062256 max_load_reduction=24321 max_load_reduction_pct=0.742218 moved_to_echo=19950 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1280, 6144, 12928, 256, 384, 128, 2560, 9728, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[14336, 128, 3584, 512, 896, 128, 2432, 128, 10880, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[256, 2176, 128, 4992, 2304, 5760, 1408, 11008, 5376, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3328, 896, 256, 128, 15104, 512, 512, 128, 12544, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1152, 128, 4096, 2176, 128, 128, 384, 23168, 2048, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4736, 10752, 128, 4736, 128, 128, 256, 12672, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 16768, 3200, 128, 128, 256, 8576, 128, 3072, 1152, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[256, 128, 128, 128, 128, 4736, 128, 27776, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[5, 0, 0, 1735, 5, 0, 2107, 0, 0, 1432, 1897, 0, 0, 0, 7, 279, 7, 1394, 0, 2, 15, 0, 0, 0, 0, 0, 5, 253, 880, 4075, 0, 0, 1443, 219, 28, 16, 0, 535, 2972, 0, 0, 0, 0, 16, 135, 539, 3220, 1, 5, 59, 3776, 0, 62, 0, 0, 3524, 169, 0, 0, 0, 14, 0, 27, 1910] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[10, 0, 8, 3, 0, 2, 0, 0, 745, 0, 2, 1793, 0, 0, 67, 4, 490, 2575, 4060, 0, 0, 5, 0, 0, 0, 0, 0, 1315, 755, 2660, 0, 0, 3790, 3, 255, 0, 36, 3, 1063, 0, 0, 0, 0, 63, 2463, 1, 159, 5, 1, 1628, 0, 0, 592, 0, 0, 4061, 4088, 0, 0, 0, 0, 2, 36, 25] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[31, 2, 0, 0, 0, 0, 0, 1, 0, 2102, 7, 236, 76, 5, 5, 0, 5, 1171, 82, 49, 0, 1, 0, 0, 9, 1, 19, 0, 2269, 0, 0, 9, 3701, 78, 20, 9, 2351, 0, 1407, 0, 0, 0, 0, 0, 3959, 509, 0, 28, 3984, 120, 0, 0, 3526, 6, 0, 3935, 156, 0, 2, 4, 1027, 1866, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[343, 0, 0, 0, 0, 1, 0, 0, 0, 2, 48, 763, 0, 0, 0, 0, 2, 4092, 1650, 6, 0, 4, 0, 0, 0, 0, 0, 25, 1, 9, 0, 1, 3849, 4, 26, 13, 3, 0, 3870, 0, 0, 0, 51, 55, 3715, 0, 20, 45, 1210, 892, 0, 0, 1136, 0, 0, 3749, 2985, 0, 0, 0, 4044, 6, 8, 140] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 95, 0, 2, 0, 0, 2, 0, 46, 2605, 0, 2747, 33, 0, 2, 0, 0, 1529, 8, 16, 2, 0, 0, 20, 30, 2, 7, 74, 85, 2666, 17, 0, 1616, 260, 1, 12, 0, 3, 36, 1, 0, 92, 0, 1539, 3703, 0, 3087, 244, 126, 1081, 821, 0, 1759, 0, 0, 4023, 3992, 0, 0, 0, 1, 39, 344, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[12, 0, 0, 0, 0, 0, 23, 0, 16, 176, 18, 20, 0, 0, 0, 0, 475, 3837, 725, 27, 4, 0, 0, 0, 0, 0, 0, 2378, 13, 4, 0, 278, 3771, 3, 3197, 232, 447, 0, 3796, 0, 0, 2, 0, 3, 586, 0, 0, 0, 4053, 64, 18, 3, 236, 0, 258, 3936, 3113, 4, 0, 0, 954, 33, 48, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[390, 21, 0, 0, 3, 0, 0, 6, 0, 0, 7, 4064, 0, 0, 0, 1, 2, 2772, 1750, 0, 31, 0, 0, 7, 0, 0, 0, 4, 0, 6, 0, 0, 2769, 194, 65, 12, 0, 2, 73, 0, 0, 0, 0, 3258, 3830, 0, 0, 0, 4069, 3639, 0, 0, 4067, 0, 0, 1546, 0, 0, 0, 0, 165, 6, 9, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[152, 0, 0, 0, 2359, 15, 0, 66, 505, 3791, 0, 0, 1, 0, 0, 0, 373, 3729, 631, 0, 12, 1, 0, 0, 79, 0, 1, 0, 0, 0, 186, 18, 3696, 0, 75, 48, 2, 2297, 3883, 0, 0, 0, 0, 0, 4030, 2827, 0, 1, 130, 240, 227, 0, 3209, 1, 24, 145, 11, 0, 1, 0, 0, 0, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 0, 1735, 5, 0, 2107, 0, 3524, 26, 0, 0, 0, 0, 0, 0, 0, 1432, 1897, 0, 0, 0, 7, 279, 20, 253, 0, 0, 0, 0, 0, 0, 7, 1394, 0, 2, 15, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 253, 880, 4075, 0, 0, 857, 0, 0, 0, 0, 0, 0, 0, 307, 219, 28, 16, 0, 535, 2972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 103, 539, 3220, 1, 0, 0, 0, 0, 0, 0, 0, 0, 5, 59, 3776, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 0, 0, 0, 14, 0, 27, 1910, 25, 14, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7402, 3888, 1425, 6070, 4077, 3879, 3868, 2159] avg=4096.0000 max=7402 max_rank=0 min=1425 min_rank=2 imbalance_ratio=1.807129 max_load_reduction=25366 max_load_reduction_pct=0.774109 moved_to_echo=13983 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 8, 3, 0, 2, 0, 0, 4061, 69, 0, 0, 0, 0, 0, 0, 745, 0, 2, 1793, 0, 0, 67, 4, 200, 666, 0, 0, 0, 0, 0, 0, 490, 2575, 4060, 0, 0, 5, 0, 0, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1315, 755, 2660, 0, 0, 2251, 0, 0, 0, 0, 0, 0, 0, 804, 3, 255, 0, 36, 3, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 1869, 1, 159, 5, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1628, 0, 0, 258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 2, 36, 25, 462, 134, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4153, 3477, 7262, 6981, 2164, 2097, 1887, 4747] avg=4096.0000 max=7262 max_rank=2 min=1887 min_rank=6 imbalance_ratio=1.772949 max_load_reduction=25506 max_load_reduction_pct=0.778381 moved_to_echo=10895 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[31, 2, 0, 0, 0, 0, 0, 1, 3935, 67, 0, 0, 0, 0, 0, 0, 0, 2102, 7, 236, 76, 5, 5, 0, 1192, 650, 0, 0, 0, 0, 0, 0, 5, 1171, 82, 49, 0, 1, 0, 0, 213, 0, 0, 0, 0, 0, 0, 0, 9, 1, 19, 0, 2269, 0, 0, 9, 2198, 0, 0, 0, 0, 0, 0, 0, 786, 78, 20, 9, 2351, 0, 1407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3003, 509, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 3984, 120, 0, 0, 1532, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 2, 4, 1027, 1866, 0, 0, 743, 802, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4036, 4273, 1521, 4505, 4651, 3540, 5642, 4600] avg=4096.0000 max=5642 max_rank=6 min=1521 min_rank=2 imbalance_ratio=1.377441 max_load_reduction=27126 max_load_reduction_pct=0.827820 moved_to_echo=18433 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 95, 0, 2, 0, 0, 2, 0, 4023, 29, 0, 0, 0, 0, 0, 0, 46, 2605, 0, 2747, 33, 0, 2, 0, 594, 284, 0, 0, 0, 0, 0, 0, 0, 1529, 8, 16, 2, 0, 0, 20, 199, 0, 0, 0, 0, 0, 0, 0, 30, 2, 7, 74, 85, 2666, 17, 0, 959, 0, 0, 0, 0, 0, 0, 0, 344, 260, 1, 12, 0, 3, 36, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 1539, 2809, 0, 3087, 244, 0, 0, 0, 0, 0, 0, 0, 0, 126, 1081, 821, 0, 765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3992, 0, 0, 0, 1, 39, 344, 0, 695, 400, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4151, 6311, 1774, 3840, 657, 7771, 2793, 5471] avg=4096.0000 max=7771 max_rank=5 min=657 min_rank=4 imbalance_ratio=1.897217 max_load_reduction=24997 max_load_reduction_pct=0.762848 moved_to_echo=16692 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[343, 0, 0, 0, 0, 1, 0, 0, 3749, 70, 0, 0, 0, 0, 0, 0, 0, 2, 48, 763, 0, 0, 0, 0, 384, 676, 0, 0, 0, 0, 0, 0, 2, 4092, 1650, 6, 0, 4, 0, 0, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 1, 9, 0, 1, 2286, 0, 0, 0, 0, 0, 0, 0, 817, 4, 26, 13, 3, 0, 3870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 55, 2819, 0, 20, 45, 0, 0, 0, 0, 0, 0, 0, 0, 1210, 892, 0, 0, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2985, 0, 0, 0, 4044, 6, 8, 140, 697, 258, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4163, 1873, 5953, 2322, 4733, 2990, 2596, 8138] avg=4096.0000 max=8138 max_rank=7 min=1873 min_rank=1 imbalance_ratio=1.986816 max_load_reduction=24630 max_load_reduction_pct=0.751648 moved_to_echo=18457 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[390, 21, 0, 0, 3, 0, 0, 6, 1546, 50, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4064, 0, 0, 0, 1, 1375, 487, 0, 0, 0, 0, 0, 0, 2, 2772, 1750, 0, 31, 0, 0, 7, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 6, 0, 0, 1644, 0, 0, 0, 0, 0, 0, 0, 588, 194, 65, 12, 0, 2, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3258, 2905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 3639, 0, 0, 1767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 6, 9, 0, 719, 925, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2016, 5934, 4768, 1654, 934, 6163, 9475, 1824] avg=4096.0000 max=9475 max_rank=6 min=934 min_rank=4 imbalance_ratio=2.313232 max_load_reduction=23293 max_load_reduction_pct=0.710846 moved_to_echo=18396 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[152, 0, 0, 0, 2359, 15, 0, 66, 145, 67, 0, 0, 0, 0, 0, 0, 505, 3791, 0, 0, 1, 0, 0, 0, 1085, 650, 0, 0, 0, 0, 0, 0, 373, 3729, 631, 0, 12, 1, 0, 0, 216, 0, 0, 0, 0, 0, 0, 0, 79, 0, 1, 0, 0, 0, 186, 18, 2195, 0, 0, 0, 0, 0, 0, 0, 784, 0, 75, 48, 2, 2297, 3883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3058, 2827, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 130, 240, 227, 0, 1394, 1, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 1, 0, 0, 0, 2, 0, 756, 730, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2804, 6032, 4962, 2479, 7089, 5886, 2016, 1500] avg=4096.0000 max=7089 max_rank=4 min=1500 min_rank=7 imbalance_ratio=1.730713 max_load_reduction=25679 max_load_reduction_pct=0.783661 moved_to_echo=16491 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 0, 0, 0, 0, 23, 0, 3936, 72, 0, 0, 0, 0, 0, 0, 16, 176, 18, 20, 0, 0, 0, 0, 83, 667, 0, 0, 0, 0, 0, 0, 475, 3837, 725, 27, 4, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2378, 13, 4, 0, 278, 2244, 0, 0, 0, 0, 0, 0, 0, 788, 3, 3197, 232, 447, 0, 3796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 64, 18, 3, 95, 0, 258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3113, 4, 0, 0, 954, 33, 48, 5, 114, 58, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4043, 980, 5103, 4917, 8463, 442, 4491, 4329] avg=4096.0000 max=8463 max_rank=4 min=442 min_rank=5 imbalance_ratio=2.066162 max_load_reduction=24305 max_load_reduction_pct=0.741730 moved_to_echo=17725 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[5248, 768, 3712, 384, 2944, 2944, 17152, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1408, 10112, 2048, 9728, 128, 128, 128, 384, 4992, 4352, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[13696, 7808, 4864, 128, 6400, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[14592, 128, 128, 128, 6272, 2048, 512, 2176, 4224, 3328, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 4992, 17024, 3968, 6528, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1408, 21120, 8960, 128, 128, 128, 0, 128, 1280, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 4096, 4096, 9472, 256, 384, 14720, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1024, 128, 128, 1792, 2432, 128, 2176, 128, 24960, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 656, 0, 3004, 3902, 0, 2591, 0, 3316, 0, 0, 1, 0, 13, 4078, 21, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 13, 1264, 4096, 835, 2089, 0, 0, 0, 63, 0, 0, 0, 1, 0, 0, 1490, 209, 0, 0, 0, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 728, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 2683, 0, 2629, 4079, 4, 4093, 0, 0, 0, 0, 16, 0, 0, 1621, 0, 163, 16, 149, 0, 0, 1, 25, 0, 0, 150, 2468, 0, 3949, 4, 0, 0, 3983, 23, 4091, 0, 255, 0, 7, 1, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 1492, 0, 0, 0, 844] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3314, 3973, 0, 3979, 0, 2462, 0, 10, 6, 0, 0, 3373, 0, 251, 2, 0, 2, 0, 0, 0, 0, 0, 0, 3957, 0, 0, 2, 0, 3851, 4079, 444, 653, 0, 0, 0, 143, 56, 0, 146, 6, 0, 0, 305, 81, 33, 0, 0, 25, 1295, 0, 158, 0, 0, 0, 0, 159, 1, 0, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 3694, 0, 3944, 0, 4006, 0, 0, 0, 13, 2, 136, 1, 0, 26, 0, 1, 35, 0, 7, 0, 3, 126, 4053, 0, 280, 0, 89, 3, 3981, 0, 4042, 0, 0, 0, 0, 0, 6, 2586, 3924, 4, 0, 0, 0, 38, 0, 19, 0, 361, 1, 3, 132, 0, 0, 0, 871, 92, 8, 0, 277] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3232, 4066, 0, 3753, 0, 7, 0, 0, 15, 0, 4, 2, 826, 1, 0, 0, 235, 8, 0, 0, 0, 0, 0, 3588, 0, 1, 9, 0, 4026, 4096, 2475, 4081, 0, 0, 0, 158, 0, 0, 0, 0, 2, 0, 26, 0, 0, 0, 0, 0, 2006, 0, 7, 0, 0, 0, 0, 3, 0, 0, 0, 141] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 9, 2250, 3769, 0, 3885, 8, 664, 116, 1, 0, 23, 0, 0, 101, 190, 52, 0, 0, 1, 0, 0, 1, 0, 114, 3569, 0, 3535, 0, 0, 17, 3928, 3502, 4082, 1, 807, 0, 368, 2, 0, 0, 0, 0, 5, 626, 0, 0, 0, 0, 571, 499, 0, 0, 69, 0, 0, 0, 0, 3, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 12, 5, 0, 4077, 0, 130, 3, 57, 0, 52, 1, 3905, 0, 1422, 490, 91, 0, 139, 0, 2, 0, 0, 0, 2956, 8, 3903, 2195, 11, 0, 2811, 457, 73, 13, 0, 0, 2448, 67, 2, 802, 0, 22, 0, 1, 0, 275, 3, 0, 0, 0, 4, 18, 0, 0, 4, 52, 871, 2627, 1, 0, 2757] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4090, 3803, 0, 4092, 0, 3886, 0, 0, 4, 0, 0, 1818, 431, 0, 144, 5, 4, 0, 2, 0, 8, 0, 0, 2065, 0, 0, 11, 0, 342, 3653, 0, 4046, 0, 1, 0, 6, 56, 0, 36, 1, 0, 0, 0, 4, 0, 0, 0, 273, 3895, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 656, 0, 3004, 1556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3316, 0, 0, 1, 0, 13, 4078, 461, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 6, 0, 0, 3733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2810, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 1264, 0, 835, 1016, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1490, 209, 1707, 882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 299, 0, 0, 0, 0, 1709, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 728, 0, 0, 1073, 822, 639, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5216, 7869, 3760, 2823, 3178, 4289, 2371, 3262] avg=4096.0000 max=7869 max_rank=1 min=2371 min_rank=6 imbalance_ratio=1.921143 max_load_reduction=24899 max_load_reduction_pct=0.759857 moved_to_echo=13100 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2683, 0, 2629, 1627, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 1621, 278, 0, 0, 0, 0, 0, 0, 0, 0, 163, 16, 149, 0, 0, 1, 25, 3631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 1695, 0, 3949, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 1990, 0, 255, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 1, 1784, 1393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 2700, 352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1492, 0, 0, 0, 844, 2101, 495, 668, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6943, 1915, 3985, 5798, 2275, 3181, 3071, 5600] avg=4096.0000 max=6943 max_rank=0 min=1915 min_rank=1 imbalance_ratio=1.695068 max_load_reduction=25825 max_load_reduction_pct=0.788116 moved_to_echo=14127 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 1473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 0, 0, 0, 13, 2, 136, 457, 0, 0, 0, 0, 0, 0, 0, 1, 0, 26, 0, 1, 35, 0, 7, 3628, 0, 0, 0, 0, 0, 0, 0, 0, 3, 126, 2782, 0, 280, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2586, 3924, 4, 0, 0, 0, 1616, 1343, 0, 0, 0, 0, 0, 0, 38, 0, 19, 0, 361, 1, 3, 132, 2601, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 871, 92, 8, 0, 277, 2076, 814, 605, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1477, 4614, 3698, 3280, 1969, 9479, 3508, 4743] avg=4096.0000 max=9479 max_rank=5 min=1477 min_rank=0 imbalance_ratio=2.314209 max_load_reduction=23289 max_load_reduction_pct=0.710724 moved_to_echo=19699 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3232, 1622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 15, 0, 4, 2, 404, 0, 0, 0, 0, 0, 0, 0, 826, 1, 0, 0, 235, 8, 0, 0, 3734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2464, 0, 1, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 0, 2475, 1985, 0, 0, 0, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 26, 0, 1778, 1278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2006, 0, 7, 0, 2475, 362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 141, 2096, 720, 666, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4854, 432, 4804, 2474, 8644, 3084, 4850, 3626] avg=4096.0000 max=8644 max_rank=4 min=432 min_rank=1 imbalance_ratio=2.110352 max_load_reduction=24124 max_load_reduction_pct=0.736206 moved_to_echo=20204 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3314, 1585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2462, 0, 10, 6, 0, 0, 3373, 446, 0, 0, 0, 0, 0, 0, 0, 0, 251, 2, 0, 2, 0, 0, 0, 3719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2717, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3851, 0, 444, 318, 0, 0, 0, 143, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 146, 6, 0, 0, 305, 81, 1738, 1356, 0, 0, 0, 0, 0, 0, 33, 0, 0, 25, 1295, 0, 158, 0, 2623, 360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 159, 1, 0, 0, 2, 335, 794, 650, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4899, 6297, 3974, 2719, 4756, 3688, 4494, 1941] avg=4096.0000 max=6297 max_rank=1 min=1941 min_rank=7 imbalance_ratio=1.537354 max_load_reduction=26471 max_load_reduction_pct=0.807831 moved_to_echo=14879 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 9, 2250, 1504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 664, 116, 1, 0, 23, 0, 0, 402, 0, 0, 0, 0, 0, 0, 0, 101, 190, 52, 0, 0, 1, 0, 0, 3580, 0, 0, 0, 0, 0, 0, 0, 1, 0, 114, 2450, 0, 3535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 3502, 1985, 1, 807, 0, 368, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 5, 626, 0, 1648, 1323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 571, 499, 0, 0, 69, 2562, 348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 2097, 717, 617, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3763, 1214, 3924, 6100, 6680, 3604, 4049, 3434] avg=4096.0000 max=6680 max_rank=4 min=1214 min_rank=1 imbalance_ratio=1.630859 max_load_reduction=26088 max_load_reduction_pct=0.796143 moved_to_echo=17767 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 12, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 3, 57, 0, 52, 1, 3905, 333, 0, 0, 0, 0, 0, 0, 0, 0, 1422, 490, 91, 0, 139, 0, 2, 2563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2030, 8, 3903, 2195, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 457, 36, 13, 0, 0, 2448, 0, 0, 0, 0, 0, 0, 0, 0, 67, 2, 802, 0, 22, 0, 1, 0, 2, 1389, 0, 0, 0, 0, 0, 0, 275, 3, 0, 0, 0, 4, 18, 0, 2688, 248, 0, 0, 0, 0, 0, 0, 0, 4, 52, 871, 2627, 1, 0, 2757, 37, 593, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16, 4481, 4707, 8147, 2954, 2285, 3236, 6942] avg=4096.0000 max=8147 max_rank=3 min=16 min_rank=0 imbalance_ratio=1.989014 max_load_reduction=24621 max_load_reduction_pct=0.751373 moved_to_echo=15417 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4090, 1510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3886, 0, 0, 4, 0, 0, 1818, 238, 0, 0, 0, 0, 0, 0, 0, 431, 0, 144, 5, 4, 0, 2, 0, 3330, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 1408, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 342, 0, 0, 1963, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 36, 1, 0, 0, 0, 4, 1667, 1394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 273, 3895, 0, 0, 0, 2698, 323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 2, 2083, 419, 626, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5600, 5946, 3916, 1427, 2312, 3158, 7189, 3220] avg=4096.0000 max=7189 max_rank=6 min=1427 min_rank=3 imbalance_ratio=1.755127 max_load_reduction=25579 max_load_reduction_pct=0.780609 moved_to_echo=15879 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[256, 128, 3584, 3968, 128, 128, 2560, 384, 12032, 10368, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 3456, 128, 18560, 10880, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[9600, 0, 7808, 11264, 128, 1152, 0, 3200, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 3584, 2816, 768, 0, 4096, 11904, 5376, 4480, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 512, 18432, 128, 11776, 2304, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 14592, 128, 128, 128, 128, 128, 14976, 3072, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[384, 128, 128, 1280, 8064, 128, 256, 256, 20096, 2816, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1408, 2048, 768, 256, 256, 256, 128, 128, 28032, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[34, 0, 1, 1, 2, 0, 0, 0, 0, 0, 56, 13, 0, 0, 0, 4013, 0, 251, 4011, 4095, 0, 200, 4096, 0, 0, 0, 691, 3807, 0, 0, 0, 1275, 0, 1, 0, 0, 0, 0, 0, 1, 3054, 3, 43, 1, 0, 0, 166, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2375, 0, 0, 12, 483, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 41, 4024, 2, 3, 1, 0, 5, 0, 8, 0, 0, 109, 0, 125, 0, 652, 3865, 4092, 141, 4, 3071, 0, 0, 0, 53, 3443, 0, 0, 0, 4008, 0, 0, 0, 365, 0, 0, 4, 0, 4, 0, 77, 0, 202, 0, 3553, 2, 814, 2, 28, 0, 0, 2, 0, 0, 0, 0, 4066, 0, 0, 2, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[18, 6, 4093, 168, 0, 0, 0, 0, 1587, 0, 1778, 15, 0, 0, 0, 4, 6, 701, 4, 4096, 0, 3377, 1547, 0, 0, 0, 1, 3617, 0, 0, 11, 9, 0, 2, 0, 85, 0, 0, 0, 1, 3989, 4, 0, 39, 0, 0, 7, 1058, 107, 15, 0, 2323, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 7, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[341, 0, 3547, 9, 0, 10, 20, 48, 2200, 0, 0, 125, 0, 0, 0, 0, 0, 999, 159, 3226, 0, 219, 4058, 0, 0, 0, 0, 1044, 0, 0, 0, 4017, 34, 0, 0, 0, 263, 132, 0, 4, 0, 2, 4, 21, 0, 0, 3421, 0, 0, 7, 0, 273, 781, 1, 10, 0, 0, 0, 3776, 0, 0, 3948, 0, 69] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[13, 0, 0, 14, 48, 44, 0, 0, 104, 54, 0, 0, 0, 0, 0, 54, 1, 197, 1877, 3969, 2102, 0, 2824, 0, 0, 3, 80, 2661, 0, 45, 10, 2951, 0, 0, 268, 0, 8, 0, 0, 0, 800, 0, 35, 699, 8, 0, 3991, 47, 744, 237, 84, 926, 20, 0, 33, 0, 464, 0, 3723, 0, 0, 3628, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[9, 1, 8, 0, 0, 807, 0, 0, 3109, 0, 0, 120, 0, 0, 0, 0, 0, 1243, 274, 4091, 7, 3372, 2625, 146, 0, 0, 2, 3904, 0, 0, 1, 2409, 0, 0, 26, 12, 0, 0, 9, 0, 1010, 0, 0, 0, 0, 0, 15, 104, 0, 110, 1, 1332, 326, 0, 0, 2, 0, 0, 3710, 0, 0, 3983, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 5, 3667, 0, 3987, 61, 0, 625, 3678, 203, 12, 0, 0, 19, 0, 0, 467, 0, 28, 4041, 0, 16, 3457, 0, 0, 0, 293, 1894, 0, 0, 11, 3526, 0, 0, 15, 0, 0, 1903, 3, 0, 68, 0, 404, 0, 1, 0, 0, 278, 0, 530, 0, 43, 0, 62, 280, 0, 0, 0, 0, 98, 0, 3093, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 3715, 4061, 0, 0, 0, 3, 0, 7, 0, 0, 0, 0, 32, 0, 0, 0, 1, 2931, 4090, 966, 3775, 678, 0, 0, 0, 4, 3980, 0, 0, 0, 207, 0, 0, 1, 9, 3322, 0, 43, 0, 4, 0, 0, 75, 0, 0, 36, 5, 243, 20, 1, 241, 7, 0, 0, 0, 0, 0, 4083, 0, 0, 228, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 41, 4024, 2, 3, 1, 0, 524, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8, 0, 0, 109, 0, 125, 1582, 530, 45, 0, 0, 0, 0, 0, 0, 652, 3865, 0, 141, 4, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 1861, 0, 0, 0, 4008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 365, 0, 0, 4, 0, 3389, 0, 0, 0, 0, 0, 0, 0, 4, 0, 77, 0, 202, 0, 3553, 2, 858, 0, 0, 0, 0, 0, 0, 0, 814, 2, 28, 0, 0, 2, 0, 0, 2434, 703, 0, 0, 0, 0, 0, 0, 0, 0, 2639, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4595, 2404, 4769, 5922, 3758, 4696, 3983, 2641] avg=4096.0000 max=5922 max_rank=3 min=2404 min_rank=1 imbalance_ratio=1.445801 max_load_reduction=26846 max_load_reduction_pct=0.819275 moved_to_echo=15078 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[34, 0, 1, 1, 2, 0, 0, 0, 306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 13, 0, 0, 0, 4013, 1749, 707, 26, 0, 0, 0, 0, 0, 0, 251, 4011, 0, 0, 200, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 691, 2058, 0, 0, 0, 1275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 3391, 0, 0, 0, 0, 0, 0, 0, 3054, 3, 43, 1, 0, 0, 166, 4083, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3246, 704, 0, 0, 0, 0, 0, 0, 0, 0, 1542, 0, 0, 12, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[344, 6564, 4605, 4024, 3393, 7851, 3950, 2037] avg=4096.0000 max=7851 max_rank=5 min=344 min_rank=0 imbalance_ratio=1.916748 max_load_reduction=24917 max_load_reduction_pct=0.760406 moved_to_echo=17231 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[18, 6, 4093, 168, 0, 0, 0, 0, 527, 0, 0, 0, 0, 0, 0, 0, 1587, 0, 1778, 15, 0, 0, 0, 4, 1662, 267, 45, 0, 0, 0, 0, 0, 6, 701, 4, 0, 0, 3377, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1955, 0, 0, 11, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 85, 0, 0, 0, 1, 3393, 0, 0, 0, 0, 0, 0, 0, 3989, 4, 0, 39, 0, 0, 7, 1058, 864, 0, 0, 0, 0, 0, 0, 0, 107, 15, 0, 2323, 0, 0, 0, 0, 1226, 703, 0, 0, 0, 0, 0, 0, 0, 0, 2656, 0, 0, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4812, 5358, 4142, 1976, 3481, 5961, 4374, 2664] avg=4096.0000 max=5961 max_rank=5 min=1976 min_rank=3 imbalance_ratio=1.455322 max_load_reduction=26807 max_load_reduction_pct=0.818085 moved_to_echo=16480 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[341, 0, 3547, 9, 0, 10, 20, 48, 487, 0, 0, 0, 0, 0, 0, 0, 2200, 0, 0, 125, 0, 0, 0, 0, 479, 700, 42, 0, 0, 0, 0, 0, 0, 999, 159, 0, 0, 219, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 565, 0, 0, 0, 4017, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 263, 132, 0, 4, 2672, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 21, 0, 0, 3421, 0, 797, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 273, 781, 1, 10, 0, 3216, 554, 0, 0, 0, 0, 0, 0, 0, 0, 2450, 0, 0, 3948, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4462, 3546, 1519, 4582, 3105, 4245, 4842, 6467] avg=4096.0000 max=6467 max_rank=7 min=1519 min_rank=2 imbalance_ratio=1.578857 max_load_reduction=26301 max_load_reduction_pct=0.802643 moved_to_echo=18659 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[13, 0, 0, 14, 48, 44, 0, 0, 480, 0, 0, 0, 0, 0, 0, 0, 104, 54, 0, 0, 0, 0, 0, 54, 1222, 487, 41, 0, 0, 0, 0, 0, 1, 197, 1877, 0, 2102, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 80, 1439, 0, 45, 10, 2951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 268, 0, 8, 0, 0, 0, 3287, 0, 0, 0, 0, 0, 0, 0, 800, 0, 35, 699, 8, 0, 3991, 47, 786, 0, 0, 0, 0, 0, 0, 0, 744, 237, 84, 926, 20, 0, 33, 0, 2238, 682, 0, 0, 0, 0, 0, 0, 464, 0, 2416, 0, 0, 3628, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[599, 1962, 4276, 4528, 3563, 6366, 4964, 6510] avg=4096.0000 max=6510 max_rank=7 min=599 min_rank=0 imbalance_ratio=1.589355 max_load_reduction=26258 max_load_reduction_pct=0.801331 moved_to_echo=21403 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 3667, 0, 3987, 61, 0, 625, 0, 0, 0, 0, 0, 0, 0, 0, 3678, 203, 12, 0, 0, 19, 0, 0, 870, 596, 0, 0, 0, 0, 0, 0, 467, 0, 28, 0, 0, 16, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 1024, 0, 0, 11, 3526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 1903, 3, 0, 3348, 0, 0, 0, 0, 0, 0, 0, 68, 0, 404, 0, 1, 0, 0, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 0, 43, 0, 62, 280, 0, 2740, 693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 3093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8345, 5378, 632, 4854, 5269, 751, 4348, 3191] avg=4096.0000 max=8345 max_rank=0 min=632 min_rank=2 imbalance_ratio=2.037354 max_load_reduction=24423 max_load_reduction_pct=0.745331 moved_to_echo=13559 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[9, 1, 8, 0, 0, 807, 0, 0, 478, 0, 0, 0, 0, 0, 0, 0, 3109, 0, 0, 120, 0, 0, 0, 0, 1793, 453, 41, 0, 0, 0, 0, 0, 0, 1243, 274, 0, 7, 3372, 92, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2111, 0, 0, 1, 2409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 12, 0, 0, 9, 0, 3389, 0, 0, 0, 0, 0, 0, 0, 1010, 0, 0, 0, 0, 0, 15, 104, 783, 0, 0, 0, 0, 0, 0, 0, 0, 110, 1, 1332, 326, 0, 0, 2, 2080, 702, 0, 0, 0, 0, 0, 0, 0, 0, 2408, 0, 0, 3983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1303, 5516, 5134, 4523, 3436, 1912, 4553, 6391] avg=4096.0000 max=6391 max_rank=7 min=1303 min_rank=0 imbalance_ratio=1.560303 max_load_reduction=26377 max_load_reduction_pct=0.804962 moved_to_echo=16292 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 3715, 4061, 0, 0, 0, 3, 0, 529, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 32, 0, 0, 1832, 120, 49, 0, 0, 0, 0, 0, 0, 1, 2931, 0, 966, 3775, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2148, 0, 0, 0, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 9, 3322, 0, 43, 0, 3388, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 75, 0, 0, 36, 5, 866, 0, 0, 0, 0, 0, 0, 0, 243, 20, 1, 241, 7, 0, 0, 0, 540, 702, 0, 0, 0, 0, 0, 0, 0, 0, 2639, 0, 0, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8308, 2040, 7691, 2359, 6763, 986, 1754, 2867] avg=4096.0000 max=8308 max_rank=0 min=986 min_rank=5 imbalance_ratio=2.028320 max_load_reduction=24460 max_load_reduction_pct=0.746460 moved_to_echo=12370 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[10752, 384, 1920, 384, 0, 256, 0, 4224, 11264, 3968, 384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[512, 3840, 15488, 4224, 4096, 1024, 128, 768, 3456, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8960, 128, 640, 896, 256, 0, 11264, 5632, 5504, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1920, 1024, 128, 5248, 1152, 128, 384, 128, 17792, 5504, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 1152, 13184, 0, 128, 128, 18432, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[512, 0, 16768, 128, 0, 14976, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[512, 4096, 13184, 0, 3328, 11008, 896, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 384, 512, 3712, 2048, 128, 128, 26368, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 30, 3780, 3875, 0, 0, 4092, 0, 11, 3, 0, 0, 8, 0, 514, 0, 0, 2863, 0, 4060, 0, 0, 1480, 8, 59, 6, 92, 0, 0, 0, 0, 616, 2, 0, 0, 0, 795, 3546, 0, 0, 17, 1, 0, 34, 1758, 1, 0, 1, 9, 0, 1, 1, 4081, 0, 0, 194, 0, 390, 0, 0, 0, 0, 440, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 7, 816, 3938, 0, 0, 4015, 0, 19, 0, 0, 0, 0, 0, 0, 205, 0, 2, 198, 4060, 186, 0, 0, 0, 0, 1, 0, 0, 8, 583, 0, 2450, 0, 0, 0, 1500, 4096, 0, 0, 0, 0, 15, 168, 0, 712, 362, 0, 1391, 4008, 40, 0, 13, 0, 0, 15, 0, 0, 8, 0, 1, 0, 0, 0, 3951] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 87, 3622, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 2, 530, 4066, 2438, 0, 1, 0, 0, 4066, 323, 0, 8, 0, 0, 32, 0, 0, 0, 0, 4059, 0, 0, 13, 0, 34, 0, 6, 3279, 0, 0, 0, 4057, 0, 2193, 23, 0, 0, 0, 3715, 3, 0, 4, 14, 60, 0, 0, 28] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 46, 0, 4064, 0, 0, 2764, 0, 0, 13, 0, 0, 0, 0, 10, 0, 1, 0, 5, 0, 0, 34, 253, 0, 1432, 3792, 4084, 1, 58, 0, 0, 0, 0, 59, 0, 221, 3776, 131, 0, 64, 0, 9, 1, 0, 165, 0, 0, 0, 3941, 0, 1837, 0, 0, 5, 0, 1565, 0, 222, 52, 118, 0, 0, 0, 4045] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1, 0, 2, 3584, 0, 0, 4082, 2, 0, 47, 8, 0, 5, 0, 18, 0, 13, 379, 1, 3868, 0, 0, 0, 0, 636, 144, 52, 0, 5, 1956, 0, 37, 0, 0, 0, 0, 3582, 2851, 0, 0, 0, 30, 0, 1, 0, 0, 4, 2534, 4028, 0, 0, 424, 0, 0, 6, 2521, 2, 8, 0, 0, 0, 0, 0, 1937] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 3, 1, 3930, 1, 0, 4031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3804, 0, 0, 1409, 0, 0, 0, 0, 3421, 1088, 0, 3416, 2867, 0, 4, 0, 0, 0, 388, 4090, 86, 0, 0, 0, 0, 2, 10, 1372, 0, 0, 0, 5, 0, 584, 30, 0, 0, 0, 0, 0, 290, 0, 0, 0, 0, 0, 1936] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 2878, 3868, 65, 0, 2742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 2203, 17, 0, 0, 0, 662, 1, 4092, 0, 0, 0, 0, 353, 0, 0, 1, 0, 3655, 0, 0, 2, 0, 0, 0, 0, 148, 0, 0, 0, 3289, 0, 4069, 0, 0, 0, 0, 68, 6, 54, 245, 209, 200, 0, 3, 3923] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 4079, 3, 0, 3943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 129, 0, 0, 11, 0, 282, 4093, 0, 4069, 9, 0, 1, 0, 0, 133, 3986, 3989, 34, 0, 451, 0, 0, 0, 594, 0, 0, 0, 0, 4096, 0, 191, 1, 0, 0, 0, 119, 10, 225, 0, 14, 64, 0, 0, 2053] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 87, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3622, 163, 0, 0, 0, 0, 0, 0, 102, 2, 530, 4066, 2438, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 43, 0, 8, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2794, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 6, 3279, 0, 0, 0, 1976, 203, 0, 0, 0, 0, 0, 0, 1918, 0, 2193, 23, 0, 0, 0, 3715, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4, 14, 60, 0, 0, 28, 1265, 77, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[90, 3785, 7139, 4149, 2807, 5498, 7849, 1451] avg=4096.0000 max=7849 max_rank=6 min=90 min_rank=0 imbalance_ratio=1.916260 max_load_reduction=24919 max_load_reduction_pct=0.760468 moved_to_echo=17605 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 816, 0, 0, 0, 3919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 205, 3938, 161, 0, 0, 0, 0, 0, 0, 0, 2, 198, 4060, 186, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8, 583, 0, 2450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1500, 2820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 168, 0, 712, 362, 0, 1391, 1952, 0, 0, 0, 0, 0, 0, 0, 1895, 40, 0, 13, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 1, 0, 0, 0, 3951, 1276, 0, 28, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4742, 4323, 4514, 3042, 4320, 4600, 1963, 5264] avg=4096.0000 max=5264 max_rank=7 min=1963 min_rank=6 imbalance_ratio=1.285156 max_load_reduction=27504 max_load_reduction_pct=0.839355 moved_to_echo=16147 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 30, 3780, 0, 0, 0, 3993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 3, 0, 0, 8, 0, 514, 0, 3875, 0, 0, 0, 0, 0, 0, 0, 0, 2863, 0, 4060, 0, 0, 1480, 8, 70, 0, 0, 0, 0, 0, 0, 0, 59, 6, 13, 0, 0, 0, 0, 616, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 548, 3546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 1, 0, 34, 1758, 1, 0, 1, 4, 57, 0, 0, 0, 0, 0, 0, 5, 0, 1, 1, 4081, 0, 0, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 390, 0, 0, 0, 0, 440, 0, 247, 22, 29, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7803, 4411, 8481, 694, 4096, 1873, 4282, 1128] avg=4096.0000 max=8481 max_rank=2 min=694 min_rank=3 imbalance_ratio=2.070557 max_load_reduction=24287 max_load_reduction_pct=0.741180 moved_to_echo=11379 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 46, 0, 0, 0, 0, 2698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 10, 0, 4064, 159, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 34, 253, 0, 47, 0, 0, 0, 0, 0, 0, 0, 1432, 3792, 531, 1, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 221, 2599, 131, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 0, 165, 0, 0, 0, 1919, 2571, 0, 0, 0, 0, 0, 0, 1863, 0, 1837, 0, 0, 5, 0, 1565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 222, 52, 118, 0, 0, 0, 4045, 1177, 982, 19, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2744, 4246, 340, 5814, 3074, 4665, 5270, 6615] avg=4096.0000 max=6615 max_rank=7 min=340 min_rank=2 imbalance_ratio=1.614990 max_load_reduction=26153 max_load_reduction_pct=0.798126 moved_to_echo=19624 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 1, 0, 1, 0, 3933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3930, 0, 0, 0, 0, 0, 0, 0, 0, 3804, 0, 0, 1409, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 3421, 142, 0, 3416, 2867, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 388, 2815, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 10, 1372, 0, 0, 0, 2, 685, 0, 0, 0, 0, 0, 0, 3, 0, 584, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 290, 0, 0, 0, 0, 0, 1936, 1275, 261, 29, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3938, 3930, 5282, 9850, 3289, 2071, 617, 3791] avg=4096.0000 max=9850 max_rank=3 min=617 min_rank=6 imbalance_ratio=2.404785 max_load_reduction=22918 max_load_reduction_pct=0.699402 moved_to_echo=9768 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 2, 0, 0, 0, 3983, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 8, 0, 5, 0, 18, 0, 3584, 162, 0, 0, 0, 0, 0, 0, 13, 379, 1, 3868, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 636, 144, 8, 0, 5, 1956, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2466, 2851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 1, 0, 0, 4, 2534, 1962, 32, 0, 0, 0, 0, 0, 0, 1904, 0, 0, 424, 0, 0, 6, 2521, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8, 0, 0, 0, 0, 0, 1937, 1116, 12, 29, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3988, 3824, 4331, 2786, 5317, 4563, 4855, 3104] avg=4096.0000 max=5317 max_rank=4 min=2786 min_rank=3 imbalance_ratio=1.298096 max_load_reduction=27451 max_load_reduction_pct=0.837738 moved_to_echo=17839 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2878, 0, 65, 0, 2676, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3868, 132, 0, 0, 0, 0, 0, 0, 0, 0, 15, 2203, 17, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 662, 1, 532, 0, 0, 0, 0, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2516, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 0, 0, 0, 1602, 2576, 0, 0, 0, 0, 0, 0, 1555, 0, 4069, 0, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 6, 54, 245, 209, 200, 0, 3, 3923, 1139, 984, 19, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5619, 4000, 2282, 1548, 2519, 4326, 5692, 6782] avg=4096.0000 max=6782 max_rank=7 min=1548 min_rank=3 imbalance_ratio=1.655762 max_load_reduction=25986 max_load_reduction_pct=0.793030 moved_to_echo=19319 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 0, 3841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 129, 0, 0, 11, 70, 0, 0, 0, 0, 0, 0, 0, 0, 282, 524, 0, 4069, 9, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 133, 3986, 2742, 34, 0, 451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 594, 0, 0, 0, 0, 1997, 2581, 0, 0, 0, 0, 0, 0, 1929, 0, 191, 1, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 10, 225, 0, 14, 64, 0, 0, 2053, 1247, 988, 32, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3844, 4249, 399, 4885, 7346, 5172, 2240, 4633] avg=4096.0000 max=7346 max_rank=4 min=399 min_rank=2 imbalance_ratio=1.793457 max_load_reduction=25422 max_load_reduction_pct=0.775818 moved_to_echo=19391 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 256, 768, 7552, 384, 128, 3968, 11520, 8832, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[11136, 128, 8960, 512, 4096, 128, 128, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 7168, 768, 18560, 4224, 128, 1792, 128, 512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[2816, 11776, 1920, 128, 7680, 5504, 0, 3584, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 1280, 384, 384, 384, 0, 512, 17920, 8832, 3328, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 7680, 0, 128, 128, 25088, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 256, 6144, 19328, 6656, 0, 640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 128, 0, 640, 256, 30976, 1024, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 3734, 242, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 1995, 1337, 4094, 0, 0, 0, 0, 0, 3395, 0, 0, 0, 0, 4093, 0, 0, 2460, 959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3993, 573, 190, 2, 1, 3, 0, 0, 0, 0, 2624, 0, 0, 2441, 150, 0, 0, 5, 445, 0, 1, 0, 0, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[127, 4092, 0, 0, 0, 11, 0, 0, 0, 0, 1, 1, 25, 3571, 16, 4091, 236, 0, 0, 0, 0, 212, 0, 0, 1, 6, 4054, 129, 0, 3972, 0, 0, 14, 0, 0, 1, 45, 0, 0, 2602, 252, 0, 3713, 0, 12, 887, 0, 0, 0, 0, 155, 12, 0, 3345, 1185, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 3999, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3883, 14, 0, 4078, 77, 0, 0, 0, 0, 0, 143, 0, 0, 121, 4046, 23, 36, 2, 0, 295, 0, 0, 0, 0, 4083, 0, 0, 2, 3743, 0, 1, 0, 5, 0, 10, 0, 0, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 55, 0, 0, 0, 0, 0, 258, 0, 318, 0, 0, 1, 4029, 1, 857, 0, 4, 3, 0, 76, 0, 95, 0, 0, 53, 3954, 28, 0, 3887, 0, 0, 302, 1, 0, 2, 4004, 116, 0, 0, 2917, 0, 3903, 0, 3018, 0, 0, 575, 2, 0, 0, 3, 26, 7, 4091, 0, 0, 1, 168, 0, 12, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[9, 1106, 0, 0, 0, 0, 0, 0, 230, 8, 53, 0, 0, 4031, 36, 4061, 14, 0, 1, 0, 0, 8, 0, 0, 0, 1, 2692, 16, 0, 4087, 769, 0, 2, 0, 0, 0, 3533, 10, 0, 127, 2103, 0, 2533, 6, 0, 2054, 150, 0, 12, 1, 0, 0, 20, 7, 4037, 0, 0, 0, 113, 0, 0, 938, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 3303, 0, 0, 0, 7, 0, 0, 0, 304, 5, 1, 48, 3970, 21, 4096, 0, 0, 0, 0, 0, 0, 77, 0, 1, 4, 110, 232, 0, 4071, 24, 0, 0, 0, 0, 0, 190, 0, 0, 0, 4058, 0, 2, 0, 7, 1275, 0, 0, 0, 0, 3925, 0, 0, 1035, 2832, 0, 0, 0, 0, 0, 0, 3170, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 3657, 0, 0, 0, 380, 0, 0, 0, 38, 0, 0, 4, 3765, 1, 4077, 0, 0, 120, 14, 9, 0, 48, 0, 105, 0, 545, 0, 0, 4096, 434, 0, 0, 249, 3, 0, 4081, 0, 0, 68, 1, 41, 3946, 2, 0, 7, 0, 12, 0, 0, 3976, 0, 0, 204, 2851, 0, 21, 10, 0, 0, 3, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 2618, 0, 0, 0, 67, 0, 0, 0, 0, 1, 1, 6, 0, 4, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 3, 1, 3912, 2, 0, 4077, 31, 0, 17, 617, 0, 49, 111, 0, 5, 936, 3900, 6, 25, 23, 105, 4036, 0, 0, 0, 0, 4087, 0, 0, 0, 4031, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[127, 4092, 0, 0, 0, 11, 0, 0, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 25, 3571, 16, 676, 0, 0, 0, 0, 0, 0, 0, 0, 236, 0, 0, 0, 0, 212, 0, 0, 3415, 499, 0, 0, 0, 0, 0, 0, 1, 6, 4054, 129, 0, 858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 1, 45, 0, 0, 2602, 439, 81, 0, 0, 0, 0, 0, 0, 64, 0, 3713, 0, 12, 887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 12, 0, 3345, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2615, 393, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4337, 4290, 4362, 5048, 3182, 4676, 3865, 3008] avg=4096.0000 max=5048 max_rank=3 min=3008 min_rank=7 imbalance_ratio=1.232422 max_load_reduction=27720 max_load_reduction_pct=0.845947 moved_to_echo=14731 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3734, 242, 0, 0, 27, 0, 0, 1698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1995, 1337, 677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3395, 0, 0, 3417, 309, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 531, 959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 1298, 0, 0, 0, 0, 0, 0, 997, 573, 190, 2, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2624, 0, 0, 2441, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 445, 0, 1, 0, 0, 4, 1620, 49, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5701, 4009, 7121, 5583, 1353, 1766, 5111, 2124] avg=4096.0000 max=7121 max_rank=2 min=1353 min_rank=4 imbalance_ratio=1.738525 max_load_reduction=25647 max_load_reduction_pct=0.782684 moved_to_echo=10354 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[9, 1106, 0, 0, 0, 0, 0, 0, 894, 0, 0, 0, 0, 0, 0, 0, 230, 8, 53, 0, 0, 4031, 36, 671, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 1, 0, 0, 8, 0, 0, 3390, 513, 0, 0, 0, 0, 0, 0, 0, 1, 2692, 16, 0, 883, 769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3533, 10, 0, 127, 1497, 683, 0, 0, 0, 0, 0, 0, 526, 0, 2533, 6, 0, 2054, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 0, 0, 20, 7, 1199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 0, 0, 938, 0, 0, 2691, 1341, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2009, 5029, 3926, 4361, 5852, 5269, 1239, 5083] avg=4096.0000 max=5852 max_rank=4 min=1239 min_rank=6 imbalance_ratio=1.428711 max_load_reduction=26916 max_load_reduction_pct=0.821411 moved_to_echo=17443 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3999, 0, 0, 0, 29, 0, 0, 1720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3411, 512, 0, 0, 0, 0, 0, 0, 0, 6, 3883, 14, 0, 881, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 0, 121, 1388, 1315, 0, 0, 0, 0, 0, 0, 1011, 23, 36, 2, 0, 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 2, 1111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 10, 0, 0, 5, 2685, 1244, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5748, 4756, 3923, 4861, 2967, 1367, 5196, 3950] avg=4096.0000 max=5748 max_rank=0 min=1367 min_rank=5 imbalance_ratio=1.403320 max_load_reduction=27020 max_load_reduction_pct=0.824585 moved_to_echo=13480 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 55, 0, 0, 0, 0, 0, 258, 1240, 0, 0, 0, 0, 0, 0, 0, 0, 318, 0, 0, 1, 4029, 1, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 0, 76, 0, 95, 0, 715, 488, 0, 0, 0, 0, 0, 0, 0, 53, 3954, 28, 0, 840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 302, 1, 0, 2, 4004, 116, 0, 0, 1517, 948, 0, 0, 0, 0, 0, 0, 729, 0, 3903, 0, 3018, 0, 0, 575, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3, 26, 7, 1215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 168, 0, 12, 1, 0, 0, 2559, 1359, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1553, 4491, 1381, 4875, 6890, 8225, 1253, 4100] avg=4096.0000 max=8225 max_rank=5 min=1253 min_rank=6 imbalance_ratio=2.008057 max_load_reduction=24543 max_load_reduction_pct=0.748993 moved_to_echo=20468 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3303, 0, 0, 0, 7, 0, 0, 1726, 0, 0, 0, 0, 0, 0, 0, 0, 304, 5, 1, 48, 3970, 21, 677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 3419, 511, 0, 0, 0, 0, 0, 0, 1, 4, 110, 232, 0, 879, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 190, 0, 0, 0, 1050, 1319, 0, 0, 0, 0, 0, 0, 1013, 0, 2, 0, 7, 1275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3925, 0, 0, 1035, 841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3170, 0, 0, 2681, 941, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5036, 5026, 4007, 1250, 2559, 2297, 5801, 6792] avg=4096.0000 max=6792 max_rank=7 min=1250 min_rank=3 imbalance_ratio=1.658203 max_load_reduction=25976 max_load_reduction_pct=0.792725 moved_to_echo=17449 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3657, 0, 0, 0, 380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 4, 3765, 1, 674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 14, 9, 0, 48, 0, 3403, 514, 0, 0, 0, 0, 0, 0, 105, 0, 545, 0, 0, 885, 434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 249, 3, 0, 4081, 0, 0, 68, 1057, 0, 0, 0, 0, 0, 0, 0, 1, 41, 3946, 2, 0, 7, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3976, 0, 0, 204, 847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 10, 0, 0, 3, 0, 0, 0, 2697, 947, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4037, 4482, 4108, 1969, 5458, 4009, 5027, 3678] avg=4096.0000 max=5458 max_rank=4 min=1969 min_rank=3 imbalance_ratio=1.332520 max_load_reduction=27310 max_load_reduction_pct=0.833435 moved_to_echo=18172 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 2618, 0, 0, 0, 67, 0, 0, 1662, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 6, 0, 4, 673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3423, 516, 0, 0, 0, 0, 0, 0, 3, 1, 3912, 2, 0, 872, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 617, 0, 49, 111, 0, 5, 936, 1498, 1274, 0, 0, 0, 0, 0, 0, 964, 6, 25, 23, 105, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 1189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2689, 1344, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4347, 685, 3940, 4821, 4507, 5159, 5276, 4033] avg=4096.0000 max=5276 max_rank=6 min=685 min_rank=1 imbalance_ratio=1.288086 max_load_reduction=27492 max_load_reduction_pct=0.838989 moved_to_echo=18975 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 768, 0, 128, 4224, 0, 128, 20352, 7680, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[5376, 768, 14464, 128, 3200, 8576, 256, 640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[256, 128, 128, 128, 128, 3712, 256, 0, 24704, 3968, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 18944, 128, 128, 7168, 6912, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[384, 896, 128, 128, 12160, 128, 128, 3968, 8576, 7040, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 23296, 512, 0, 6656, 2304, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 22656, 256, 0, 0, 640, 0, 384, 9088, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[256, 768, 128, 128, 128, 25472, 1536, 4992, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 3935, 168, 25, 2, 179, 0, 0, 21, 68, 2, 0, 0, 68, 0, 0, 667, 0, 0, 0, 0, 93, 4087, 0, 0, 0, 2, 37, 3845, 3510, 0, 0, 2591, 3, 3, 0, 0, 0, 1794, 0, 0, 4, 0, 0, 0, 102, 0, 1, 4006, 0, 0, 0, 0, 0, 91, 51, 0, 48, 0, 4091, 3274, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 104, 104, 0, 0, 4019, 3, 1, 226, 3, 0, 184, 0, 0, 0, 0, 3780, 0, 0, 0, 0, 3994, 89, 3824, 0, 298, 0, 8, 4, 1044, 0, 3271, 531, 8, 0, 0, 0, 4, 58, 0, 0, 3690, 0, 0, 3, 0, 0, 0, 0, 2158, 0, 64, 0, 4, 76, 13, 0, 1110, 0, 4090, 0, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4, 71, 2, 0, 4095, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4092, 3582, 1, 0, 0, 5, 0, 871, 3643, 0, 0, 3966, 93, 0, 0, 0, 82, 0, 0, 0, 339, 0, 1183, 0, 0, 0, 0, 3885, 2617, 0, 0, 0, 0, 5, 0, 0, 0, 16, 4005, 50, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 56, 348, 7, 0, 515, 3113, 1, 0, 0, 0, 279, 0, 0, 0, 0, 0, 71, 299, 0, 1, 0, 3821, 1653, 0, 0, 0, 0, 470, 4006, 2, 0, 0, 4006, 0, 0, 0, 0, 10, 264, 0, 0, 145, 0, 0, 0, 31, 0, 0, 0, 3922, 1, 0, 0, 0, 1148, 3105, 0, 0, 0, 3962, 1523, 0, 9] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[4, 0, 0, 2, 3681, 728, 1576, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 38, 0, 1, 4064, 1244, 0, 0, 0, 59, 3, 1, 1336, 0, 52, 3764, 0, 15, 2, 0, 164, 0, 0, 0, 0, 0, 233, 2, 0, 0, 0, 0, 4089, 0, 0, 0, 10, 4092, 0, 0, 0, 0, 3649, 3924, 0, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 149, 19, 0, 0, 0, 46, 5, 0, 0, 1, 0, 0, 2, 1, 0, 0, 0, 2514, 4096, 0, 0, 0, 371, 0, 0, 81, 0, 0, 3967, 87, 2, 0, 0, 0, 1198, 0, 0, 136, 2, 0, 0, 0, 0, 8, 1750, 969, 0, 0, 0, 0, 1786, 3, 0, 2, 30, 3975, 4082, 3391, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 1, 316, 1870, 5, 0, 4024, 251, 0, 0, 0, 0, 0, 0, 928, 0, 0, 1716, 149, 271, 2, 0, 2361, 679, 45, 0, 0, 82, 0, 2, 95, 0, 0, 3123, 2, 0, 1169, 0, 5, 0, 0, 0, 353, 0, 21, 0, 0, 7, 0, 4, 3939, 0, 2, 0, 0, 3943, 0, 0, 0, 27, 4028, 1954, 1394, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 12, 3638, 3285, 0, 774, 355, 0, 0, 0, 0, 0, 0, 127, 27, 0, 2, 0, 0, 0, 0, 466, 3976, 7, 0, 0, 0, 3698, 0, 247, 0, 0, 3771, 2, 0, 0, 0, 526, 0, 0, 0, 2972, 0, 268, 0, 0, 0, 573, 0, 413, 0, 47, 0, 0, 3482, 0, 0, 0, 0, 4091, 9, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 71, 2, 0, 2035, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2906, 1460, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1216, 3582, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5, 0, 871, 3643, 0, 783, 272, 0, 0, 0, 0, 0, 0, 0, 3694, 93, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 339, 0, 1183, 0, 0, 0, 1491, 1416, 1277, 0, 0, 0, 0, 0, 0, 3885, 1126, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 1099, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2271, 4366, 4800, 5575, 3869, 5706, 5016, 1165] avg=4096.0000 max=5706 max_rank=5 min=1165 min_rank=7 imbalance_ratio=1.393066 max_load_reduction=27062 max_load_reduction_pct=0.825867 moved_to_echo=15756 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3935, 168, 25, 2, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 68, 2, 0, 0, 68, 0, 2968, 33, 0, 0, 0, 0, 0, 0, 0, 667, 0, 0, 0, 0, 28, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 37, 3845, 3510, 0, 34, 177, 0, 0, 0, 0, 0, 0, 0, 2414, 3, 3, 0, 0, 0, 1794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 102, 0, 0, 32, 55, 0, 0, 0, 0, 0, 1, 4006, 0, 0, 0, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 48, 0, 1123, 3274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4220, 3160, 4782, 7605, 4214, 193, 4098, 4496] avg=4096.0000 max=7605 max_rank=3 min=193 min_rank=5 imbalance_ratio=1.856689 max_load_reduction=25163 max_load_reduction_pct=0.767914 moved_to_echo=13001 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 104, 104, 0, 0, 1997, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 226, 3, 0, 184, 0, 0, 0, 2968, 1425, 0, 0, 0, 0, 0, 0, 0, 3780, 0, 0, 0, 0, 1187, 89, 0, 0, 0, 0, 0, 0, 0, 0, 3824, 0, 298, 0, 8, 4, 1044, 0, 769, 36, 0, 0, 0, 0, 0, 0, 3271, 495, 8, 0, 0, 0, 4, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3690, 0, 0, 3, 0, 0, 1229, 1382, 1253, 0, 0, 0, 0, 0, 0, 0, 929, 0, 64, 0, 4, 76, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 1110, 0, 1122, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2208, 4807, 5056, 5983, 3836, 7557, 1073, 2248] avg=4096.0000 max=7557 max_rank=5 min=1073 min_rank=6 imbalance_ratio=1.844971 max_load_reduction=25211 max_load_reduction_pct=0.769379 moved_to_echo=14714 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 56, 348, 7, 0, 515, 1547, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 279, 0, 0, 0, 0, 2875, 1363, 0, 0, 0, 0, 0, 0, 0, 71, 299, 0, 1, 0, 1136, 1653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 4006, 2, 0, 595, 274, 0, 0, 0, 0, 0, 0, 0, 3732, 0, 0, 0, 0, 10, 264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 0, 0, 31, 0, 2234, 1322, 971, 0, 0, 0, 0, 0, 0, 0, 1688, 1, 0, 0, 0, 1148, 0, 0, 0, 0, 0, 0, 0, 0, 3105, 0, 0, 0, 1087, 1523, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2474, 4517, 3160, 5347, 4006, 4703, 2837, 5724] avg=4096.0000 max=5724 max_rank=7 min=2474 min_rank=0 imbalance_ratio=1.397461 max_load_reduction=27044 max_load_reduction_pct=0.825317 moved_to_echo=17270 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 0, 2, 3681, 728, 784, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2648, 1450, 0, 0, 0, 0, 0, 0, 0, 1, 0, 38, 0, 1, 1208, 1244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 3, 1, 1336, 0, 301, 258, 0, 0, 0, 0, 0, 0, 52, 3506, 0, 15, 2, 0, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 233, 2, 0, 0, 2329, 1406, 491, 0, 0, 0, 0, 0, 0, 0, 1760, 0, 0, 0, 10, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 3924, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5230, 4098, 2492, 1958, 3739, 4461, 5862, 4928] avg=4096.0000 max=5862 max_rank=6 min=1958 min_rank=3 imbalance_ratio=1.431152 max_load_reduction=26906 max_load_reduction_pct=0.821106 moved_to_echo=18990 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 149, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 5, 0, 0, 1, 0, 2884, 897, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 747, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 371, 0, 0, 81, 0, 3, 272, 0, 0, 0, 0, 0, 0, 0, 3695, 87, 2, 0, 0, 0, 1198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 2, 0, 0, 0, 0, 552, 870, 5, 0, 0, 0, 0, 0, 8, 1750, 417, 0, 0, 0, 0, 1786, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 30, 1091, 4082, 3391, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4255, 3833, 4846, 727, 4982, 1565, 3961, 8599] avg=4096.0000 max=8599 max_rank=7 min=727 min_rank=3 imbalance_ratio=2.099365 max_load_reduction=24169 max_load_reduction_pct=0.737579 moved_to_echo=19107 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 316, 1870, 5, 0, 1999, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 928, 0, 2923, 842, 0, 0, 0, 0, 0, 0, 0, 1716, 149, 271, 2, 0, 702, 679, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 82, 0, 2, 95, 0, 770, 214, 0, 0, 0, 0, 0, 0, 0, 2909, 2, 0, 1169, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 353, 0, 21, 0, 0, 7, 2244, 817, 1255, 0, 0, 0, 0, 0, 0, 4, 1695, 0, 2, 0, 0, 3943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 1105, 1954, 1394, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4442, 4693, 3519, 1208, 4085, 4697, 5644, 4480] avg=4096.0000 max=5644 max_rank=6 min=1208 min_rank=3 imbalance_ratio=1.377930 max_load_reduction=27124 max_load_reduction_pct=0.827759 moved_to_echo=18906 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 12, 3638, 3285, 0, 378, 355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 27, 2972, 168, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 135, 3976, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 3698, 0, 247, 0, 151, 262, 0, 0, 0, 0, 0, 0, 0, 3509, 2, 0, 0, 0, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2972, 0, 268, 0, 0, 0, 238, 163, 245, 0, 0, 0, 0, 0, 573, 0, 175, 0, 47, 0, 0, 3482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1119, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7668, 3294, 4113, 4365, 4037, 3886, 4277, 1128] avg=4096.0000 max=7668 max_rank=0 min=1128 min_rank=7 imbalance_ratio=1.872070 max_load_reduction=25100 max_load_reduction_pct=0.765991 moved_to_echo=13328 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 0, 7680, 128, 1792, 128, 256, 128, 10368, 7424, 5632, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3200, 0, 1280, 128, 8832, 14848, 4864, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3968, 0, 384, 640, 4224, 8832, 9984, 0, 3456, 1792, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 256, 128, 384, 256, 0, 1152, 128, 23168, 7680, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[640, 9728, 7808, 128, 128, 0, 128, 14720, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 6272, 512, 384, 128, 128, 6400, 19456, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 4736, 5888, 11136, 1408, 8960, 896, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3328, 24064, 256, 128, 1280, 0, 896, 3328, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 3314, 3982, 4, 2, 1, 0, 3484, 14, 0, 0, 4096, 0, 914, 126, 3872, 0, 0, 851, 0, 2625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2594, 87, 0, 1, 9, 0, 19, 4067, 1098, 19, 1, 25, 0, 0, 0, 0, 0, 221, 0, 0, 10, 0, 0, 0, 3, 0, 0, 0, 1329, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 3910, 3283, 4009, 64, 1, 0, 19, 3422, 0, 0, 87, 3985, 22, 0, 672, 1, 0, 258, 561, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 97, 3938, 0, 0, 0, 96, 0, 0, 1533, 0, 0, 0, 133, 0, 0, 0, 19, 3, 0, 1, 0, 0, 0, 0, 2909, 0, 0, 0, 0, 3744, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 17, 2928, 151, 0, 0, 0, 4095, 266, 0, 0, 4096, 0, 0, 0, 4075, 0, 0, 0, 0, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 80, 0, 1, 0, 0, 0, 4041, 0, 0, 2663, 10, 0, 0, 0, 0, 0, 1278, 7, 0, 6, 0, 0, 20, 33, 0, 0, 0, 4096, 0, 0, 374] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[4, 0, 0, 0, 110, 0, 109, 0, 37, 532, 0, 0, 0, 4091, 0, 502, 17, 3328, 0, 3811, 7, 0, 3986, 0, 0, 325, 0, 0, 0, 0, 0, 0, 3624, 1971, 0, 0, 0, 0, 0, 172, 1148, 0, 21, 72, 48, 4, 0, 4, 0, 105, 4040, 591, 0, 2, 0, 9, 0, 45, 0, 0, 4043, 7, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 4081, 0, 4038, 0, 2, 0, 0, 3966, 1, 0, 1253, 4096, 3, 0, 0, 119, 0, 57, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4062, 0, 0, 9, 0, 0, 0, 3623, 0, 0, 0, 0, 0, 0, 0, 451, 89, 2762, 9, 3, 0, 0, 0, 3912, 0, 13, 0, 0, 184, 0, 33, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[45, 0, 0, 0, 1301, 0, 0, 1, 0, 3599, 13, 75, 389, 4064, 0, 175, 1422, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 1, 0, 0, 0, 0, 4095, 16, 4027, 0, 3824, 0, 0, 4079, 1567, 0, 88, 0, 0, 0, 0, 0, 0, 0, 20, 2487, 0, 0, 0, 880, 0, 0, 0, 0, 583, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 1414, 1492, 3717, 330, 0, 0, 0, 3401, 0, 0, 322, 4096, 0, 0, 9, 5, 0, 9, 3826, 0, 2, 0, 0, 0, 0, 0, 0, 232, 0, 0, 4081, 0, 0, 0, 251, 0, 0, 15, 0, 0, 0, 0, 4, 0, 0, 22, 29, 0, 3825, 6, 0, 0, 0, 691, 0, 2385, 0, 0, 2589, 0, 0, 15] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 2663, 1, 0, 0, 0, 0, 0, 1364, 0, 0, 4, 4096, 0, 0, 1216, 202, 0, 39, 3925, 1958, 5, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4, 417, 0, 0, 0, 0, 0, 0, 29, 264, 1, 4081, 23, 0, 0, 0, 141, 0, 3541, 0, 703, 3959, 0, 33, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3314, 2356, 4, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3484, 14, 0, 0, 604, 0, 914, 0, 0, 0, 0, 0, 0, 0, 0, 126, 2320, 0, 0, 851, 0, 2625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3492, 362, 0, 0, 0, 0, 0, 0, 412, 87, 0, 1, 9, 0, 19, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 1098, 19, 1, 25, 0, 0, 0, 0, 1820, 686, 0, 0, 0, 0, 0, 0, 0, 221, 0, 0, 10, 0, 0, 0, 1449, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1329, 0, 0, 0, 940, 103, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5677, 5016, 5922, 3854, 4595, 3649, 1680, 2375] avg=4096.0000 max=5922 max_rank=2 min=1680 min_rank=6 imbalance_ratio=1.445801 max_load_reduction=26846 max_load_reduction_pct=0.819275 moved_to_echo=12299 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4081, 0, 2389, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3966, 1, 0, 1253, 604, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 57, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3492, 568, 0, 0, 0, 0, 0, 0, 644, 0, 0, 9, 0, 0, 0, 3623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 451, 2850, 696, 0, 0, 0, 0, 0, 0, 89, 2762, 9, 3, 0, 0, 0, 3912, 44, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 184, 0, 33, 0, 953, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6472, 5827, 130, 4061, 4276, 3997, 6819, 1186] avg=4096.0000 max=6819 max_rank=6 min=130 min_rank=2 imbalance_ratio=1.664795 max_load_reduction=25949 max_load_reduction_pct=0.791901 moved_to_echo=16278 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 17, 1733, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 266, 0, 0, 604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2442, 0, 0, 0, 0, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3492, 572, 0, 0, 0, 0, 0, 0, 650, 80, 0, 1, 0, 0, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2663, 10, 0, 0, 0, 0, 2874, 504, 0, 0, 0, 0, 0, 0, 0, 1278, 7, 0, 6, 0, 0, 20, 1525, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 4096, 0, 0, 374, 691, 108, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1901, 4965, 2877, 4064, 4772, 6051, 2836, 5302] avg=4096.0000 max=6051 max_rank=5 min=1901 min_rank=0 imbalance_ratio=1.477295 max_load_reduction=26717 max_load_reduction_pct=0.815338 moved_to_echo=18961 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3910, 3283, 2372, 64, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 3422, 0, 0, 87, 588, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 672, 1, 0, 258, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 97, 3397, 550, 0, 0, 0, 0, 0, 0, 625, 0, 0, 0, 96, 0, 0, 1533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 133, 0, 0, 0, 19, 2763, 691, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 2909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3744, 0, 0, 0, 946, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9630, 4138, 1492, 4045, 2254, 3606, 2913, 4690] avg=4096.0000 max=9630 max_rank=0 min=1492 min_rank=2 imbalance_ratio=2.351074 max_load_reduction=23138 max_load_reduction_pct=0.706116 moved_to_echo=13463 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 0, 0, 67, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 532, 0, 0, 0, 604, 0, 502, 0, 0, 0, 0, 0, 0, 0, 0, 17, 1995, 0, 3811, 7, 0, 3986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 325, 0, 0, 0, 0, 0, 0, 3487, 506, 0, 0, 0, 0, 0, 0, 575, 1971, 0, 0, 0, 0, 0, 172, 0, 0, 0, 0, 0, 0, 0, 0, 1148, 0, 21, 72, 48, 4, 0, 4, 2543, 18, 0, 0, 0, 0, 0, 0, 0, 105, 4040, 591, 0, 2, 0, 9, 1245, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 4043, 7, 3, 0, 25, 88, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[180, 1675, 9816, 4318, 2718, 3858, 5992, 4211] avg=4096.0000 max=9816 max_rank=2 min=180 min_rank=0 imbalance_ratio=2.396484 max_load_reduction=22952 max_load_reduction_pct=0.700439 moved_to_echo=16779 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[45, 0, 0, 0, 762, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3599, 13, 75, 389, 600, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 1422, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 1, 0, 0, 0, 0, 3464, 572, 0, 0, 0, 0, 0, 0, 650, 16, 4027, 0, 3824, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 1567, 0, 88, 0, 0, 0, 0, 0, 2873, 228, 0, 0, 0, 0, 0, 0, 0, 0, 20, 2487, 0, 0, 0, 880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 583, 0, 0, 0, 311, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[808, 4851, 1423, 4053, 12596, 4756, 3387, 894] avg=4096.0000 max=12596 max_rank=4 min=808 min_rank=0 imbalance_ratio=3.075195 max_load_reduction=20172 max_load_reduction_pct=0.615601 moved_to_echo=21633 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1414, 1492, 2200, 330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3401, 0, 0, 322, 604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4, 0, 9, 3826, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 232, 0, 0, 3492, 570, 0, 0, 0, 0, 0, 0, 648, 0, 0, 0, 251, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 22, 2863, 640, 0, 0, 0, 0, 0, 0, 29, 0, 3825, 6, 0, 0, 0, 691, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2385, 0, 0, 2589, 0, 0, 15, 877, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5436, 4327, 3850, 4294, 914, 3529, 4552, 5866] avg=4096.0000 max=5866 max_rank=7 min=914 min_rank=4 imbalance_ratio=1.432129 max_load_reduction=26902 max_load_reduction_pct=0.820984 moved_to_echo=14861 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2663, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1364, 0, 0, 4, 601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1216, 115, 0, 39, 3925, 1958, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 3495, 578, 0, 0, 0, 0, 0, 0, 639, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 417, 0, 0, 0, 0, 0, 0, 29, 2876, 0, 0, 0, 0, 0, 0, 0, 264, 1, 4081, 23, 0, 0, 0, 141, 79, 0, 0, 0, 0, 0, 0, 0, 0, 3541, 0, 703, 3959, 0, 33, 0, 0, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2664, 1969, 7258, 4079, 643, 3322, 4589, 8244] avg=4096.0000 max=8244 max_rank=7 min=643 min_rank=4 imbalance_ratio=2.012695 max_load_reduction=24524 max_load_reduction_pct=0.748413 moved_to_echo=16798 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3584, 7040, 0, 4224, 9216, 2048, 7168, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 12160, 8192, 11904, 640, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4864, 2176, 4096, 128, 4224, 0, 128, 17536, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 6016, 0, 768, 20608, 128, 128, 512, 4864, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4352, 128, 2816, 256, 128, 128, 0, 640, 21504, 3584, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 384, 0, 128, 0, 256, 0, 128, 27904, 4352, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[512, 4480, 12032, 3200, 128, 128, 0, 8576, 4352, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 23936, 384, 128, 2176, 4864, 128, 1664, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 1554, 0, 3746, 0, 0, 0, 2711, 3844, 1692, 0, 4008, 4030, 0, 4092, 15, 0, 0, 0, 0, 975, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 12, 0, 0, 3, 0, 0, 24, 0, 0, 0, 0, 18, 221, 87, 2187, 0, 3528, 0, 1, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[14, 0, 352, 2, 4095, 0, 0, 0, 0, 2, 15, 19, 1857, 3859, 0, 3775, 0, 0, 0, 0, 0, 4096, 0, 0, 4061, 0, 16, 0, 0, 631, 0, 0, 4072, 2, 0, 8, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 1576, 0, 30, 4, 153, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1544, 0, 0, 3, 4095, 0, 0, 43, 0, 2397, 2, 43, 240, 3107, 0, 3860, 262, 0, 0, 0, 936, 4080, 0, 0, 2403, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2327, 6, 0, 0, 0, 0, 255, 4096, 0, 2, 0, 17, 3028, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 29, 1, 374, 4013, 0, 0, 0, 0, 21, 4076, 0, 389, 3873, 26, 3943, 2, 0, 0, 0, 1628, 3941, 0, 0, 0, 130, 0, 0, 2, 0, 10, 0, 0, 0, 0, 9, 4, 0, 0, 0, 0, 0, 0, 2041, 4067, 115, 713, 0, 0, 4, 0, 213, 0, 0, 0, 0, 3002, 0, 0, 139, 0, 0, 0, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 141, 9, 74, 4096, 0, 0, 0, 0, 46, 0, 141, 2133, 3888, 35, 3, 0, 0, 0, 0, 8, 3684, 0, 8, 2497, 11, 0, 280, 48, 4069, 245, 0, 328, 2, 0, 3438, 0, 1772, 0, 21, 0, 1, 2, 0, 3401, 0, 674, 0, 1406, 1, 0, 20, 0, 0, 152, 131, 0, 0, 3, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 4, 330, 34, 3937, 0, 0, 0, 5, 2271, 72, 0, 2683, 4058, 0, 4096, 0, 0, 0, 0, 0, 3905, 0, 0, 2821, 0, 1, 822, 0, 0, 0, 0, 0, 0, 0, 4046, 0, 334, 3, 0, 0, 0, 0, 47, 44, 1, 2568, 0, 0, 0, 648, 20, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 9, 3394, 2255, 0, 0, 0, 19, 2972, 17, 0, 21, 0, 1, 165, 0, 0, 0, 0, 59, 4078, 0, 0, 3914, 107, 0, 0, 0, 3952, 65, 0, 0, 3711, 0, 50, 0, 242, 0, 0, 0, 6, 0, 3691, 0, 1, 0, 0, 7, 3146, 152, 17, 0, 0, 0, 570, 0, 0, 0, 147, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[13, 0, 79, 0, 4096, 0, 136, 791, 0, 4092, 3504, 1, 0, 4095, 0, 3926, 0, 0, 0, 0, 0, 4096, 0, 0, 102, 0, 5, 0, 1, 0, 0, 0, 0, 0, 0, 1914, 0, 2, 0, 0, 0, 0, 0, 183, 13, 0, 0, 0, 69, 1836, 432, 3382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[14, 0, 352, 2, 3219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 15, 19, 1857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 134, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 16, 0, 0, 631, 0, 0, 0, 383, 0, 0, 0, 0, 0, 0, 4072, 2, 0, 8, 0, 0, 1, 0, 1195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1576, 0, 359, 0, 0, 0, 0, 0, 0, 0, 30, 4, 153, 0, 0, 0, 0, 0, 2180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 3859, 400, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3587, 1893, 4230, 5091, 5278, 6031, 2367, 4291] avg=4096.0000 max=6031 max_rank=5 min=1893 min_rank=1 imbalance_ratio=1.472412 max_load_reduction=26737 max_load_reduction_pct=0.815948 moved_to_echo=17967 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1554, 0, 2946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2711, 2226, 1692, 0, 4008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 975, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 910, 350, 0, 0, 0, 0, 0, 0, 12, 0, 0, 3, 0, 0, 24, 0, 1295, 708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 221, 87, 2187, 0, 328, 0, 0, 0, 0, 0, 0, 0, 3528, 0, 1, 7, 0, 0, 0, 0, 2362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4030, 435, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4500, 10637, 1112, 1273, 2042, 2841, 5898, 4465] avg=4096.0000 max=10637 max_rank=1 min=1112 min_rank=2 imbalance_ratio=2.596924 max_load_reduction=22131 max_load_reduction_pct=0.675385 moved_to_echo=15246 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 9, 3394, 1775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 1722, 17, 0, 21, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 4078, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 3914, 107, 0, 0, 0, 3952, 65, 0, 703, 210, 0, 0, 0, 0, 0, 0, 0, 3711, 0, 50, 0, 242, 0, 0, 53, 547, 0, 0, 0, 0, 0, 0, 0, 6, 0, 3691, 0, 1, 0, 0, 197, 0, 0, 0, 0, 0, 0, 0, 7, 3146, 152, 17, 0, 0, 0, 570, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5178, 1780, 4210, 8951, 4603, 3895, 3987, 164] avg=4096.0000 max=8951 max_rank=3 min=164 min_rank=7 imbalance_ratio=2.185303 max_load_reduction=23817 max_load_reduction_pct=0.726837 moved_to_echo=12649 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1544, 0, 0, 3, 3219, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1389, 2, 43, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 262, 0, 0, 0, 936, 4080, 0, 0, 134, 0, 0, 0, 0, 0, 0, 0, 2403, 0, 0, 20, 0, 0, 0, 0, 567, 383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2327, 6, 0, 1223, 441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 4096, 0, 2, 0, 359, 0, 0, 0, 0, 0, 0, 0, 17, 3028, 0, 0, 0, 0, 0, 0, 2228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 3107, 409, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4809, 1674, 5412, 3373, 3997, 4712, 5273, 3518] avg=4096.0000 max=5412 max_rank=2 min=1674 min_rank=1 imbalance_ratio=1.321289 max_load_reduction=27356 max_load_reduction_pct=0.834839 moved_to_echo=17500 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 330, 34, 3095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1316, 72, 0, 2683, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3905, 0, 0, 129, 0, 0, 0, 0, 0, 0, 0, 2821, 0, 1, 822, 0, 0, 0, 0, 537, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 0, 334, 3, 0, 1296, 418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 44, 1, 2568, 0, 345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 648, 20, 0, 0, 18, 0, 2366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 434, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3463, 4076, 4034, 4549, 6097, 3005, 3052, 4492] avg=4096.0000 max=6097 max_rank=4 min=3005 min_rank=5 imbalance_ratio=1.488525 max_load_reduction=26671 max_load_reduction_pct=0.813934 moved_to_echo=16646 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 29, 1, 374, 3155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4076, 0, 389, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1628, 3941, 0, 0, 131, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 0, 2, 0, 10, 0, 4, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4, 0, 0, 0, 1248, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2041, 4067, 115, 713, 0, 352, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 213, 0, 0, 0, 0, 2276, 0, 0, 0, 0, 0, 0, 0, 3002, 0, 0, 139, 0, 0, 0, 3, 3873, 419, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3559, 4505, 5702, 521, 1264, 7288, 2493, 7436] avg=4096.0000 max=7436 max_rank=7 min=521 min_rank=3 imbalance_ratio=1.815430 max_load_reduction=25332 max_load_reduction_pct=0.773071 moved_to_echo=18481 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 141, 9, 74, 3220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 141, 2133, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3684, 0, 8, 134, 0, 0, 0, 0, 0, 0, 0, 2497, 11, 0, 280, 48, 4069, 245, 0, 10, 383, 0, 0, 0, 0, 0, 0, 328, 2, 0, 3438, 0, 1772, 0, 21, 0, 8, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 3401, 0, 674, 0, 359, 0, 0, 0, 0, 0, 0, 0, 1406, 1, 0, 20, 0, 0, 152, 131, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3888, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3444, 2337, 3834, 7543, 5569, 4437, 1713, 3891] avg=4096.0000 max=7543 max_rank=3 min=1713 min_rank=6 imbalance_ratio=1.841553 max_load_reduction=25225 max_load_reduction_pct=0.769806 moved_to_echo=15610 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[13, 0, 79, 0, 3209, 0, 136, 791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2361, 3504, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 102, 0, 5, 0, 1, 0, 0, 0, 973, 386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1914, 0, 2, 0, 0, 1244, 758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 13, 0, 0, 0, 363, 0, 0, 0, 0, 0, 0, 0, 69, 1836, 432, 3382, 0, 0, 0, 0, 2266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 416, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4228, 5866, 4234, 1467, 3918, 559, 7985, 4511] avg=4096.0000 max=7985 max_rank=6 min=559 min_rank=5 imbalance_ratio=1.949463 max_load_reduction=24783 max_load_reduction_pct=0.756317 moved_to_echo=16973 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[15872, 256, 128, 1152, 128, 8704, 384, 0, 3712, 2944, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4480, 3840, 0, 9472, 128, 4736, 128, 128, 7680, 2944, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2816, 9088, 9472, 256, 11392, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 6272, 16000, 256, 7808, 0, 2688, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[5120, 8064, 1408, 3712, 0, 0, 256, 768, 13824, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3072, 0, 128, 384, 0, 0, 0, 128, 27008, 2560, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1664, 256, 2432, 3968, 23936, 0, 256, 896, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[384, 0, 0, 0, 2688, 28928, 0, 128, 1024, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 5, 45, 0, 1420, 0, 3688, 0, 0, 7, 50, 0, 0, 23, 0, 231, 2312, 0, 0, 0, 4092, 2645, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 381, 0, 0, 0, 98, 0, 27, 3918, 0, 0, 0, 0, 0, 0, 1724, 0, 4096, 0, 3918, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 0, 1245, 0, 2942, 0, 4091, 0, 0, 32, 0, 0, 0, 0, 0, 0, 3469, 0, 1850, 0, 3945, 8, 0, 0, 4048, 0, 0, 24, 0, 0, 247, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2281, 0, 0, 434, 4, 0, 0, 0, 50, 3, 4045, 0, 4033, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4039, 0, 0, 0, 0, 27, 0, 4063, 1, 91, 0, 50, 0, 0, 0, 0, 285, 0, 4, 4051, 0, 0, 0, 3, 256, 3475, 0, 0, 5, 1, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 1711, 0, 332, 209, 419, 0, 3769, 0, 282, 0, 0, 0, 0, 0, 1763, 0, 3853, 0, 4059, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3320, 0, 1944, 0, 0, 0, 0, 168, 0, 0, 0, 0, 0, 3490, 1153, 0, 0, 0, 1, 0, 0, 0, 4088, 0, 0, 43, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 0, 0, 1, 2172, 2, 472, 0, 0, 27, 0, 0, 0, 2448, 1237, 1, 4096, 0, 3937, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 1, 2405, 0, 2, 4096, 0, 776, 0, 3657, 0, 0, 0, 0, 0, 0, 24, 0, 0, 1, 0, 10, 0, 872, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 138, 0, 0, 0, 0, 4047, 0, 0, 0, 291, 0, 58, 4082, 0, 0, 11, 0, 0, 2, 2, 0, 3982, 0, 4067, 148] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 3343, 0, 0, 175, 1083, 131, 0, 0, 0, 0, 2, 0, 0, 698, 42, 0, 24, 758, 3609, 0, 0, 1267, 3637, 0, 8, 6, 0, 0, 68, 312, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 318, 0, 0, 3974, 3921, 0, 0, 0, 0, 1, 4, 1237, 0, 3981, 0, 4080, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4, 722, 0, 9, 6, 1, 0, 74, 3177, 0, 0, 0, 44, 0, 0, 484, 0, 4030, 2167, 0, 1491, 3, 29, 109, 0, 0, 3807, 0, 0, 0, 0, 0, 0, 347, 0, 0, 0, 0, 3, 0, 7, 0, 0, 0, 0, 0, 4040, 0, 0, 1941, 0, 12, 0, 7, 0, 1, 3249, 0, 2910, 17, 4077, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 11, 3804, 0, 3995, 0, 0, 0, 0, 0, 138, 0, 0, 23, 0, 136, 4058, 0, 0, 0, 1, 108, 0, 0, 4088, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 207, 0, 0, 0, 4, 33, 0, 0, 213, 2649, 0, 17, 4071, 0, 0, 0, 0, 0, 3, 729, 0, 4096, 0, 4076, 277] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 5, 45, 0, 842, 724, 0, 0, 0, 0, 0, 0, 1420, 0, 3688, 0, 0, 7, 50, 0, 206, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 231, 2312, 0, 0, 0, 425, 0, 0, 0, 0, 0, 0, 0, 4092, 2645, 0, 0, 2051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 381, 0, 0, 0, 2029, 633, 122, 0, 0, 0, 0, 0, 98, 0, 27, 2563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1724, 0, 2621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1617, 5371, 2991, 8788, 3803, 3165, 2688, 4345] avg=4096.0000 max=8788 max_rank=3 min=1617 min_rank=0 imbalance_ratio=2.145508 max_load_reduction=23980 max_load_reduction_pct=0.731812 moved_to_echo=14001 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 0, 1245, 0, 831, 0, 0, 0, 0, 0, 0, 0, 2942, 0, 4091, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3469, 0, 1850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3945, 8, 0, 0, 2034, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 14, 0, 0, 0, 0, 3907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2014, 625, 126, 0, 0, 0, 0, 0, 0, 0, 2281, 0, 0, 434, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 3, 2589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2079, 7065, 5319, 6011, 4168, 2765, 2719, 2642] avg=4096.0000 max=7065 max_rank=1 min=2079 min_rank=0 imbalance_ratio=1.724854 max_load_reduction=25703 max_load_reduction_pct=0.784393 moved_to_echo=12294 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3320, 0, 842, 0, 0, 0, 0, 0, 0, 0, 1944, 0, 0, 0, 0, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3490, 1153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2055, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 0, 0, 0, 0, 3815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 0, 0, 1, 2033, 633, 122, 0, 0, 0, 0, 0, 2172, 2, 472, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2448, 1237, 1, 2621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4162, 2112, 4643, 2099, 3951, 6821, 2673, 6307] avg=4096.0000 max=6821 max_rank=5 min=2099 min_rank=3 imbalance_ratio=1.665283 max_load_reduction=25947 max_load_reduction_pct=0.791840 moved_to_echo=19752 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4039, 0, 0, 0, 0, 792, 0, 0, 0, 0, 0, 0, 0, 27, 0, 4063, 1, 91, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 285, 0, 4, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 256, 1747, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 20, 0, 0, 0, 3933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1711, 0, 332, 209, 1728, 595, 126, 0, 0, 0, 0, 0, 419, 0, 3769, 0, 282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1763, 0, 2466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4831, 4232, 4340, 2011, 3954, 4701, 4470, 4229] avg=4096.0000 max=4831 max_rank=0 min=2011 min_rank=3 imbalance_ratio=1.179443 max_load_reduction=27937 max_load_reduction_pct=0.852570 moved_to_echo=17354 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 3343, 0, 818, 725, 0, 0, 0, 0, 0, 0, 0, 175, 1083, 131, 0, 0, 0, 0, 206, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 698, 42, 0, 24, 758, 426, 0, 0, 0, 0, 0, 0, 0, 3609, 0, 0, 1267, 1828, 0, 8, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 312, 0, 0, 0, 0, 3952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 318, 1809, 615, 128, 0, 0, 0, 0, 0, 0, 0, 3974, 2564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 1237, 0, 2548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4887, 1595, 1950, 6718, 4332, 2958, 6538, 3790] avg=4096.0000 max=6718 max_rank=3 min=1595 min_rank=1 imbalance_ratio=1.640137 max_load_reduction=26050 max_load_reduction_pct=0.794983 moved_to_echo=17618 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 722, 0, 9, 6, 1, 598, 359, 0, 0, 0, 0, 0, 0, 0, 74, 3177, 0, 0, 0, 44, 0, 102, 0, 0, 0, 0, 0, 0, 0, 0, 484, 0, 4030, 2167, 0, 1491, 3, 210, 0, 0, 0, 0, 0, 0, 0, 29, 109, 0, 0, 1913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 347, 0, 0, 0, 0, 3949, 0, 0, 0, 0, 0, 0, 0, 3, 0, 7, 0, 0, 0, 0, 0, 1894, 449, 128, 0, 0, 0, 0, 0, 4040, 0, 0, 1270, 0, 12, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3249, 0, 1863, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1699, 3397, 8385, 2051, 4296, 2481, 5329, 5130] avg=4096.0000 max=8385 max_rank=2 min=1699 min_rank=0 imbalance_ratio=2.047119 max_load_reduction=24383 max_load_reduction_pct=0.744110 moved_to_echo=17236 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 2405, 0, 2, 4096, 0, 818, 755, 0, 0, 0, 0, 0, 0, 776, 0, 3657, 0, 0, 0, 0, 0, 214, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 1, 0, 10, 0, 443, 0, 0, 0, 0, 0, 0, 0, 872, 0, 0, 0, 2058, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 3941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 2037, 615, 126, 0, 0, 0, 0, 0, 291, 0, 58, 2670, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 2549, 0, 0, 148, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8077, 4647, 478, 2931, 4079, 6825, 3030, 2701] avg=4096.0000 max=8077 max_rank=0 min=478 min_rank=2 imbalance_ratio=1.971924 max_load_reduction=24691 max_load_reduction_pct=0.753510 moved_to_echo=16635 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 11, 3804, 0, 844, 755, 0, 0, 0, 0, 0, 0, 3995, 0, 0, 0, 0, 0, 138, 0, 216, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 136, 4058, 0, 0, 0, 445, 0, 0, 0, 0, 0, 0, 0, 1, 108, 0, 0, 2050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 207, 3949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 33, 0, 0, 213, 2038, 637, 127, 0, 0, 0, 0, 0, 2649, 0, 17, 2655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 729, 0, 2615, 0, 0, 277, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5416, 4349, 4662, 2159, 4185, 3052, 5321, 3624] avg=4096.0000 max=5416 max_rank=0 min=2159 min_rank=3 imbalance_ratio=1.322266 max_load_reduction=27352 max_load_reduction_pct=0.834717 moved_to_echo=16182 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 10240, 0, 384, 768, 15616, 4864, 1024, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[12672, 2944, 128, 1536, 15744, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 640, 0, 8960, 13312, 128, 7552, 768, 2048, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 2560, 10112, 128, 19968, 128, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[9728, 128, 10624, 11776, 384, 512, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 128, 7168, 0, 128, 15872, 128, 6400, 3328, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[11136, 256, 19840, 256, 128, 256, 384, 0, 1024, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 512, 768, 128, 0, 0, 384, 31360, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3157, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 1736, 12, 0, 579, 0, 4071, 4043, 2, 1227, 0, 0, 0, 209, 28, 0, 0, 567, 0, 1537, 0, 2830, 0, 4096, 2, 0, 0, 432, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 22] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 8, 142, 0, 0, 0, 4035, 0, 99, 0, 4053, 0, 3786, 0, 10, 0, 0, 1, 0, 39, 3986, 12, 0, 0, 0, 0, 0, 6, 2139, 13, 3879, 2587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3656, 4, 0, 0, 1884, 0, 0, 0, 0, 0, 0, 0, 2, 64, 4, 0, 0, 0, 1603, 0, 756] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 3, 1, 0, 0, 332, 3, 2, 0, 1, 0, 4094, 0, 0, 0, 2, 0, 0, 5, 49, 0, 0, 0, 0, 0, 2049, 36, 418, 0, 4046, 3060, 0, 4007, 0, 18, 0, 0, 0, 27, 0, 0, 0, 4086, 96, 4096, 181, 3102, 0, 0, 0, 0, 0, 0, 0, 2974, 0, 0, 0, 0, 80, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[29, 0, 0, 9, 0, 176, 2396, 593, 331, 0, 1248, 0, 1500, 0, 43, 0, 0, 0, 0, 0, 3288, 6, 0, 0, 0, 0, 0, 0, 0, 0, 3855, 104, 0, 1, 0, 0, 5, 0, 0, 0, 0, 0, 2518, 237, 0, 6, 3920, 3997, 0, 0, 0, 0, 3, 753, 4, 7, 0, 0, 0, 115, 0, 3528, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 75, 3165, 0, 4009, 0, 0, 97, 0, 0, 4096, 0, 0, 0, 54, 2, 0, 0, 2245, 2, 0, 0, 0, 0, 0, 0, 0, 3, 1, 3194, 6, 0, 0, 0, 17, 0, 0, 2101, 17, 4076, 2, 3947, 0, 718, 3327, 40, 0, 0, 0, 0, 0, 1574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 2337, 0, 0, 0, 4090, 17, 2, 0, 8, 0, 4000, 0, 0, 0, 100, 0, 0, 322, 604, 0, 0, 0, 0, 0, 0, 9, 1, 0, 6, 4088, 1245, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 3905, 0, 3809, 4055, 3642, 0, 0, 0, 0, 0, 0, 0, 6, 429, 0, 0, 0, 0, 89, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 50, 0, 0, 311, 0, 4096, 0, 0, 0, 3461, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3870, 0, 0, 0, 0, 6, 0, 0, 54, 0, 2046, 2355, 317, 3, 0, 474, 0, 0, 16, 0, 0, 0, 0, 4084, 0, 3103, 1, 15, 2, 0, 0, 2541, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1865] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 2, 128, 0, 0, 0, 3999, 4, 394, 0, 3235, 0, 4088, 0, 0, 0, 393, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4079, 4040, 3172, 0, 0, 5, 0, 0, 0, 693, 0, 0, 21, 138, 173, 4096, 0, 0, 0, 3, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 581, 724, 202, 0, 0, 0, 0, 0, 0, 1736, 12, 0, 579, 0, 4071, 642, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1227, 0, 0, 0, 209, 28, 0, 2345, 408, 0, 0, 0, 0, 0, 0, 0, 567, 0, 0, 0, 0, 0, 3372, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 432, 0, 0, 0, 29, 1537, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 2915, 485, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3894, 2168, 5600, 7040, 4219, 3939, 2486, 3422] avg=4096.0000 max=7040 max_rank=3 min=2168 min_rank=1 imbalance_ratio=1.718750 max_load_reduction=25728 max_load_reduction_pct=0.785156 moved_to_echo=14066 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 1, 0, 0, 316, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 2811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 5, 49, 0, 0, 0, 754, 548, 16, 0, 0, 0, 0, 0, 0, 0, 2049, 36, 418, 0, 4046, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4007, 0, 18, 0, 0, 0, 27, 3395, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 181, 2554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2974, 4086, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 2206, 701, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[323, 2814, 1374, 7035, 7976, 2831, 7428, 2987] avg=4096.0000 max=7976 max_rank=4 min=323 min_rank=0 imbalance_ratio=1.947266 max_load_reduction=24792 max_load_reduction_pct=0.756592 moved_to_echo=21222 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 142, 0, 0, 0, 3836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 0, 4053, 0, 2600, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 39, 3986, 12, 0, 0, 697, 333, 199, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2139, 13, 3879, 411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3656, 311, 0, 0, 0, 0, 0, 0, 64, 4, 0, 0, 0, 1603, 0, 756, 1865, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3986, 6762, 5267, 6448, 489, 1555, 3969, 4292] avg=4096.0000 max=6762 max_rank=1 min=489 min_rank=4 imbalance_ratio=1.650879 max_load_reduction=26006 max_load_reduction_pct=0.793640 moved_to_echo=10305 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[29, 0, 0, 9, 0, 176, 2278, 593, 0, 0, 0, 0, 0, 0, 0, 0, 331, 0, 1248, 0, 1031, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3288, 6, 0, 0, 276, 709, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3855, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5, 0, 0, 0, 5, 193, 0, 0, 0, 0, 0, 0, 0, 0, 2518, 0, 0, 0, 3920, 3288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 753, 4, 7, 237, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 3528, 0, 4096, 74, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3085, 2653, 4397, 3873, 204, 9726, 1016, 7814] avg=4096.0000 max=9726 max_rank=5 min=204 min_rank=4 imbalance_ratio=2.374512 max_load_reduction=23042 max_load_reduction_pct=0.703186 moved_to_echo=18760 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 75, 3165, 0, 3811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 2813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 2, 0, 0, 2245, 2, 0, 0, 754, 7, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 507, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 17, 0, 0, 2101, 595, 529, 0, 0, 0, 0, 0, 0, 17, 4076, 2, 0, 0, 0, 3327, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1574, 0, 0, 3947, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2303, 123, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7051, 2910, 3262, 511, 3248, 7455, 5905, 2426] avg=4096.0000 max=7455 max_rank=5 min=511 min_rank=3 imbalance_ratio=1.820068 max_load_reduction=25313 max_load_reduction_pct=0.772491 moved_to_echo=19034 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 50, 0, 0, 311, 0, 3894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3461, 0, 2813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3870, 0, 0, 0, 754, 2, 202, 0, 0, 0, 0, 0, 0, 6, 0, 0, 54, 0, 2046, 374, 0, 0, 0, 0, 0, 0, 0, 0, 317, 3, 0, 474, 0, 0, 16, 0, 2572, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 13, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2541, 0, 0, 0, 1, 4084, 283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1865, 1698, 531, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4255, 6274, 4828, 2480, 3911, 14, 6911, 4095] avg=4096.0000 max=6911 max_rank=6 min=14 min_rank=5 imbalance_ratio=1.687256 max_load_reduction=25857 max_load_reduction_pct=0.789093 moved_to_echo=14931 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2337, 0, 0, 0, 3888, 17, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8, 0, 2747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 322, 604, 0, 0, 0, 736, 643, 202, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 0, 6, 649, 0, 0, 0, 0, 0, 0, 0, 0, 1245, 0, 0, 4, 0, 0, 0, 0, 3158, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 2999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3905, 492, 0, 0, 0, 0, 0, 0, 429, 0, 0, 0, 0, 89, 0, 0, 2947, 651, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6242, 2757, 2607, 665, 4924, 7054, 4403, 4116] avg=4096.0000 max=7054 max_rank=5 min=665 min_rank=3 imbalance_ratio=1.722168 max_load_reduction=25714 max_load_reduction_pct=0.784729 moved_to_echo=20497 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 128, 0, 0, 0, 3798, 4, 0, 0, 0, 0, 0, 0, 0, 0, 394, 0, 3235, 0, 2801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 393, 0, 0, 0, 4083, 0, 0, 0, 756, 0, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4079, 634, 0, 0, 0, 0, 0, 0, 0, 0, 3172, 0, 0, 5, 0, 0, 0, 693, 3396, 531, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 19, 0, 0, 0, 0, 138, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2916, 700, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3932, 6430, 5433, 4716, 7797, 194, 650, 3616] avg=4096.0000 max=7797 max_rank=4 min=194 min_rank=5 imbalance_ratio=1.903564 max_load_reduction=24971 max_load_reduction_pct=0.762054 moved_to_echo=12257 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 4736, 2560, 0, 384, 0, 11520, 13824, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4864, 5248, 0, 512, 128, 256, 128, 2944, 15488, 3840, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 0, 3072, 128, 2432, 128, 3072, 21632, 2944, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 1792, 2176, 128, 3200, 128, 22016, 3840, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[896, 128, 12032, 0, 19840, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[512, 128, 0, 128, 128, 5248, 0, 6784, 17024, 3200, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 2688, 128, 3584, 256, 25728, 640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[640, 128, 0, 384, 22272, 128, 0, 0, 5376, 3072, 1408, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[4025, 83, 22, 423, 0, 4, 4080, 0, 0, 0, 51, 1144, 0, 0, 0, 0, 0, 248, 0, 0, 1, 0, 0, 3972, 0, 3796, 0, 0, 0, 70, 2796, 0, 32, 0, 0, 11, 0, 0, 4056, 1, 0, 0, 23, 32, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 3876, 0, 0, 3735, 285, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[4095, 29, 0, 665, 296, 0, 3842, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3650, 0, 3930, 114, 0, 6, 0, 35, 0, 0, 1, 0, 4084, 0, 0, 2, 8, 0, 0, 0, 4096, 3, 2, 3379, 3, 0, 0, 91, 102, 0, 0, 3, 0, 4083, 0, 247, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[4095, 4047, 0, 2616, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 2, 0, 11, 0, 0, 2, 0, 0, 0, 3832, 4096, 0, 0, 26, 263, 0, 0, 1471, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 2045, 42, 0, 0, 1, 3837, 2, 363, 0, 325, 0, 2032, 36, 0, 14, 0, 0, 0, 0, 0, 0, 0, 2861, 77, 3958, 4, 177, 0, 836, 4072, 0, 0, 422, 0, 2823, 0, 255, 0, 5, 0, 1, 1045, 4065, 0, 0, 0, 0, 0, 0, 0, 233, 0, 15, 0, 0, 0, 0, 0, 0, 3150, 0, 0, 72] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[3891, 1793, 5, 286, 0, 73, 2760, 0, 0, 0, 2, 42, 0, 0, 0, 0, 0, 0, 0, 0, 219, 0, 0, 3742, 0, 3431, 0, 0, 0, 12, 820, 0, 3, 0, 0, 4082, 0, 0, 4025, 0, 0, 0, 2665, 3675, 0, 0, 0, 520, 0, 0, 3, 0, 15, 148, 1, 0, 0, 0, 3, 2, 1, 313, 229, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[3797, 0, 316, 2953, 5, 0, 4082, 0, 0, 3, 0, 589, 0, 0, 0, 0, 0, 29, 8, 0, 0, 0, 0, 0, 0, 1477, 4, 4, 0, 4, 4012, 0, 0, 0, 0, 184, 0, 0, 0, 0, 70, 1269, 1414, 4060, 0, 0, 167, 4064, 20, 0, 0, 0, 0, 0, 0, 13, 4039, 2, 0, 141, 0, 0, 0, 42] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[3985, 0, 0, 4081, 0, 0, 3597, 0, 0, 0, 0, 11, 0, 0, 17, 0, 0, 3370, 0, 841, 5, 0, 0, 601, 0, 0, 0, 0, 0, 0, 3403, 0, 0, 0, 47, 7, 11, 0, 3998, 0, 3, 0, 4058, 104, 0, 0, 0, 6, 0, 0, 0, 0, 31, 0, 0, 116, 3549, 288, 27, 162, 0, 185, 265, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[4078, 4096, 492, 2807, 0, 0, 4089, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 8, 0, 0, 3, 0, 0, 0, 0, 64, 0, 0, 0, 10, 701, 0, 0, 0, 0, 2, 0, 0, 2000, 0, 0, 0, 4093, 2520, 0, 1025, 2, 3992, 0, 0, 46, 0, 4, 0, 0, 0, 1396, 0, 0, 4, 3, 0, 1295, 10] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[805, 83, 22, 423, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 1144, 0, 0, 0, 0, 3210, 8, 0, 0, 0, 0, 0, 0, 0, 248, 0, 0, 1, 0, 0, 3972, 18, 0, 0, 0, 0, 0, 0, 0, 0, 2861, 0, 0, 0, 70, 2796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 11, 0, 0, 4056, 1, 705, 4, 0, 0, 0, 0, 0, 0, 0, 0, 23, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 4080, 10, 0, 0, 0, 0, 0, 0, 3876, 0, 0, 3735, 285, 0, 0, 0, 230, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1337, 4413, 4239, 5727, 4809, 25, 4092, 8126] avg=4096.0000 max=8126 max_rank=7 min=25 min_rank=5 imbalance_ratio=1.983887 max_load_reduction=24642 max_load_reduction_pct=0.752014 moved_to_echo=17052 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[819, 29, 0, 665, 296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3266, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3650, 2389, 0, 0, 0, 0, 0, 0, 0, 0, 2962, 114, 0, 6, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4084, 0, 0, 2, 8, 730, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 3, 2, 3379, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 102, 0, 0, 3, 0, 3842, 10, 0, 0, 0, 0, 0, 0, 4083, 0, 247, 0, 0, 0, 0, 0, 238, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1809, 4308, 6039, 3117, 5339, 3540, 4048, 4568] avg=4096.0000 max=6039 max_rank=2 min=1809 min_rank=0 imbalance_ratio=1.474365 max_load_reduction=26729 max_load_reduction_pct=0.815704 moved_to_echo=17495 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[819, 4047, 0, 2616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 3266, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2389, 0, 0, 0, 0, 0, 0, 0, 0, 3064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 11, 0, 0, 2, 0, 755, 514, 0, 0, 0, 0, 0, 0, 0, 0, 3832, 153, 0, 0, 26, 263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1471, 0, 0, 0, 0, 0, 4096, 10, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 247, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7482, 4354, 2389, 3064, 1284, 4274, 5577, 4344] avg=4096.0000 max=7482 max_rank=0 min=1284 min_rank=4 imbalance_ratio=1.826660 max_load_reduction=25286 max_load_reduction_pct=0.771667 moved_to_echo=15479 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 2045, 42, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 363, 0, 325, 0, 2032, 36, 0, 14, 0, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2861, 2371, 0, 0, 0, 0, 0, 0, 0, 77, 2983, 4, 177, 0, 836, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 422, 0, 2823, 0, 255, 0, 5, 735, 510, 0, 0, 0, 0, 0, 0, 0, 1, 1045, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 233, 0, 15, 0, 0, 3837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3150, 0, 0, 72, 240, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2090, 3802, 5232, 8149, 4750, 1198, 4085, 3462] avg=4096.0000 max=8149 max_rank=3 min=1198 min_rank=5 imbalance_ratio=1.989502 max_load_reduction=24619 max_load_reduction_pct=0.751312 moved_to_echo=13495 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[760, 0, 316, 2953, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 589, 0, 0, 0, 0, 3028, 1031, 0, 0, 0, 0, 0, 0, 0, 29, 8, 0, 0, 0, 0, 0, 2368, 0, 0, 0, 0, 0, 0, 0, 0, 1114, 4, 4, 0, 4, 4012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 0, 0, 0, 0, 274, 509, 0, 0, 0, 0, 0, 0, 70, 1269, 1414, 152, 0, 0, 167, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 13, 4082, 9, 0, 0, 0, 0, 0, 0, 4039, 2, 0, 141, 0, 0, 0, 42, 89, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4034, 4651, 2405, 5138, 967, 7136, 4124, 4313] avg=4096.0000 max=7136 max_rank=5 min=967 min_rank=4 imbalance_ratio=1.742188 max_load_reduction=25632 max_load_reduction_pct=0.782227 moved_to_echo=16540 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[779, 1793, 5, 286, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 42, 0, 0, 0, 0, 3103, 933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 219, 0, 0, 3742, 2143, 0, 0, 0, 0, 0, 0, 0, 0, 2586, 0, 0, 0, 12, 820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4082, 0, 0, 4025, 0, 637, 461, 0, 0, 0, 0, 0, 0, 0, 0, 2665, 138, 0, 0, 0, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 15, 148, 1, 0, 2760, 9, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 1, 313, 229, 7, 208, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2936, 4080, 6104, 3418, 9208, 3323, 2936, 763] avg=4096.0000 max=9208 max_rank=4 min=763 min_rank=7 imbalance_ratio=2.248047 max_load_reduction=23560 max_load_reduction_pct=0.718994 moved_to_echo=16230 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[797, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 17, 0, 3178, 26, 0, 0, 0, 0, 0, 0, 0, 3370, 0, 841, 5, 0, 0, 601, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 7, 11, 0, 3998, 0, 0, 13, 0, 0, 0, 0, 0, 0, 3, 0, 4058, 5, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 116, 3597, 10, 0, 0, 0, 0, 0, 0, 3549, 288, 27, 162, 0, 185, 265, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4878, 3232, 4877, 3403, 4076, 4072, 3754, 4476] avg=4096.0000 max=4878 max_rank=0 min=3232 min_rank=1 imbalance_ratio=1.190918 max_load_reduction=27890 max_load_reduction_pct=0.851135 moved_to_echo=16378 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[807, 4096, 492, 2807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 3258, 642, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 3, 0, 0, 0, 1472, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 10, 701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2000, 0, 15, 318, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 88, 0, 1025, 2, 3992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 4, 0, 0, 0, 4089, 13, 0, 0, 0, 0, 0, 0, 1396, 0, 0, 4, 3, 0, 1295, 10, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 3928, 1483, 752, 2335, 9200, 4152, 2716] avg=4096.0000 max=9200 max_rank=5 min=752 min_rank=3 imbalance_ratio=2.246094 max_load_reduction=23568 max_load_reduction_pct=0.719238 moved_to_echo=18403 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 1280, 17152, 896, 128, 1152, 3584, 8960, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[5632, 12160, 896, 13952, 384, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 15616, 128, 256, 128, 1024, 15872, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 3712, 128, 896, 256, 0, 0, 14848, 13312, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[384, 128, 512, 1792, 2048, 128, 128, 128, 22400, 5760, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 512, 128, 11264, 128, 256, 14208, 128, 3968, 2944, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[21120, 384, 384, 4096, 3456, 512, 1792, 256, 1280, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 0, 1664, 384, 128, 256, 128, 256, 30464, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4084, 0, 0, 0, 0, 0, 849, 0, 4094, 0, 0, 2090, 4018, 0, 3106, 3, 0, 0, 3490, 2657, 0, 0, 0, 0, 26, 3, 0, 0, 0, 0, 0, 3902, 0, 0, 0, 0, 46, 0, 0, 2, 0, 0, 0, 0, 4067, 1, 0, 0, 3, 0, 0, 56, 0, 0, 0, 0, 9, 201, 0, 60] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4094, 20, 0, 0, 8, 4, 0, 3801, 0, 0, 0, 4070, 0, 23, 2, 0, 0, 3546, 1018, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 2849, 0, 0, 0, 0, 4086, 2, 0, 0, 0, 905, 174, 0, 3039, 107, 0, 0, 459, 0, 0, 26, 244, 0, 0, 162, 0, 7, 4, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4062, 403, 5, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 4096, 0, 17, 0, 0, 0, 3846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 744, 0, 0, 4083, 0, 0, 0, 0, 4096, 3993, 0, 0, 0, 0, 0, 0, 3308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 119, 3569, 2089, 760, 0, 0, 0, 1, 0, 4063, 0, 0, 51, 1071, 0, 3058, 17, 0, 1886, 3992, 45, 0, 0, 0, 0, 965, 0, 0, 87, 0, 0, 0, 3733, 0, 0, 0, 0, 2609, 64, 0, 116, 0, 0, 0, 0, 456, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 15, 1, 0, 3993] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4057, 3545, 749, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 145, 0, 0, 0, 610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3627, 4, 0, 3910, 0, 0, 0, 0, 2872, 282, 0, 0, 0, 0, 62, 3, 0, 1986, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 2698] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[5, 0, 0, 33, 3825, 4018, 0, 0, 0, 0, 0, 0, 3627, 0, 0, 0, 659, 0, 137, 0, 0, 2543, 4096, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 3974, 0, 17, 0, 0, 3388, 2683, 47, 0, 7, 0, 0, 85, 12, 1, 0, 0, 0, 0, 0, 714, 303, 0, 0, 2, 2, 2533, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 1477, 4096, 177, 3906, 0, 0, 0, 4, 0, 4095, 0, 0, 2690, 3696, 0, 4067, 52, 0, 0, 4063, 0, 98, 0, 0, 0, 61, 0, 0, 0, 0, 13, 0, 19, 0, 0, 0, 0, 3777, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 1, 2, 0, 461] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3262, 48, 0, 0, 0, 0, 2, 0, 3993, 0, 0, 3134, 0, 0, 154, 0, 2, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 777, 0, 4039, 48, 0, 13, 0, 0, 3900, 67, 0, 0, 0, 0, 1941, 3195, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 1490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 849, 0, 3077, 0, 0, 2090, 0, 0, 0, 0, 0, 0, 0, 0, 2553, 0, 3106, 3, 0, 0, 0, 2657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 3, 0, 0, 3490, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 1, 0, 0, 3, 0, 0, 56, 1017, 453, 5, 350, 0, 0, 0, 0, 0, 0, 0, 0, 9, 201, 0, 60, 2099, 1115, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1491, 6016, 8319, 4014, 3449, 43, 5952, 3484] avg=4096.0000 max=8319 max_rank=2 min=43 min_rank=5 imbalance_ratio=2.031006 max_load_reduction=24449 max_load_reduction_pct=0.746124 moved_to_echo=12928 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1482, 403, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2602, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3846, 492, 0, 0, 0, 0, 0, 0, 744, 0, 0, 3609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3617, 3993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3308, 0, 0, 0, 0, 0, 0, 0, 1013, 474, 479, 357, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 2088, 1137, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1890, 3067, 2619, 4338, 4353, 7610, 5631, 3260] avg=4096.0000 max=7610 max_rank=5 min=1890 min_rank=0 imbalance_ratio=1.857910 max_load_reduction=25158 max_load_reduction_pct=0.767761 moved_to_echo=20854 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1495, 4094, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4, 0, 2857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2585, 0, 23, 2, 0, 0, 0, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 3546, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3608, 2, 0, 0, 0, 905, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3039, 107, 0, 0, 459, 0, 0, 26, 944, 331, 478, 355, 0, 0, 0, 0, 244, 0, 0, 162, 0, 7, 4, 0, 2105, 1130, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5609, 2869, 3628, 4064, 2518, 4689, 5739, 3652] avg=4096.0000 max=5739 max_rank=6 min=2518 min_rank=4 imbalance_ratio=1.401123 max_load_reduction=27029 max_load_reduction_pct=0.824860 moved_to_echo=16598 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 0, 33, 1395, 4018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 420, 0, 137, 0, 0, 2543, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 464, 0, 0, 0, 0, 0, 0, 0, 10, 0, 3513, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2991, 2683, 47, 0, 7, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 0, 0, 0, 0, 0, 714, 901, 461, 397, 57, 0, 0, 0, 0, 303, 0, 0, 2, 2, 2533, 0, 0, 1966, 182, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5451, 2726, 3147, 4560, 3540, 5813, 2543, 4988] avg=4096.0000 max=5813 max_rank=5 min=2543 min_rank=6 imbalance_ratio=1.419189 max_load_reduction=26955 max_load_reduction_pct=0.822601 moved_to_echo=16884 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 119, 1302, 2089, 760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3054, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 681, 0, 3058, 17, 0, 1886, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 965, 0, 0, 87, 3992, 432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2304, 64, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 456, 2, 0, 0, 0, 0, 0, 0, 1009, 433, 305, 93, 0, 0, 0, 0, 0, 0, 0, 6, 15, 1, 0, 3993, 1835, 297, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4270, 3106, 5687, 5476, 3300, 2484, 2298, 6147] avg=4096.0000 max=6147 max_rank=7 min=2298 min_rank=6 imbalance_ratio=1.500732 max_load_reduction=26621 max_load_reduction_pct=0.812408 moved_to_echo=14229 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1480, 3545, 749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2602, 0, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 610, 492, 0, 0, 0, 0, 0, 0, 3627, 4, 0, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2536, 282, 0, 0, 0, 0, 62, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1986, 0, 0, 0, 0, 0, 0, 1017, 454, 336, 357, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 2698, 2085, 1137, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5774, 3079, 2747, 1102, 7087, 2883, 4150, 5946] avg=4096.0000 max=7087 max_rank=4 min=1102 min_rank=3 imbalance_ratio=1.730225 max_load_reduction=25681 max_load_reduction_pct=0.783722 moved_to_echo=20066 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1190, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3001, 0, 0, 3134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 0, 2, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 395, 0, 0, 0, 0, 0, 0, 4086, 777, 0, 3570, 48, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3900, 67, 0, 0, 0, 0, 1941, 0, 0, 0, 0, 0, 0, 0, 0, 3195, 0, 0, 0, 0, 0, 0, 0, 992, 469, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1677, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1238, 6137, 162, 4491, 8494, 5908, 4656, 1682] avg=4096.0000 max=8494 max_rank=4 min=162 min_rank=2 imbalance_ratio=2.073730 max_load_reduction=24274 max_load_reduction_pct=0.740784 moved_to_echo=20740 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1477, 1485, 177, 3906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3074, 0, 0, 2690, 0, 0, 0, 0, 0, 0, 0, 0, 2340, 0, 4067, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 0, 61, 0, 0, 0, 4063, 501, 0, 0, 0, 0, 0, 0, 0, 13, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3331, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 5, 446, 327, 0, 0, 0, 0, 6, 0, 0, 0, 1, 2, 0, 461, 2110, 1029, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7045, 5768, 6459, 4723, 27, 3338, 1799, 3609] avg=4096.0000 max=7045 max_rank=0 min=27 min_rank=4 imbalance_ratio=1.719971 max_load_reduction=25723 max_load_reduction_pct=0.785004 moved_to_echo=8773 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[640, 0, 128, 256, 128, 2816, 128, 7296, 16000, 6144, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[18432, 11008, 128, 128, 128, 1024, 256, 2048, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[14080, 2176, 0, 0, 512, 0, 0, 896, 7936, 3200, 2560, 1920, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[13824, 0, 10752, 128, 128, 4480, 0, 3840, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 0, 1664, 11392, 14464, 5504, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8576, 896, 0, 23552, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 896, 0, 23936, 0, 0, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 1152, 128, 0, 128, 27776, 3840, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4096, 2427, 0, 0, 4095, 0, 0, 4073, 0, 817, 0, 0, 224, 0, 1519, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 1648, 0, 0, 0, 0, 0, 2596, 0, 483, 0, 0, 0, 1705, 0, 83, 0, 0, 613, 0, 0, 2278, 0, 2012, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[7, 2948, 1102, 0, 4096, 3793, 0, 0, 44, 0, 0, 86, 0, 12, 0, 0, 0, 3649, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 452, 0, 0, 0, 205, 0, 0, 0, 0, 0, 0, 0, 4084, 74, 4019, 0, 0, 0, 0, 0, 7, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4087, 2085, 0, 0, 730, 1193, 0, 0, 0, 0, 0, 0, 4093, 1048, 0, 338, 0, 0, 0, 0, 0, 0, 3944, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3382, 0, 3697, 0, 0, 4, 0, 0, 4091, 4026, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4067, 3489, 0, 97, 20, 7, 0, 265, 0, 260, 0, 14, 1118, 3920, 0, 2621, 0, 0, 0, 0, 0, 0, 3995, 0, 0, 0, 0, 0, 4086, 1, 8, 0, 127, 0, 0, 0, 0, 0, 0, 0, 0, 1141, 54, 2, 0, 19, 122, 6, 3215, 0, 4054, 0, 0, 0, 0, 0, 3, 49, 0, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4096, 4094, 0, 0, 90, 4, 0, 0, 0, 4045, 0, 0, 4090, 1016, 0, 3, 0, 0, 0, 0, 0, 3417, 4083, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 785, 536, 377, 0, 0, 12, 0, 0, 169, 30, 1809, 0, 0, 0, 0, 0, 7, 7, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[16, 0, 0, 0, 4001, 4053, 0, 17, 4091, 218, 0, 0, 40, 0, 2, 0, 176, 2528, 0, 0, 0, 34, 0, 0, 0, 0, 3053, 0, 0, 11, 0, 58, 3558, 1, 43, 0, 0, 0, 6, 0, 21, 0, 0, 239, 1001, 3849, 172, 8, 0, 584, 0, 44, 3926, 591, 357, 0, 2, 1, 0, 0, 0, 51, 16, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 4081, 4093, 0, 0, 26, 51, 0, 0, 0, 0, 0, 0, 305, 3762, 0, 18, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4092, 0, 22, 0, 0, 0, 1, 0, 4057, 0, 0, 0, 0, 3392, 0, 0, 0, 3, 0, 0, 2995, 1768, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[7, 0, 24, 0, 2860, 4092, 4, 0, 0, 364, 0, 0, 29, 19, 0, 0, 4093, 920, 0, 0, 0, 1, 397, 0, 0, 13, 3514, 0, 0, 0, 0, 1, 267, 265, 0, 639, 0, 102, 4095, 0, 0, 0, 0, 0, 0, 3954, 110, 0, 0, 0, 0, 0, 3981, 637, 0, 0, 0, 74, 0, 14, 0, 2223, 69, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 55, 2427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4073, 0, 817, 0, 0, 1051, 98, 0, 0, 0, 0, 0, 0, 224, 0, 1519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1648, 0, 0, 0, 0, 0, 2596, 0, 213, 0, 0, 0, 0, 0, 0, 0, 483, 0, 0, 0, 1705, 0, 83, 0, 0, 123, 0, 0, 0, 0, 0, 0, 0, 613, 0, 0, 1104, 0, 2012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3943, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2483, 10134, 1743, 3885, 4457, 2394, 3729, 3943] avg=4096.0000 max=10134 max_rank=1 min=1743 min_rank=2 imbalance_ratio=2.474121 max_load_reduction=22634 max_load_reduction_pct=0.690735 moved_to_echo=14523 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 55, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 4, 0, 0, 0, 4045, 0, 0, 78, 98, 0, 0, 0, 0, 0, 0, 4090, 829, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3417, 3871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 212, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 785, 536, 377, 0, 104, 9, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 82, 30, 1809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 0, 0, 3943, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4150, 4315, 4922, 7288, 4392, 1811, 1933, 3957] avg=4096.0000 max=7288 max_rank=3 min=1811 min_rank=5 imbalance_ratio=1.779297 max_load_reduction=25480 max_load_reduction_pct=0.777588 moved_to_echo=12093 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 55, 3489, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0, 20, 7, 0, 265, 0, 260, 0, 14, 1483, 97, 0, 0, 0, 0, 0, 0, 1118, 3199, 0, 2621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 1, 8, 0, 127, 0, 0, 0, 208, 320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1141, 54, 2, 401, 173, 0, 0, 0, 0, 0, 0, 0, 19, 122, 6, 1559, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 49, 0, 8, 3915, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3641, 2146, 6938, 3787, 4750, 1771, 5760, 3975] avg=4096.0000 max=6938 max_rank=2 min=1771 min_rank=5 imbalance_ratio=1.693848 max_load_reduction=25830 max_load_reduction_pct=0.788269 moved_to_echo=16256 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[16, 0, 0, 0, 54, 4053, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 218, 0, 0, 40, 0, 2, 0, 1812, 96, 0, 0, 0, 0, 0, 0, 176, 2063, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2894, 0, 0, 11, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 3558, 1, 43, 0, 0, 0, 6, 0, 159, 206, 0, 0, 0, 0, 0, 0, 21, 0, 0, 239, 1001, 3849, 172, 8, 259, 212, 0, 0, 0, 0, 0, 0, 0, 584, 0, 44, 1902, 591, 357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 51, 16, 0, 3851, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4140, 6259, 2273, 2963, 3973, 5761, 3478, 3921] avg=4096.0000 max=6259 max_rank=1 min=2273 min_rank=2 imbalance_ratio=1.528076 max_load_reduction=26509 max_load_reduction_pct=0.808990 moved_to_echo=17133 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[7, 2948, 1102, 0, 55, 3793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 86, 0, 12, 0, 0, 1884, 98, 0, 0, 0, 0, 0, 0, 0, 2978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 213, 298, 0, 0, 0, 0, 0, 0, 452, 0, 0, 0, 205, 0, 0, 0, 373, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1980, 74, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 3943, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7905, 2124, 2978, 3880, 4608, 1250, 6073, 3950] avg=4096.0000 max=7905 max_rank=0 min=1250 min_rank=5 imbalance_ratio=1.929932 max_load_reduction=24863 max_load_reduction_pct=0.758759 moved_to_echo=15881 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 55, 2085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 730, 1193, 0, 0, 0, 0, 0, 0, 1888, 98, 0, 0, 0, 0, 0, 0, 4093, 856, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 1, 0, 205, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3382, 0, 3697, 0, 107, 221, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1982, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 3934, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2141, 3909, 5287, 3739, 320, 7407, 6012, 3953] avg=4096.0000 max=7407 max_rank=5 min=320 min_rank=4 imbalance_ratio=1.808350 max_load_reduction=25361 max_load_reduction_pct=0.773956 moved_to_echo=17692 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 24, 0, 39, 4092, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 364, 0, 0, 29, 19, 0, 0, 1837, 68, 0, 0, 0, 0, 0, 0, 4093, 751, 0, 0, 0, 1, 397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 3331, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 267, 265, 0, 639, 0, 102, 4095, 0, 183, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3954, 110, 0, 94, 215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1929, 637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 14, 0, 2223, 69, 0, 2753, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4166, 2317, 5242, 3345, 5626, 4373, 2566, 5133] avg=4096.0000 max=5626 max_rank=4 min=2317 min_rank=1 imbalance_ratio=1.373535 max_load_reduction=27142 max_load_reduction_pct=0.828308 moved_to_echo=17698 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 47, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 51, 0, 0, 0, 0, 0, 0, 1385, 102, 0, 0, 0, 0, 0, 0, 305, 3062, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 22, 0, 0, 0, 1, 0, 215, 312, 0, 0, 0, 0, 0, 0, 4057, 0, 0, 0, 0, 3392, 0, 0, 388, 164, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1446, 1768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 3932, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4142, 1564, 3385, 3881, 4642, 8001, 3217, 3936] avg=4096.0000 max=8001 max_rank=5 min=1564 min_rank=1 imbalance_ratio=1.953369 max_load_reduction=24767 max_load_reduction_pct=0.755829 moved_to_echo=19796 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[9216, 1920, 0, 4480, 128, 5248, 128, 128, 11520, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[5120, 0, 0, 256, 7168, 12928, 4608, 128, 1792, 1408, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 1280, 128, 128, 12032, 7168, 12288, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[14208, 13824, 1536, 3072, 0, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[21888, 384, 128, 640, 128, 128, 6784, 0, 1664, 1408, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 3072, 1152, 0, 512, 28160, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 3456, 29312, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 128, 2432, 128, 128, 30336, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 6, 0, 0, 7, 0, 0, 3755, 0, 0, 4094, 0, 0, 0, 0, 1, 0, 4096, 0, 11, 0, 4026, 0, 0, 15, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3273, 0, 0, 1967, 0, 0, 0, 0, 652, 0, 0, 1, 2467, 324, 0, 4003, 0, 0, 0, 0, 396, 0, 0, 3645, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 2342, 0, 0, 0, 223, 530, 0, 8, 73, 38, 0, 0, 22, 0, 14, 0, 0, 1395, 0, 0, 23, 3811, 0, 0, 0, 0, 0, 916, 0, 4096, 2, 0, 929, 0, 0, 0, 0, 268, 1110, 2, 155, 3144, 2312, 0, 0, 0, 948, 0, 0, 0, 3542, 0, 5, 3659, 0, 0, 0, 0, 0, 0, 1, 3200, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 3624, 0, 0, 0, 0, 0, 0, 513, 0, 0, 4075, 0, 0, 13, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 11, 0, 8, 0, 0, 0, 319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4058, 735, 0, 0, 0, 3754, 0, 0, 0, 3338, 0, 0, 4091, 0, 40, 0, 0, 0, 0, 3, 4084, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[56, 446, 0, 0, 0, 0, 0, 0, 3797, 0, 32, 3823, 0, 0, 6, 0, 0, 0, 3997, 0, 0, 0, 4078, 0, 0, 0, 0, 27, 1842, 0, 3053, 0, 1, 0, 0, 0, 39, 0, 0, 0, 126, 0, 4040, 0, 263, 20, 0, 0, 0, 0, 0, 28, 0, 136, 2281, 0, 926, 1, 23, 0, 0, 0, 3727, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 1335, 20, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 22, 0, 4096, 1, 0, 0, 4096, 0, 0, 1, 4, 0, 3641, 0, 3881, 0, 0, 0, 0, 0, 0, 220, 0, 0, 0, 42, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 2879, 125, 0, 4096, 0, 0, 0, 1, 217, 0, 0, 4062, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 4064, 0, 0, 0, 120, 0, 0, 0, 3, 0, 0, 0, 0, 3671, 0, 32, 0, 3974, 73, 1, 101, 0, 0, 1, 0, 4089, 0, 0, 0, 0, 2, 0, 3, 0, 216, 0, 0, 4096, 11, 0, 61, 3, 445, 0, 0, 0, 2763, 0, 0, 3692, 0, 0, 0, 0, 1297, 156, 47, 3846, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1, 3049, 0, 0, 0, 2647, 0, 0, 4, 0, 0, 2942, 383, 0, 0, 0, 0, 34, 4096, 0, 0, 0, 3419, 0, 0, 0, 5, 0, 3, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1584, 1327, 0, 0, 0, 146, 0, 0, 0, 3325, 0, 0, 4092, 0, 591, 0, 0, 0, 0, 0, 1038, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[184, 3426, 0, 0, 0, 3985, 0, 0, 2, 0, 0, 617, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2976, 0, 0, 0, 17, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 3837, 3, 0, 0, 0, 211, 0, 0, 0, 2, 0, 0, 4093, 0, 142, 0, 0, 0, 962, 0, 4094, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6, 0, 0, 7, 0, 0, 587, 177, 0, 0, 0, 0, 0, 0, 3755, 0, 0, 4094, 0, 0, 0, 0, 1122, 0, 0, 0, 0, 0, 0, 0, 1, 0, 859, 0, 11, 0, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 29, 0, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3273, 3237, 447, 0, 0, 0, 0, 0, 0, 0, 0, 1585, 0, 0, 0, 0, 652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2467, 324, 0, 1847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 396, 0, 0, 3645, 0, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[777, 8971, 4897, 230, 6957, 2237, 4639, 4060] avg=4096.0000 max=8971 max_rank=1 min=230 min_rank=3 imbalance_ratio=2.190186 max_load_reduction=23797 max_load_reduction_pct=0.726227 moved_to_echo=17893 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3624, 0, 0, 0, 0, 0, 0, 600, 366, 0, 0, 0, 0, 0, 0, 513, 0, 0, 4075, 0, 0, 13, 0, 1147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 859, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 8, 0, 0, 0, 319, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3237, 457, 0, 0, 0, 0, 0, 0, 0, 5, 3268, 735, 0, 0, 0, 3754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3338, 0, 0, 1887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 3, 4084, 0, 40, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4590, 5748, 860, 722, 3694, 7762, 5225, 4167] avg=4096.0000 max=7762 max_rank=5 min=722 min_rank=3 imbalance_ratio=1.895020 max_load_reduction=25006 max_load_reduction_pct=0.763123 moved_to_echo=20848 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2342, 0, 0, 0, 223, 530, 0, 537, 284, 0, 0, 0, 0, 0, 0, 8, 73, 38, 0, 0, 22, 0, 14, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 0, 0, 23, 3811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 916, 0, 4096, 2, 298, 0, 0, 0, 0, 0, 0, 0, 0, 929, 0, 0, 0, 0, 268, 1110, 1102, 408, 0, 0, 0, 0, 0, 0, 2, 155, 2531, 2312, 0, 0, 0, 948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3542, 0, 5, 1688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3200, 0, 31, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3916, 1181, 4127, 5312, 3817, 5948, 5235, 3232] avg=4096.0000 max=5948 max_rank=5 min=1181 min_rank=1 imbalance_ratio=1.452148 max_load_reduction=26820 max_load_reduction_pct=0.818481 moved_to_echo=18232 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[56, 446, 0, 0, 0, 0, 0, 0, 334, 365, 0, 0, 0, 0, 0, 0, 3797, 0, 32, 3823, 0, 0, 6, 0, 639, 0, 0, 0, 0, 0, 0, 0, 0, 0, 838, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 1842, 0, 3053, 0, 383, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 39, 0, 0, 0, 3159, 254, 0, 0, 0, 0, 0, 0, 126, 0, 3252, 0, 263, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 136, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 926, 1, 23, 0, 0, 0, 3727, 0, 40, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1201, 8297, 4916, 5305, 3453, 3661, 1218, 4717] avg=4096.0000 max=8297 max_rank=1 min=1201 min_rank=0 imbalance_ratio=2.025635 max_load_reduction=24471 max_load_reduction_pct=0.746796 moved_to_echo=13049 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 4064, 0, 0, 541, 370, 0, 0, 0, 0, 0, 0, 0, 120, 0, 0, 0, 3, 0, 0, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 770, 0, 32, 0, 3974, 73, 0, 0, 0, 0, 0, 0, 0, 0, 1, 101, 0, 0, 1, 0, 4089, 0, 388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 0, 216, 2901, 412, 0, 0, 0, 0, 0, 0, 0, 0, 3297, 11, 0, 61, 3, 445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2763, 0, 0, 1704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1297, 156, 47, 3846, 0, 41, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4976, 1158, 4849, 4580, 3534, 3817, 4467, 5387] avg=4096.0000 max=5387 max_rank=7 min=1158 min_rank=1 imbalance_ratio=1.315186 max_load_reduction=27381 max_load_reduction_pct=0.835602 moved_to_echo=17205 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1335, 20, 0, 0, 0, 0, 0, 601, 362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 1148, 0, 0, 0, 0, 0, 0, 0, 22, 0, 859, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 3641, 0, 3881, 0, 380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 220, 0, 0, 3237, 457, 0, 0, 0, 0, 0, 0, 0, 42, 3229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2879, 125, 0, 1890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 217, 0, 0, 4062, 0, 40, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2318, 1166, 4978, 7907, 3914, 3271, 4894, 4320] avg=4096.0000 max=7907 max_rank=3 min=1166 min_rank=1 imbalance_ratio=1.930420 max_load_reduction=24861 max_load_reduction_pct=0.758698 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 3049, 0, 0, 0, 2647, 0, 0, 600, 143, 0, 0, 0, 0, 0, 0, 4, 0, 0, 2942, 383, 0, 0, 0, 1147, 0, 0, 0, 0, 0, 0, 0, 0, 34, 859, 0, 0, 0, 3419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3, 0, 4078, 0, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3237, 457, 0, 0, 0, 0, 0, 0, 0, 4, 1276, 1327, 0, 0, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3325, 0, 0, 1888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 591, 0, 0, 0, 0, 0, 1038, 0, 15, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6440, 4476, 4312, 4236, 3694, 2753, 5213, 1644] avg=4096.0000 max=6440 max_rank=0 min=1644 min_rank=7 imbalance_ratio=1.572266 max_load_reduction=26328 max_load_reduction_pct=0.803467 moved_to_echo=13304 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[184, 3426, 0, 0, 0, 3985, 0, 0, 604, 351, 0, 0, 0, 0, 0, 0, 2, 0, 0, 617, 0, 0, 0, 0, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 853, 0, 0, 0, 2976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 4093, 0, 0, 0, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3243, 462, 0, 0, 0, 0, 0, 0, 0, 28, 3077, 3, 0, 0, 0, 211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 0, 0, 0, 962, 0, 4094, 0, 43, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8550, 1771, 3829, 4476, 3705, 3319, 1877, 5241] avg=4096.0000 max=8550 max_rank=0 min=1771 min_rank=1 imbalance_ratio=2.087402 max_load_reduction=24218 max_load_reduction_pct=0.739075 moved_to_echo=14142 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8192, 256, 128, 15616, 384, 128, 128, 128, 8448, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 14336, 128, 0, 0, 11008, 640, 0, 4480, 2432, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 128, 18432, 512, 256, 13952, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 1024, 0, 128, 128, 256, 384, 4608, 23424, 3456, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 10624, 0, 19584, 128, 2560, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 256, 21632, 4480, 384, 128, 128, 6272, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 6272, 128, 128, 128, 26496, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1792, 128, 128, 1920, 1152, 128, 27776, 0, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True + [2026-09-21 08:24:24] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 265.9 | throughput per GPU (TFLOP/s/GPU): 121.2 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 1.006700E+01 | loss scale: 1.0 | grad norm: 101.619 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[2770, 623, 105, 138, 138, 342, 567, 79, 133, 888, 30, 3, 87, 284, 160, 1111, 34, 539, 850, 1354, 66, 478, 517, 376, 1099, 166, 29, 1073, 471, 715, 184, 339, 198, 392, 697, 306, 541, 1044, 264, 769, 353, 825, 707, 25, 2020, 753, 88, 395, 410, 541, 835, 23, 1819, 553, 348, 193, 1214, 14, 498, 62, 10, 486, 603, 34] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[2200, 800, 69, 281, 75, 558, 530, 22, 305, 957, 164, 0, 33, 360, 150, 1091, 40, 151, 1073, 1576, 96, 718, 984, 283, 801, 125, 185, 774, 678, 870, 205, 463, 175, 709, 552, 242, 530, 763, 242, 620, 486, 688, 622, 3, 1645, 1063, 13, 494, 646, 600, 858, 138, 1500, 407, 431, 390, 1067, 15, 558, 185, 14, 154, 256, 85] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1988, 861, 130, 125, 139, 561, 489, 190, 321, 706, 38, 1, 70, 514, 422, 1262, 172, 753, 1469, 1197, 129, 376, 762, 159, 1038, 193, 142, 695, 534, 818, 178, 390, 243, 479, 967, 157, 483, 644, 289, 472, 395, 831, 1024, 8, 1815, 898, 52, 425, 470, 721, 779, 121, 1566, 482, 313, 133, 759, 6, 516, 184, 34, 364, 262, 54] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[2453, 723, 173, 154, 32, 512, 364, 69, 82, 524, 70, 7, 106, 320, 162, 1264, 52, 449, 1339, 1315, 111, 640, 664, 177, 1000, 147, 43, 976, 578, 765, 207, 567, 175, 456, 640, 238, 520, 618, 356, 611, 450, 854, 633, 31, 1878, 1231, 34, 432, 868, 574, 629, 94, 1937, 431, 285, 231, 817, 12, 604, 40, 20, 265, 677, 82] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1997, 621, 348, 249, 179, 452, 771, 129, 104, 563, 200, 2, 41, 278, 223, 1560, 307, 335, 619, 1678, 99, 272, 677, 191, 744, 110, 229, 1387, 408, 708, 103, 450, 396, 312, 380, 95, 860, 572, 190, 318, 1058, 961, 838, 25, 1441, 614, 90, 413, 577, 923, 1115, 234, 1757, 386, 259, 270, 819, 4, 701, 275, 18, 353, 405, 75] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1982, 760, 55, 211, 122, 474, 618, 38, 255, 480, 328, 1, 39, 244, 106, 1344, 95, 339, 1240, 1383, 143, 889, 740, 285, 1153, 89, 74, 918, 521, 554, 69, 276, 168, 674, 464, 143, 626, 1115, 215, 484, 523, 789, 581, 22, 1887, 1052, 49, 715, 518, 555, 1006, 158, 1690, 406, 316, 347, 693, 11, 947, 229, 27, 240, 239, 54] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[2530, 747, 123, 145, 62, 482, 276, 45, 234, 568, 98, 2, 106, 344, 112, 1405, 78, 373, 1110, 1447, 44, 388, 648, 351, 934, 134, 102, 647, 705, 1220, 384, 406, 59, 521, 762, 105, 873, 896, 212, 829, 330, 1189, 795, 39, 1645, 529, 54, 475, 404, 907, 940, 82, 1578, 252, 320, 164, 674, 8, 674, 147, 29, 361, 532, 133] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[2177, 812, 112, 107, 249, 757, 388, 126, 342, 794, 46, 2, 79, 298, 155, 683, 79, 644, 928, 1417, 176, 687, 629, 419, 818, 143, 45, 1110, 548, 587, 145, 255, 293, 411, 478, 105, 422, 593, 223, 564, 282, 973, 889, 8, 1872, 864, 107, 701, 576, 529, 1073, 117, 1940, 564, 226, 432, 913, 14, 706, 190, 14, 292, 481, 159] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1760, 800, 69, 281, 75, 558, 530, 22, 0, 0, 0, 0, 0, 0, 0, 0, 305, 957, 164, 0, 33, 360, 150, 1091, 543, 671, 33, 0, 0, 0, 0, 0, 40, 151, 1073, 905, 96, 718, 984, 283, 0, 0, 0, 0, 0, 0, 0, 0, 801, 125, 185, 774, 678, 870, 205, 463, 41, 0, 0, 0, 0, 0, 0, 0, 175, 709, 552, 242, 530, 763, 242, 620, 366, 0, 0, 0, 0, 0, 0, 0, 486, 688, 622, 3, 463, 1063, 13, 494, 0, 0, 0, 0, 0, 0, 0, 0, 646, 600, 858, 138, 720, 407, 431, 390, 0, 0, 0, 0, 0, 0, 0, 0, 1067, 15, 558, 185, 14, 154, 256, 85, 1182, 237, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4307, 4250, 4142, 4199, 3832, 4190, 3753] avg=4096.0000 max=4307 max_rank=1 min=3753 min_rank=7 imbalance_ratio=1.051514 max_load_reduction=28461 max_load_reduction_pct=0.868561 moved_to_echo=15974 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2216, 623, 105, 138, 138, 342, 567, 79, 0, 0, 0, 0, 0, 0, 0, 0, 133, 888, 30, 3, 87, 284, 160, 1111, 658, 576, 41, 0, 0, 0, 0, 0, 34, 539, 850, 778, 66, 478, 517, 376, 0, 0, 0, 0, 0, 0, 0, 0, 1099, 166, 29, 1073, 471, 715, 184, 339, 52, 0, 0, 0, 0, 0, 0, 0, 198, 392, 697, 306, 541, 1044, 264, 769, 461, 0, 0, 0, 0, 0, 0, 0, 353, 825, 707, 25, 568, 753, 88, 395, 0, 0, 0, 0, 0, 0, 0, 0, 410, 541, 835, 23, 873, 553, 348, 193, 0, 0, 0, 0, 0, 0, 0, 0, 1214, 14, 498, 62, 10, 486, 603, 34, 1452, 288, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4208, 3971, 3638, 4128, 4672, 3714, 3776, 4661] avg=4096.0000 max=4672 max_rank=4 min=3638 min_rank=2 imbalance_ratio=1.140625 max_load_reduction=28096 max_load_reduction_pct=0.857422 moved_to_echo=16823 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1961, 723, 173, 154, 32, 512, 364, 69, 0, 0, 0, 0, 0, 0, 0, 0, 82, 524, 70, 7, 106, 320, 162, 1264, 701, 560, 37, 0, 0, 0, 0, 0, 52, 449, 1339, 755, 111, 640, 664, 177, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 147, 43, 976, 578, 765, 207, 567, 46, 0, 0, 0, 0, 0, 0, 0, 175, 456, 640, 238, 520, 618, 356, 611, 409, 0, 0, 0, 0, 0, 0, 0, 450, 854, 633, 31, 528, 1231, 34, 432, 0, 0, 0, 0, 0, 0, 0, 0, 868, 574, 629, 94, 929, 431, 285, 231, 0, 0, 0, 0, 0, 0, 0, 0, 817, 12, 604, 40, 20, 265, 677, 82, 1350, 307, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3988, 3833, 4187, 4329, 4023, 4193, 4041, 4174] avg=4096.0000 max=4329 max_rank=3 min=3833 min_rank=1 imbalance_ratio=1.056885 max_load_reduction=28439 max_load_reduction_pct=0.867889 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1597, 621, 348, 249, 179, 452, 771, 129, 0, 0, 0, 0, 0, 0, 0, 0, 104, 563, 200, 2, 41, 278, 223, 1560, 636, 714, 30, 0, 0, 0, 0, 0, 307, 335, 619, 964, 99, 272, 677, 191, 0, 0, 0, 0, 0, 0, 0, 0, 744, 110, 229, 1387, 408, 708, 103, 450, 37, 0, 0, 0, 0, 0, 0, 0, 396, 312, 380, 95, 860, 572, 190, 318, 333, 0, 0, 0, 0, 0, 0, 0, 1058, 961, 838, 25, 405, 614, 90, 413, 0, 0, 0, 0, 0, 0, 0, 0, 577, 923, 1115, 234, 843, 386, 259, 270, 0, 0, 0, 0, 0, 0, 0, 0, 819, 4, 701, 275, 18, 353, 405, 75, 1036, 278, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4346, 4351, 3464, 4176, 3456, 4404, 4607, 3964] avg=4096.0000 max=4607 max_rank=6 min=3456 min_rank=4 imbalance_ratio=1.124756 max_load_reduction=28161 max_load_reduction_pct=0.859406 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1590, 861, 130, 125, 139, 561, 489, 190, 0, 0, 0, 0, 0, 0, 0, 0, 321, 706, 38, 1, 70, 514, 422, 1262, 567, 509, 30, 0, 0, 0, 0, 0, 172, 753, 1469, 688, 129, 376, 762, 159, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 193, 142, 695, 534, 818, 178, 390, 37, 0, 0, 0, 0, 0, 0, 0, 243, 479, 967, 157, 483, 644, 289, 472, 331, 0, 0, 0, 0, 0, 0, 0, 395, 831, 1024, 8, 510, 898, 52, 425, 0, 0, 0, 0, 0, 0, 0, 0, 470, 721, 779, 121, 751, 482, 313, 133, 0, 0, 0, 0, 0, 0, 0, 0, 759, 6, 516, 184, 34, 364, 262, 54, 1305, 248, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4440, 4508, 4025, 4065, 4143, 3770, 3732] avg=4096.0000 max=4508 max_rank=2 min=3732 min_rank=7 imbalance_ratio=1.100586 max_load_reduction=28260 max_load_reduction_pct=0.862427 moved_to_echo=15710 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2023, 747, 123, 145, 62, 482, 276, 45, 0, 0, 0, 0, 0, 0, 0, 0, 234, 568, 98, 2, 106, 344, 112, 1405, 571, 616, 38, 0, 0, 0, 0, 0, 78, 373, 1110, 831, 44, 388, 648, 351, 0, 0, 0, 0, 0, 0, 0, 0, 934, 134, 102, 647, 705, 1220, 384, 406, 48, 0, 0, 0, 0, 0, 0, 0, 59, 521, 762, 105, 873, 896, 212, 829, 421, 0, 0, 0, 0, 0, 0, 0, 330, 1189, 795, 39, 463, 529, 54, 475, 0, 0, 0, 0, 0, 0, 0, 0, 404, 907, 940, 82, 757, 252, 320, 164, 0, 0, 0, 0, 0, 0, 0, 0, 674, 8, 674, 147, 29, 361, 532, 133, 1182, 250, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3903, 4094, 3823, 4580, 4678, 3874, 3826, 3990] avg=4096.0000 max=4678 max_rank=4 min=3823 min_rank=2 imbalance_ratio=1.142090 max_load_reduction=28090 max_load_reduction_pct=0.857239 moved_to_echo=16368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1585, 760, 55, 211, 122, 474, 618, 38, 0, 0, 0, 0, 0, 0, 0, 0, 255, 480, 328, 1, 39, 244, 106, 1344, 611, 588, 30, 0, 0, 0, 0, 0, 95, 339, 1240, 795, 143, 889, 740, 285, 0, 0, 0, 0, 0, 0, 0, 0, 1153, 89, 74, 918, 521, 554, 69, 276, 37, 0, 0, 0, 0, 0, 0, 0, 168, 674, 464, 143, 626, 1115, 215, 484, 330, 0, 0, 0, 0, 0, 0, 0, 523, 789, 581, 22, 531, 1052, 49, 715, 0, 0, 0, 0, 0, 0, 0, 0, 518, 555, 1006, 158, 811, 406, 316, 347, 0, 0, 0, 0, 0, 0, 0, 0, 693, 11, 947, 229, 27, 240, 239, 54, 1356, 268, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3863, 4026, 4526, 3691, 4219, 4262, 4117, 4064] avg=4096.0000 max=4526 max_rank=2 min=3691 min_rank=3 imbalance_ratio=1.104980 max_load_reduction=28242 max_load_reduction_pct=0.861877 moved_to_echo=16662 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1729, 812, 112, 107, 249, 757, 388, 126, 0, 0, 0, 0, 0, 0, 0, 0, 342, 794, 46, 2, 79, 298, 155, 683, 705, 607, 35, 0, 0, 0, 0, 0, 79, 644, 928, 810, 176, 687, 629, 419, 0, 0, 0, 0, 0, 0, 0, 0, 818, 143, 45, 1110, 548, 587, 145, 255, 46, 0, 0, 0, 0, 0, 0, 0, 293, 411, 478, 105, 422, 593, 223, 564, 367, 0, 0, 0, 0, 0, 0, 0, 282, 973, 889, 8, 522, 864, 107, 701, 0, 0, 0, 0, 0, 0, 0, 0, 576, 529, 1073, 117, 924, 564, 226, 432, 0, 0, 0, 0, 0, 0, 0, 0, 913, 14, 706, 190, 14, 292, 481, 159, 1350, 311, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4280, 3746, 4372, 3697, 3456, 4346, 4441, 4430] avg=4096.0000 max=4441 max_rank=6 min=3456 min_rank=4 imbalance_ratio=1.084229 max_load_reduction=28327 max_load_reduction_pct=0.864471 moved_to_echo=16673 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1792, 3968, 4992, 1408, 4864, 6272, 2048, 4736, 3072, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[7680, 1152, 896, 7680, 4480, 6272, 1536, 3200, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4480, 5376, 7296, 1024, 6656, 3584, 2560, 2176, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1792, 5504, 1024, 128, 640, 2688, 1536, 9728, 4992, 4864, 384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[7040, 128, 5248, 1408, 256, 2560, 3456, 768, 10240, 2304, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[896, 3584, 8704, 6528, 896, 4480, 5632, 2304, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3968, 7168, 6144, 256, 4096, 7040, 512, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[14464, 6016, 1152, 1536, 1024, 4224, 4096, 768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[5, 469, 3620, 0, 0, 0, 190, 0, 10, 0, 50, 971, 1627, 6, 38, 0, 1218, 0, 0, 2, 18, 3745, 0, 658, 0, 51, 0, 57, 1071, 2, 3, 8, 431, 0, 178, 0, 0, 4, 1, 386, 56, 3999, 2395, 0, 3272, 0, 54, 58, 3126, 1719, 0, 788, 5, 0, 0, 1065, 423, 702, 19, 0, 264, 0, 4, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 1993, 2719, 29, 0, 1, 136, 0, 28, 0, 109, 28, 120, 16, 2, 13, 2, 24, 13, 0, 1, 3932, 1, 3347, 0, 134, 1, 0, 1078, 0, 4, 154, 141, 0, 26, 0, 9, 0, 43, 163, 174, 3836, 2594, 0, 1009, 2, 6, 4, 1727, 3802, 0, 2507, 0, 0, 0, 2508, 47, 188, 3, 8, 3, 0, 83, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[49, 2798, 2879, 1, 0, 0, 738, 0, 44, 0, 27, 1518, 58, 0, 1, 1, 45, 2, 1, 0, 0, 2594, 37, 3652, 5, 193, 0, 5, 443, 23, 12, 74, 1605, 11, 72, 10, 0, 1, 41, 574, 1, 2492, 1087, 0, 88, 1, 2323, 456, 2324, 810, 0, 3331, 0, 1, 0, 1971, 35, 262, 3, 36, 25, 0, 8, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[4, 1077, 2250, 0, 0, 0, 206, 17, 197, 7, 1733, 529, 210, 0, 16, 58, 268, 0, 0, 0, 4, 2224, 0, 4087, 1, 0, 1, 145, 736, 4, 43, 30, 3745, 0, 126, 144, 8, 0, 17, 179, 111, 591, 356, 0, 1298, 0, 1249, 33, 2629, 2501, 0, 1548, 12, 1, 0, 3907, 10, 9, 24, 0, 423, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 2680, 355, 4, 9, 0, 1, 0, 342, 3, 31, 839, 45, 0, 0, 2, 69, 0, 172, 0, 0, 1607, 0, 4059, 0, 0, 2, 9, 792, 22, 255, 700, 2334, 0, 70, 0, 48, 0, 94, 2038, 4, 812, 1334, 2, 3690, 14, 968, 0, 3217, 794, 3, 787, 0, 0, 0, 2857, 215, 29, 3, 0, 1427, 0, 29, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 2620, 3966, 0, 2, 39, 1350, 0, 65, 0, 257, 588, 73, 143, 0, 0, 133, 0, 143, 0, 62, 3416, 2, 3012, 0, 8, 1, 11, 1334, 1, 79, 47, 1968, 0, 663, 0, 32, 0, 0, 15, 125, 1778, 2970, 0, 54, 14, 99, 24, 1884, 1983, 21, 440, 0, 8, 25, 3148, 23, 1, 2, 2, 1, 0, 136, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 3885, 3799, 0, 13, 0, 2395, 0, 97, 19, 220, 748, 13, 0, 0, 0, 18, 0, 15, 1, 0, 2237, 0, 4050, 0, 44, 0, 125, 335, 34, 99, 5, 538, 17, 407, 0, 0, 0, 86, 494, 0, 1560, 2435, 8, 88, 0, 354, 88, 1465, 1550, 0, 11, 1, 0, 0, 3642, 169, 7, 20, 0, 1142, 0, 534, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[125, 1021, 209, 1, 0, 1, 514, 3, 714, 0, 0, 973, 528, 0, 64, 34, 210, 21, 0, 24, 70, 1840, 1, 3728, 43, 31, 0, 1574, 40, 4, 45, 128, 2125, 1, 3, 1, 0, 1, 15, 1615, 31, 4009, 2296, 0, 3617, 0, 13, 312, 2174, 1404, 0, 198, 1, 0, 22, 2357, 127, 435, 10, 2, 10, 0, 43, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1993, 1428, 29, 0, 1, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 109, 28, 120, 16, 2, 13, 1256, 890, 0, 0, 0, 0, 0, 0, 2, 24, 13, 0, 1, 3932, 1, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 1, 0, 1078, 0, 4, 154, 2958, 1008, 0, 0, 0, 0, 0, 0, 141, 0, 26, 0, 9, 0, 43, 163, 1291, 402, 45, 0, 0, 0, 0, 0, 174, 0, 2192, 0, 1009, 2, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 792, 3802, 0, 2507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 188, 3, 8, 3, 0, 83, 0, 2508, 878, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3587, 2462, 5056, 5337, 2120, 3387, 7101, 3718] avg=4096.0000 max=7101 max_rank=6 min=2120 min_rank=4 imbalance_ratio=1.733643 max_load_reduction=25667 max_load_reduction_pct=0.783295 moved_to_echo=16326 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4, 1077, 1182, 0, 0, 0, 206, 17, 0, 0, 0, 0, 0, 0, 0, 0, 197, 7, 1733, 529, 210, 0, 16, 58, 1534, 1356, 0, 0, 0, 0, 0, 0, 268, 0, 0, 0, 4, 2224, 0, 1322, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 145, 736, 4, 43, 30, 455, 1231, 0, 0, 0, 0, 0, 0, 3745, 0, 126, 144, 8, 0, 17, 179, 1068, 55, 68, 0, 0, 0, 0, 0, 111, 0, 301, 0, 1298, 0, 1249, 33, 0, 0, 0, 0, 0, 0, 0, 0, 1205, 2501, 0, 1548, 12, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 9, 24, 0, 423, 0, 0, 0, 3907, 136, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2486, 5640, 3818, 2646, 5410, 2992, 5267, 4509] avg=4096.0000 max=5640 max_rank=1 min=2486 min_rank=0 imbalance_ratio=1.376953 max_load_reduction=27128 max_load_reduction_pct=0.827881 moved_to_echo=18178 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 3885, 1990, 0, 13, 0, 2395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 19, 220, 748, 13, 0, 0, 0, 1524, 760, 0, 0, 0, 0, 0, 0, 18, 0, 15, 1, 0, 2237, 0, 1304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 125, 335, 34, 99, 5, 1203, 1222, 0, 0, 0, 0, 0, 0, 538, 17, 407, 0, 0, 0, 86, 494, 1809, 381, 42, 0, 0, 0, 0, 0, 0, 0, 2054, 8, 88, 0, 354, 88, 0, 0, 0, 0, 0, 0, 0, 0, 663, 1550, 0, 11, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 7, 20, 0, 1142, 0, 534, 0, 3642, 357, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8283, 3381, 3575, 3067, 3774, 2592, 2225, 5871] avg=4096.0000 max=8283 max_rank=0 min=2225 min_rank=6 imbalance_ratio=2.022217 max_load_reduction=24485 max_load_reduction_pct=0.747223 moved_to_echo=14462 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 2620, 2082, 0, 2, 39, 1350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 257, 588, 73, 143, 0, 0, 1131, 971, 0, 0, 0, 0, 0, 0, 133, 0, 143, 0, 62, 3416, 2, 974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 11, 1334, 1, 79, 47, 1372, 907, 0, 0, 0, 0, 0, 0, 1968, 0, 663, 0, 32, 0, 0, 15, 1884, 460, 49, 0, 0, 0, 0, 0, 125, 0, 2510, 0, 54, 14, 99, 24, 0, 0, 0, 0, 0, 0, 0, 0, 864, 1983, 21, 440, 0, 8, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 1, 2, 2, 1, 0, 136, 0, 3148, 406, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6093, 3228, 4730, 3760, 5071, 2826, 3341, 3719] avg=4096.0000 max=6093 max_rank=0 min=2826 min_rank=5 imbalance_ratio=1.487549 max_load_reduction=26675 max_load_reduction_pct=0.814056 moved_to_echo=14957 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 2680, 187, 4, 9, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 342, 3, 31, 839, 45, 0, 0, 2, 1524, 1659, 0, 0, 0, 0, 0, 0, 69, 0, 172, 0, 0, 1607, 0, 1313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 9, 792, 22, 255, 700, 627, 1222, 0, 0, 0, 0, 0, 0, 2334, 0, 70, 0, 48, 0, 94, 2038, 168, 206, 83, 0, 0, 0, 0, 0, 4, 0, 1128, 2, 3690, 14, 968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1475, 794, 3, 787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 215, 29, 3, 0, 1427, 0, 29, 1, 2857, 185, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2881, 4445, 3161, 3629, 5041, 5806, 3059, 4746] avg=4096.0000 max=5806 max_rank=5 min=2881 min_rank=0 imbalance_ratio=1.417480 max_load_reduction=26962 max_load_reduction_pct=0.822815 moved_to_echo=18652 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[125, 1021, 110, 1, 0, 1, 514, 3, 0, 0, 0, 0, 0, 0, 0, 0, 714, 0, 0, 973, 528, 0, 64, 34, 1399, 1121, 0, 0, 0, 0, 0, 0, 210, 21, 0, 24, 70, 1840, 1, 1206, 0, 0, 0, 0, 0, 0, 0, 0, 43, 31, 0, 1574, 40, 4, 45, 128, 3092, 1123, 0, 0, 0, 0, 0, 0, 2125, 1, 3, 1, 0, 1, 15, 1615, 99, 356, 56, 0, 0, 0, 0, 0, 31, 0, 1940, 0, 3617, 0, 13, 312, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1404, 0, 198, 1, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 435, 10, 2, 10, 0, 43, 0, 2357, 917, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1775, 4833, 3372, 6080, 4272, 5913, 2622, 3901] avg=4096.0000 max=6080 max_rank=3 min=1775 min_rank=0 imbalance_ratio=1.484375 max_load_reduction=26688 max_load_reduction_pct=0.814453 moved_to_echo=16708 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[49, 2798, 1512, 1, 0, 0, 738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 27, 1518, 58, 0, 1, 1, 1371, 1198, 0, 0, 0, 0, 0, 0, 45, 2, 1, 0, 0, 2594, 37, 1181, 0, 0, 0, 0, 0, 0, 0, 0, 5, 193, 0, 5, 443, 23, 12, 74, 1921, 1100, 0, 0, 0, 0, 0, 0, 1605, 11, 72, 10, 0, 1, 41, 574, 1367, 168, 60, 0, 0, 0, 0, 0, 1, 0, 919, 0, 88, 1, 2323, 456, 0, 0, 0, 0, 0, 0, 0, 0, 1066, 810, 0, 3331, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 262, 3, 36, 25, 0, 8, 0, 1971, 571, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5098, 4218, 3860, 3776, 3909, 3788, 5208, 2911] avg=4096.0000 max=5208 max_rank=6 min=2911 min_rank=7 imbalance_ratio=1.271484 max_load_reduction=27560 max_load_reduction_pct=0.841064 moved_to_echo=15816 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[5, 469, 1901, 0, 0, 0, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 50, 971, 1627, 6, 38, 0, 247, 1612, 0, 0, 0, 0, 0, 0, 1218, 0, 0, 2, 18, 3745, 0, 213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 57, 1071, 2, 3, 8, 3083, 198, 0, 0, 0, 0, 0, 0, 431, 0, 178, 0, 0, 4, 1, 386, 1719, 371, 81, 0, 0, 0, 0, 0, 56, 0, 2024, 0, 3272, 0, 54, 58, 0, 0, 0, 0, 0, 0, 0, 0, 1433, 1719, 0, 788, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 423, 702, 19, 0, 264, 0, 4, 0, 1065, 916, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2565, 4561, 5196, 4473, 3171, 5464, 3945, 3393] avg=4096.0000 max=5464 max_rank=5 min=2565 min_rank=0 imbalance_ratio=1.333984 max_load_reduction=27304 max_load_reduction_pct=0.833252 moved_to_echo=15973 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 512, 128, 2048, 5888, 128, 640, 1152, 14720, 8064, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[256, 16640, 10496, 128, 128, 128, 5632, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1536, 128, 2432, 6272, 2688, 256, 128, 128, 10112, 9600, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[12928, 128, 1664, 256, 128, 128, 384, 5504, 9472, 2432, 512, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8576, 14592, 128, 9728, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[512, 0, 13184, 128, 13184, 128, 5120, 1024, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[2048, 128, 384, 128, 256, 21632, 128, 8704, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1152, 1664, 128, 128, 3328, 0, 896, 128, 21504, 4480, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[2336, 0, 77, 0, 319, 0, 0, 0, 0, 137, 0, 2, 2, 0, 0, 0, 0, 4076, 0, 63, 1, 1, 0, 0, 0, 0, 0, 0, 3824, 0, 0, 0, 1, 0, 2112, 0, 508, 1, 39, 0, 0, 5, 0, 3627, 3613, 990, 0, 0, 0, 3226, 4, 5, 136, 1, 0, 4068, 3551, 22, 0, 0, 0, 0, 21, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[481, 0, 0, 0, 9, 1, 0, 0, 0, 467, 5, 493, 0, 0, 0, 0, 1, 3751, 9, 3891, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 41, 0, 0, 59, 15, 3992, 0, 1, 0, 2, 0, 0, 0, 0, 3873, 455, 4016, 684, 3, 1, 62, 0, 0, 2, 47, 0, 3497, 256, 0, 0, 0, 0, 0, 2558, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[2, 48, 0, 21, 158, 0, 1, 1, 8, 0, 0, 3368, 0, 0, 34, 0, 0, 4094, 0, 524, 53, 0, 0, 0, 0, 1, 0, 0, 491, 2353, 0, 0, 0, 2, 4049, 0, 2553, 0, 1, 0, 17, 0, 0, 3117, 2593, 5, 0, 1, 0, 3777, 0, 0, 1, 128, 0, 3998, 21, 0, 0, 0, 1, 0, 1278, 69] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[3806, 1, 7, 0, 19, 0, 1, 2, 5, 247, 0, 85, 10, 1, 0, 0, 12, 3889, 288, 0, 0, 20, 3, 0, 0, 0, 1, 50, 2930, 2861, 0, 1, 98, 0, 1385, 0, 2588, 0, 1414, 0, 0, 0, 4, 4069, 1802, 1662, 78, 0, 5, 321, 2, 0, 1, 0, 312, 4015, 84, 0, 0, 2, 0, 0, 687, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1194, 0, 50, 0, 69, 2, 0, 0, 186, 616, 1390, 98, 0, 0, 0, 40, 2, 4048, 3, 4043, 0, 0, 0, 1, 0, 0, 0, 1, 909, 494, 0, 0, 48, 2, 3972, 0, 2098, 0, 2327, 0, 5, 0, 0, 3793, 10, 2659, 0, 0, 1, 316, 252, 0, 0, 0, 84, 1633, 15, 6, 0, 1, 0, 5, 2295, 100] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[3243, 1, 0, 0, 1, 0, 0, 11, 0, 0, 0, 4091, 0, 0, 0, 1, 0, 4089, 13, 2269, 0, 0, 0, 2, 0, 0, 0, 0, 4090, 0, 0, 0, 19, 0, 696, 5, 467, 0, 109, 0, 0, 0, 0, 3988, 1, 2, 0, 0, 1012, 3512, 3, 5, 4, 0, 0, 4062, 356, 0, 676, 0, 0, 0, 40, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1667, 0, 2, 217, 2, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 4096, 1, 389, 0, 0, 0, 0, 0, 9, 0, 2, 3977, 23, 0, 82, 2732, 68, 4018, 0, 2211, 1, 508, 0, 0, 0, 0, 4049, 74, 454, 203, 0, 0, 313, 9, 3, 44, 0, 0, 4008, 2088, 0, 0, 0, 0, 0, 1502, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[3120, 0, 285, 0, 7, 0, 8, 0, 2, 10, 0, 3824, 25, 0, 8, 0, 0, 4087, 0, 2469, 0, 0, 0, 0, 2, 0, 0, 0, 4075, 81, 0, 0, 1, 1, 185, 0, 2117, 0, 23, 0, 2, 0, 0, 1486, 947, 93, 14, 0, 0, 3021, 0, 144, 1, 1, 0, 3228, 454, 0, 0, 0, 0, 0, 3047, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[481, 0, 0, 0, 9, 1, 0, 0, 1336, 509, 0, 0, 0, 0, 0, 0, 0, 467, 5, 493, 0, 0, 0, 0, 2161, 230, 0, 0, 0, 0, 0, 0, 1, 2185, 9, 3891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 41, 0, 0, 464, 0, 0, 0, 0, 0, 0, 0, 59, 15, 2495, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1712, 455, 4016, 684, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 62, 0, 0, 2, 47, 0, 1965, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0, 0, 2558, 0, 1023, 1033, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2336, 3356, 6086, 4601, 2572, 6870, 2077, 4870] avg=4096.0000 max=6870 max_rank=5 min=2077 min_rank=6 imbalance_ratio=1.677246 max_load_reduction=25898 max_load_reduction_pct=0.790344 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2336, 0, 77, 0, 319, 0, 0, 0, 1451, 592, 0, 0, 0, 0, 0, 0, 0, 137, 0, 2, 2, 0, 0, 0, 2023, 250, 0, 0, 0, 0, 0, 0, 0, 2375, 0, 63, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3824, 0, 0, 0, 245, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1320, 0, 508, 1, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 1604, 3613, 990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3226, 4, 5, 136, 1, 0, 2286, 0, 0, 0, 0, 0, 0, 0, 0, 3551, 22, 0, 0, 0, 0, 21, 0, 1190, 547, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4775, 2414, 2440, 4069, 1869, 6212, 5658, 5331] avg=4096.0000 max=6212 max_rank=5 min=1869 min_rank=4 imbalance_ratio=1.516602 max_load_reduction=26556 max_load_reduction_pct=0.810425 moved_to_echo=19070 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1194, 0, 50, 0, 69, 2, 0, 0, 1441, 237, 0, 0, 0, 0, 0, 0, 186, 616, 1390, 98, 0, 0, 0, 40, 2116, 248, 0, 0, 0, 0, 0, 0, 2, 2359, 3, 4043, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 909, 494, 0, 0, 462, 0, 0, 0, 0, 0, 0, 0, 48, 2, 2482, 0, 2098, 0, 2327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 1677, 10, 2659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 316, 252, 0, 0, 0, 84, 919, 0, 0, 0, 0, 0, 0, 0, 0, 15, 6, 0, 1, 0, 5, 2295, 100, 477, 1028, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2993, 4694, 6408, 1866, 6957, 4351, 1572, 3927] avg=4096.0000 max=6957 max_rank=4 min=1572 min_rank=6 imbalance_ratio=1.698486 max_load_reduction=25811 max_load_reduction_pct=0.787689 moved_to_echo=16807 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3806, 1, 7, 0, 19, 0, 1, 2, 1385, 585, 0, 0, 0, 0, 0, 0, 5, 247, 0, 85, 10, 1, 0, 0, 2270, 239, 0, 0, 0, 0, 0, 0, 12, 2265, 288, 0, 0, 20, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 50, 2930, 2861, 0, 1, 161, 0, 0, 0, 0, 0, 0, 0, 98, 0, 866, 0, 2588, 0, 1414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1799, 1802, 1662, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 321, 2, 0, 1, 0, 312, 2256, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 2, 0, 0, 687, 0, 1174, 358, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5806, 2857, 2588, 6004, 4966, 5345, 2897, 2305] avg=4096.0000 max=6004 max_rank=3 min=2305 min_rank=7 imbalance_ratio=1.465820 max_load_reduction=26764 max_load_reduction_pct=0.816772 moved_to_echo=15513 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3120, 0, 285, 0, 7, 0, 8, 0, 1455, 470, 0, 0, 0, 0, 0, 0, 2, 10, 0, 3824, 25, 0, 8, 0, 829, 251, 0, 0, 0, 0, 0, 0, 0, 2381, 0, 2469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4075, 81, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 1, 1, 117, 0, 2117, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 657, 947, 93, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3021, 0, 144, 1, 1, 0, 1814, 0, 0, 0, 0, 0, 0, 0, 0, 454, 0, 0, 0, 0, 0, 3047, 0, 944, 47, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5345, 4949, 4850, 4179, 2259, 1713, 4981, 4492] avg=4096.0000 max=5345 max_rank=0 min=1713 min_rank=5 imbalance_ratio=1.304932 max_load_reduction=27423 max_load_reduction_pct=0.836884 moved_to_echo=13445 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 48, 0, 21, 158, 0, 1, 1, 1458, 582, 0, 0, 0, 0, 0, 0, 8, 0, 0, 3368, 0, 0, 34, 0, 1739, 251, 0, 0, 0, 0, 0, 0, 0, 2385, 0, 524, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 491, 2353, 0, 0, 470, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2531, 0, 2553, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 1378, 2593, 5, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3777, 0, 0, 1, 128, 0, 2247, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 1, 0, 1278, 69, 1169, 1048, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2271, 5400, 2962, 3315, 5087, 3994, 6153, 3586] avg=4096.0000 max=6153 max_rank=6 min=2271 min_rank=0 imbalance_ratio=1.502197 max_load_reduction=26615 max_load_reduction_pct=0.812225 moved_to_echo=18820 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1667, 0, 2, 217, 2, 0, 0, 0, 1458, 584, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 2259, 251, 0, 0, 0, 0, 0, 0, 0, 2387, 1, 389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 2, 3977, 23, 0, 82, 467, 0, 0, 0, 0, 0, 0, 0, 2732, 68, 2511, 0, 2211, 1, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1790, 74, 454, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 313, 9, 3, 44, 0, 0, 2252, 0, 0, 0, 0, 0, 0, 0, 0, 2088, 0, 0, 0, 0, 0, 1502, 5, 1172, 1040, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3930, 2521, 2777, 4560, 8031, 2521, 2621, 5807] avg=4096.0000 max=8031 max_rank=4 min=2521 min_rank=1 imbalance_ratio=1.960693 max_load_reduction=24737 max_load_reduction_pct=0.754913 moved_to_echo=18980 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3243, 1, 0, 0, 1, 0, 0, 11, 1460, 596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 1, 2229, 256, 0, 0, 0, 0, 0, 0, 0, 2373, 13, 2269, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 19, 0, 427, 5, 467, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1759, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 3512, 3, 5, 4, 0, 0, 2273, 0, 0, 0, 0, 0, 0, 0, 0, 356, 0, 676, 0, 0, 0, 40, 0, 1193, 185, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5312, 6577, 4657, 4174, 1027, 1762, 6809, 2450] avg=4096.0000 max=6809 max_rank=6 min=1027 min_rank=4 imbalance_ratio=1.662354 max_load_reduction=25959 max_load_reduction_pct=0.792206 moved_to_echo=12048 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3072, 128, 12800, 128, 12544, 128, 4480, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[6912, 128, 768, 128, 128, 128, 11520, 256, 8448, 5376, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[15872, 128, 512, 256, 640, 128, 128, 128, 11520, 4224, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 12416, 9600, 9984, 1024, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[256, 1536, 1408, 12032, 128, 128, 128, 128, 15744, 2048, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1024, 14592, 384, 256, 256, 256, 512, 16128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 18816, 384, 13696, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 24448, 5888, 0, 128, 2432, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33792 slack_slots=31744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 268, 0, 3926, 4038, 0, 4086, 34, 0, 0, 211, 20, 0, 0, 0, 2106, 7, 3408, 0, 46, 7, 0, 0, 0, 6, 0, 3011, 0, 2849, 0, 1, 0, 4066, 0, 3714, 0, 0, 0, 0, 0, 0, 0, 932, 0, 0, 0, 1, 1, 7, 0, 1, 0, 0, 0, 2, 0, 3, 0, 0, 6, 11, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 89, 0, 4096, 4052, 591, 3914, 0, 0, 240, 0, 0, 0, 0, 1472, 3387, 0, 76, 0, 0, 9, 0, 0, 0, 0, 0, 3179, 0, 4075, 0, 0, 20, 4096, 6, 3323, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 13, 0, 33, 0, 81] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4074, 3916, 41, 4082, 0, 0, 1, 0, 0, 424, 0, 62, 3290, 0, 3179, 0, 0, 5, 0, 38, 1368, 0, 192, 3795, 0, 4061, 0, 0, 22, 4047, 1, 70, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 5, 1, 5, 0, 16, 28, 0, 0, 0, 0, 0, 7, 0, 0, 30, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 468, 1, 4046, 4064, 360, 4096, 0, 2, 0, 0, 0, 0, 0, 1738, 823, 0, 3297, 0, 0, 2, 33, 0, 114, 0, 0, 3923, 0, 4093, 0, 2, 3, 4092, 0, 1319, 0, 0, 0, 1, 2, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 6, 0, 0, 97, 1, 0, 0, 173] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[26, 1, 0, 0, 3789, 2943, 392, 3997, 0, 100, 0, 0, 0, 0, 0, 233, 1168, 0, 3919, 0, 56, 0, 0, 0, 0, 1, 50, 4043, 0, 3371, 1, 8, 0, 3757, 927, 3463, 0, 0, 0, 0, 14, 0, 0, 56, 0, 10, 27, 0, 1, 18, 0, 0, 0, 126, 0, 0, 0, 0, 1, 56, 52, 0, 0, 162] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[458, 3, 421, 0, 4096, 4062, 1536, 3773, 0, 0, 91, 0, 0, 0, 0, 0, 2624, 0, 2929, 0, 1586, 0, 0, 0, 183, 0, 2, 1352, 0, 3746, 0, 14, 390, 2152, 6, 185, 0, 17, 0, 27, 3, 0, 1, 1, 0, 5, 0, 4, 0, 0, 0, 14, 0, 0, 1, 0, 0, 115, 35, 0, 2936, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 1218, 0, 4073, 3642, 463, 4055, 0, 71, 0, 3, 23, 0, 0, 0, 2969, 21, 3967, 89, 248, 0, 0, 1, 0, 0, 223, 155, 0, 3466, 5, 0, 119, 3961, 0, 3779, 0, 129, 0, 0, 2, 0, 0, 0, 0, 0, 55, 0, 0, 0, 1, 0, 2, 0, 0, 0, 4, 0, 0, 22, 0, 1, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[9, 0, 532, 0, 4096, 4090, 920, 3913, 1, 0, 81, 0, 0, 1322, 1, 530, 1158, 0, 0, 0, 99, 0, 0, 0, 69, 0, 0, 4093, 0, 4096, 1, 1, 136, 4094, 17, 3475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 22] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 268, 0, 0, 3273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 211, 20, 0, 0, 0, 1678, 1276, 765, 0, 0, 0, 0, 0, 2106, 7, 2132, 0, 46, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 3011, 0, 669, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1561, 0, 3714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 932, 0, 0, 0, 1, 4043, 4, 0, 0, 0, 0, 0, 0, 1, 7, 0, 1, 0, 0, 0, 2, 3926, 43, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 6, 11, 0, 0, 2176, 827, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3541, 3984, 4298, 3687, 5275, 4980, 3980, 3023] avg=4096.0000 max=5275 max_rank=4 min=3023 min_rank=7 imbalance_ratio=1.287842 max_load_reduction=27493 max_load_reduction_pct=0.839020 moved_to_echo=17258 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 89, 0, 0, 3285, 591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 0, 0, 0, 0, 1472, 1690, 28, 767, 0, 0, 0, 0, 0, 3387, 0, 48, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3179, 0, 956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 1572, 6, 3323, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3873, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 4096, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 33, 0, 81, 3113, 834, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3965, 4197, 3444, 4135, 4929, 3883, 4141, 4074] avg=4096.0000 max=4929 max_rank=4 min=3444 min_rank=2 imbalance_ratio=1.203369 max_load_reduction=27839 max_load_reduction_pct=0.849579 moved_to_echo=17027 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 468, 1, 0, 3294, 360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1738, 1688, 1234, 770, 0, 0, 0, 0, 0, 823, 0, 2063, 0, 0, 2, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 0, 0, 3923, 0, 960, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1571, 0, 1319, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3, 0, 0, 0, 0, 4053, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4046, 43, 0, 0, 0, 0, 0, 0, 6, 0, 0, 97, 1, 0, 0, 173, 3127, 833, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4123, 5432, 2921, 4999, 2894, 4064, 4098, 4237] avg=4096.0000 max=5432 max_rank=1 min=2894 min_rank=4 imbalance_ratio=1.326172 max_load_reduction=27336 max_load_reduction_pct=0.834229 moved_to_echo=15293 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 0, 3174, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 424, 0, 62, 1670, 1190, 742, 0, 0, 0, 0, 0, 3290, 0, 1989, 0, 0, 5, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 1368, 0, 192, 3795, 0, 953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 1553, 1, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 5, 1, 4040, 6, 0, 0, 0, 0, 0, 0, 5, 0, 16, 28, 0, 0, 0, 0, 4074, 42, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 30, 1, 0, 0, 3102, 824, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3217, 4089, 5322, 6308, 1646, 4057, 4165, 3964] avg=4096.0000 max=6308 max_rank=3 min=1646 min_rank=4 imbalance_ratio=1.540039 max_load_reduction=26460 max_load_reduction_pct=0.807495 moved_to_echo=13832 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[458, 3, 421, 0, 0, 3293, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 0, 0, 888, 1096, 769, 0, 0, 0, 0, 0, 2624, 0, 1833, 0, 1586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 0, 2, 1352, 0, 879, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 390, 826, 6, 185, 0, 17, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 1, 0, 5, 0, 4, 3733, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 1, 0, 4096, 40, 0, 0, 0, 0, 0, 0, 0, 115, 35, 0, 2936, 0, 0, 0, 2862, 438, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5711, 2844, 6043, 2430, 1451, 3752, 4151, 6386] avg=4096.0000 max=6386 max_rank=7 min=1451 min_rank=4 imbalance_ratio=1.559082 max_load_reduction=26382 max_load_reduction_pct=0.805115 moved_to_echo=15740 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1218, 0, 0, 2952, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 3, 23, 0, 0, 0, 1634, 1488, 690, 0, 0, 0, 0, 0, 2969, 21, 2479, 89, 248, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 223, 155, 0, 813, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 1521, 0, 3779, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 55, 0, 4013, 5, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 4073, 42, 0, 0, 0, 0, 0, 0, 4, 0, 0, 22, 0, 1, 0, 1, 2648, 806, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4633, 3909, 5807, 1196, 5548, 4075, 4118, 3482] avg=4096.0000 max=5807 max_rank=2 min=1196 min_rank=3 imbalance_ratio=1.417725 max_load_reduction=26961 max_load_reduction_pct=0.822784 moved_to_echo=17223 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[26, 1, 0, 0, 0, 2386, 392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 233, 1550, 1467, 557, 0, 0, 0, 0, 0, 1168, 0, 2452, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 50, 4043, 0, 791, 1, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1442, 927, 3463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 56, 0, 10, 27, 0, 3956, 5, 0, 0, 0, 0, 0, 0, 1, 18, 0, 0, 0, 126, 0, 0, 3789, 41, 0, 0, 0, 0, 0, 0, 0, 0, 1, 56, 52, 0, 0, 162, 2575, 765, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2805, 3907, 3676, 4894, 5832, 4068, 3975, 3611] avg=4096.0000 max=5832 max_rank=4 min=2805 min_rank=0 imbalance_ratio=1.423828 max_load_reduction=26936 max_load_reduction_pct=0.822021 moved_to_echo=17486 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[9, 0, 532, 0, 0, 3312, 920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 81, 0, 0, 1322, 1, 530, 1693, 0, 778, 0, 0, 0, 0, 0, 1158, 0, 0, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 4093, 0, 955, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 136, 1565, 17, 3475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 3873, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 4096, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 3133, 836, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4773, 4406, 1257, 5119, 5193, 3889, 4140, 3991] avg=4096.0000 max=5193 max_rank=4 min=1257 min_rank=2 imbalance_ratio=1.267822 max_load_reduction=27575 max_load_reduction_pct=0.841522 moved_to_echo=17213 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 256, 512, 256, 128, 1792, 128, 4096, 12544, 7808, 5888, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 256, 3072, 128, 0, 512, 22784, 6272, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[17536, 128, 13056, 128, 2048, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[768, 11648, 1024, 19328, 0, 256, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1792, 128, 512, 23552, 0, 7040, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[512, 128, 3072, 128, 0, 25088, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 0, 128, 1024, 128, 128, 128, 128, 31616, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 128, 256, 128, 128, 32256, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33792 slack_slots=31744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 626, 0, 94, 0, 124, 0, 0, 0, 8, 0, 0, 0, 0, 4027, 0, 1, 4096, 4048, 0, 0, 2017, 0, 0, 0, 149, 4094, 0, 0, 0, 931, 0, 1, 0, 0, 143, 3128, 4069, 1, 0, 0, 0, 100, 1, 0, 0, 4092, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 890, 0, 0, 125, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[9, 3, 1935, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3613, 33, 41, 4091, 4035, 125, 76, 0, 23, 0, 0, 3040, 4057, 0, 0, 2, 65, 0, 0, 6, 1, 2054, 0, 1394, 0, 0, 0, 50, 0, 38, 0, 0, 3985, 0, 0, 0, 0, 0, 7, 0, 0, 4, 2, 4044, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 194, 0, 1, 0, 0, 0, 29, 2025, 280, 2, 4034, 4023, 0, 1182, 71, 0, 35, 0, 4045, 4068, 1, 0, 0, 0, 0, 0, 3161, 0, 166, 278, 894, 6, 0, 1, 25, 0, 5, 0, 14, 4045, 0, 0, 8, 0, 66, 1, 0, 0, 0, 0, 4094, 0, 13, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 304, 2582, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 0, 4096, 0, 0, 4083, 4078, 0, 0, 0, 0, 1, 0, 0, 4093, 0, 0, 0, 3, 0, 0, 6, 0, 3749, 1, 3457, 23, 291, 0, 25, 0, 0, 0, 2, 1786, 2, 89, 27, 0, 0, 1, 0, 0, 0, 0, 4055, 0, 7, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1988, 0, 0, 0, 7, 0, 1, 120, 137, 0, 217, 0, 10, 0, 0, 6, 3, 0, 4069, 3975, 4, 0, 0, 404, 9, 35, 3600, 4071, 0, 5, 25, 0, 0, 0, 0, 0, 1626, 9, 1007, 0, 417, 0, 0, 0, 0, 6, 0, 3529, 3892, 2, 141, 0, 0, 0, 0, 0, 39, 1, 3383, 30, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[129, 0, 1133, 0, 0, 0, 0, 0, 98, 0, 0, 0, 0, 0, 35, 764, 877, 0, 4010, 3887, 0, 2, 7, 0, 0, 0, 3998, 3381, 0, 8, 0, 0, 2, 0, 1469, 0, 138, 2106, 4079, 1, 0, 0, 5, 0, 0, 0, 127, 3353, 0, 85, 0, 23, 0, 0, 0, 0, 14, 0, 3037, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 2, 357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1219, 0, 185, 4064, 4094, 138, 0, 3072, 0, 0, 0, 837, 3872, 0, 0, 1, 3493, 0, 12, 0, 0, 164, 58, 2128, 0, 0, 7, 0, 0, 0, 0, 0, 3867, 0, 4, 55, 0, 8, 6, 0, 1, 0, 0, 3085, 0, 0, 2031, 0, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[50, 0, 0, 0, 0, 1, 2919, 0, 1, 0, 0, 0, 0, 1, 0, 463, 0, 0, 4091, 3785, 0, 402, 754, 9, 0, 0, 123, 4073, 0, 0, 0, 1, 0, 2, 0, 0, 3502, 1, 4068, 1, 2, 0, 0, 10, 317, 0, 0, 4018, 0, 0, 0, 0, 14, 0, 0, 0, 408, 0, 3658, 0, 0, 94, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 626, 0, 94, 0, 124, 0, 2084, 529, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 4027, 1956, 413, 313, 0, 0, 0, 0, 0, 0, 1, 0, 2925, 0, 0, 2017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 1597, 0, 0, 0, 931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 143, 3128, 2113, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 1, 0, 0, 4092, 336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 890, 0, 0, 125, 0, 2, 474, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3457, 6717, 4943, 2677, 5386, 4529, 3568, 1491] avg=4096.0000 max=6717 max_rank=1 min=1491 min_rank=7 imbalance_ratio=1.639893 max_load_reduction=26051 max_load_reduction_pct=0.795013 moved_to_echo=14974 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[129, 0, 1133, 0, 0, 0, 0, 0, 1721, 517, 0, 0, 0, 0, 0, 0, 98, 0, 0, 0, 0, 0, 35, 764, 1961, 341, 300, 0, 0, 0, 0, 0, 877, 0, 0, 2810, 0, 2, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3998, 1319, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1469, 0, 138, 2106, 2118, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 127, 3353, 322, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 23, 0, 0, 0, 0, 3493, 0, 0, 0, 0, 0, 0, 0, 14, 0, 3037, 0, 0, 0, 0, 0, 455, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3500, 3499, 3696, 5325, 5834, 3807, 3601, 3506] avg=4096.0000 max=5834 max_rank=4 min=3499 min_rank=1 imbalance_ratio=1.424316 max_load_reduction=26934 max_load_reduction_pct=0.821960 moved_to_echo=16748 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[9, 3, 1935, 0, 35, 0, 0, 0, 2065, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3613, 670, 409, 312, 0, 0, 0, 0, 0, 33, 41, 0, 2916, 125, 76, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3040, 1583, 0, 0, 2, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 2054, 0, 724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 38, 0, 0, 3985, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 3563, 0, 0, 0, 0, 0, 0, 0, 4, 2, 4044, 0, 0, 0, 0, 0, 473, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4575, 5004, 3214, 4690, 2785, 4407, 3570, 4523] avg=4096.0000 max=5004 max_rank=1 min=2785 min_rank=4 imbalance_ratio=1.221680 max_load_reduction=27764 max_load_reduction_pct=0.847290 moved_to_echo=15285 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 304, 2582, 0, 0, 0, 0, 2084, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 0, 4096, 1662, 413, 315, 0, 0, 0, 0, 0, 0, 0, 0, 2947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1596, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 3749, 1, 1795, 23, 0, 0, 0, 0, 0, 0, 0, 0, 291, 0, 25, 0, 0, 0, 2, 1786, 338, 0, 0, 0, 0, 0, 0, 0, 2, 89, 27, 0, 0, 1, 0, 0, 3555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 7, 0, 0, 0, 478, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5498, 6493, 2947, 1600, 5574, 2442, 3674, 4540] avg=4096.0000 max=6493 max_rank=1 min=1600 min_rank=3 imbalance_ratio=1.585205 max_load_reduction=26275 max_load_reduction_pct=0.801849 moved_to_echo=16230 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 357, 0, 0, 0, 0, 0, 1971, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1219, 1023, 390, 317, 0, 0, 0, 0, 0, 0, 185, 0, 2959, 138, 0, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 837, 1511, 0, 0, 1, 3493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 164, 58, 1105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 3867, 339, 0, 0, 0, 0, 0, 0, 0, 0, 4, 55, 0, 8, 6, 0, 1, 3539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3085, 0, 0, 2031, 0, 8, 479, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2855, 2949, 6354, 5842, 1339, 4213, 3613, 5603] avg=4096.0000 max=6354 max_rank=2 min=1339 min_rank=4 imbalance_ratio=1.551270 max_load_reduction=26414 max_load_reduction_pct=0.806091 moved_to_echo=14768 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 2071, 522, 0, 0, 0, 0, 0, 0, 194, 0, 1, 0, 0, 0, 29, 2025, 429, 410, 311, 0, 0, 0, 0, 0, 280, 2, 0, 2908, 0, 1182, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 4045, 1587, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3161, 0, 166, 278, 465, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 25, 0, 5, 0, 14, 4045, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 66, 1, 0, 0, 3512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 13, 0, 1, 0, 471, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2593, 3399, 4443, 5668, 4076, 4423, 3587, 4579] avg=4096.0000 max=5668 max_rank=3 min=2593 min_rank=0 imbalance_ratio=1.383789 max_load_reduction=27100 max_load_reduction_pct=0.827026 moved_to_echo=16665 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1988, 0, 0, 0, 7, 0, 1, 120, 2072, 525, 0, 0, 0, 0, 0, 0, 137, 0, 217, 0, 10, 0, 0, 6, 484, 411, 307, 0, 0, 0, 0, 0, 3, 0, 0, 2873, 4, 0, 0, 404, 0, 0, 0, 0, 0, 0, 0, 0, 9, 35, 3600, 1588, 0, 5, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1626, 9, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 417, 0, 0, 0, 0, 6, 0, 3529, 329, 0, 0, 0, 0, 0, 0, 0, 3892, 2, 141, 0, 0, 0, 0, 0, 3544, 0, 0, 0, 0, 0, 0, 0, 39, 1, 3383, 30, 0, 0, 0, 0, 466, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4713, 1572, 3284, 5262, 2158, 4281, 7579, 3919] avg=4096.0000 max=7579 max_rank=6 min=1572 min_rank=1 imbalance_ratio=1.850342 max_load_reduction=25189 max_load_reduction_pct=0.768707 moved_to_echo=17937 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[50, 0, 0, 0, 0, 1, 2919, 0, 2078, 529, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 463, 1958, 415, 297, 0, 0, 0, 0, 0, 0, 0, 0, 2722, 0, 402, 754, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 1580, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3502, 1, 2110, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 10, 317, 0, 0, 4018, 319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 3562, 0, 0, 0, 0, 0, 0, 0, 408, 0, 3658, 0, 0, 94, 0, 0, 447, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5577, 3135, 3887, 1704, 5616, 4666, 3576, 4607] avg=4096.0000 max=5616 max_rank=4 min=1704 min_rank=3 imbalance_ratio=1.371094 max_load_reduction=27152 max_load_reduction_pct=0.828613 moved_to_echo=18465 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[2176, 128, 4480, 2688, 256, 128, 3072, 128, 16256, 4224, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 4736, 128, 11648, 5632, 11008, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 15872, 12416, 128, 128, 128, 4608, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[512, 0, 256, 0, 128, 128, 128, 16256, 10240, 3328, 2560, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1280, 256, 0, 23168, 384, 1664, 6016, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[512, 128, 26368, 128, 128, 2304, 128, 128, 3840, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[768, 128, 128, 128, 384, 128, 256, 28800, 2688, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3968, 256, 256, 128, 128, 128, 0, 128, 28416, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 2, 0, 4090, 0, 0, 3589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 0, 0, 4091, 0, 0, 0, 2, 1, 0, 0, 0, 4083, 4, 0, 264, 0, 1, 0, 1, 107, 0, 0, 398, 0, 0, 0, 0, 0, 8, 0, 1, 820, 0, 0, 4096, 20, 0, 1, 0, 0, 4094, 0, 3033, 0, 0, 0, 23] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 3964, 1, 0, 3139, 1, 0, 0, 37, 0, 0, 0, 0, 6, 4017, 6, 0, 3948, 0, 0, 0, 51, 481, 0, 0, 0, 2533, 0, 16, 568, 6, 13, 0, 2, 0, 1, 604, 1701, 0, 0, 2, 0, 1, 0, 8, 0, 955, 0, 3, 4096, 3624, 0, 0, 0, 0, 0, 0, 2831, 0, 6, 147, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 4071, 35, 0, 3933, 0, 3, 0, 0, 235, 0, 0, 0, 0, 3737, 0, 0, 2515, 17, 0, 0, 0, 1, 59, 1390, 1, 3817, 104, 0, 477, 0, 0, 0, 0, 25, 0, 6, 113, 0, 0, 0, 1, 0, 0, 0, 0, 4053, 0, 31, 3983, 36, 0, 0, 21, 0, 3551, 545, 6, 2, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 4066, 0, 0, 3866, 0, 0, 0, 1, 0, 0, 0, 2, 0, 4096, 9, 0, 4011, 0, 31, 0, 0, 65, 10, 67, 0, 10, 3, 0, 3096, 0, 112, 60, 0, 390, 0, 0, 50, 0, 0, 0, 0, 0, 0, 1, 3873, 34, 0, 0, 3766, 3499, 0, 0, 0, 0, 417, 0, 1129, 0, 0, 104, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 4078, 1, 1, 4061, 0, 0, 10, 0, 0, 3, 81, 0, 0, 4082, 4, 0, 3926, 1, 0, 8, 0, 23, 0, 0, 7, 1139, 0, 0, 4074, 8, 0, 0, 0, 0, 0, 273, 14, 5, 0, 0, 0, 46, 1, 0, 236, 0, 0, 0, 4075, 0, 0, 246, 0, 1, 2891, 0, 3388, 28, 0, 0, 57] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 4040, 0, 0, 403, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3383, 0, 0, 3943, 0, 0, 0, 0, 0, 0, 451, 0, 1037, 0, 0, 999, 0, 536, 0, 89, 8, 22, 228, 4094, 0, 0, 0, 609, 0, 0, 0, 0, 1752, 0, 3, 3449, 1114, 0, 0, 0, 0, 1064, 0, 3988, 6, 0, 1541, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 12, 0, 4084, 0, 105, 3014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2875, 0, 0, 0, 794, 23, 9, 482, 2, 0, 124, 0, 4016, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 4028, 0, 79, 0, 0, 0, 0, 23, 0, 882, 0, 0, 2163, 3787, 0, 5, 0, 0, 3219, 0, 2994, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 4075, 0, 0, 3541, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4057, 2, 0, 4094, 4, 0, 0, 0, 0, 0, 2136, 0, 4084, 11, 1, 2, 0, 1, 0, 0, 3291, 0, 0, 98, 0, 0, 0, 3, 0, 0, 0, 522, 34, 0, 0, 4090, 72, 0, 0, 0, 0, 2251, 0, 373, 0, 0, 23, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 891, 0, 0, 3589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3399, 868, 0, 0, 0, 0, 0, 0, 640, 0, 0, 4091, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3613, 4, 0, 264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 107, 0, 0, 398, 1254, 846, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 1, 2331, 1218, 0, 0, 0, 0, 0, 0, 820, 0, 0, 1624, 20, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2187, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4482, 4267, 4733, 3882, 3077, 3558, 2465, 6304] avg=4096.0000 max=6304 max_rank=7 min=2465 min_rank=6 imbalance_ratio=1.539062 max_load_reduction=26464 max_load_reduction_pct=0.807617 moved_to_echo=15404 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 864, 1, 0, 3139, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 6, 3380, 841, 0, 0, 0, 0, 0, 0, 637, 6, 0, 3948, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 481, 0, 0, 0, 2242, 0, 16, 568, 0, 0, 0, 0, 0, 0, 0, 0, 6, 13, 0, 2, 0, 1, 604, 1701, 1254, 789, 291, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 8, 0, 2259, 1218, 0, 0, 0, 0, 0, 0, 955, 0, 3, 1624, 3624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2042, 0, 6, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4005, 4264, 4642, 3307, 4661, 3488, 6206, 2195] avg=4096.0000 max=6206 max_rank=6 min=2195 min_rank=7 imbalance_ratio=1.515137 max_load_reduction=26562 max_load_reduction_pct=0.810608 moved_to_echo=16550 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 887, 35, 0, 3933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 235, 0, 0, 0, 0, 3145, 864, 0, 0, 0, 0, 0, 0, 592, 0, 0, 2515, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 59, 1390, 1, 3378, 104, 0, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 6, 113, 1219, 1, 439, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2320, 1184, 0, 0, 0, 0, 0, 0, 4053, 0, 31, 1580, 36, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3551, 545, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4855, 4247, 3124, 5410, 1803, 3505, 5721, 4103] avg=4096.0000 max=5721 max_rank=6 min=1803 min_rank=4 imbalance_ratio=1.396729 max_load_reduction=27047 max_load_reduction_pct=0.825409 moved_to_echo=15132 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 886, 0, 0, 3866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 3447, 863, 0, 0, 0, 0, 0, 0, 649, 9, 0, 4011, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 10, 67, 0, 9, 3, 0, 3096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 60, 0, 390, 0, 0, 50, 1153, 314, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3873, 2317, 1120, 0, 0, 0, 0, 0, 0, 34, 0, 0, 1493, 3499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 417, 0, 815, 0, 0, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4752, 4313, 4700, 3250, 2080, 7311, 5026, 1336] avg=4096.0000 max=7311 max_rank=5 min=1336 min_rank=7 imbalance_ratio=1.784912 max_load_reduction=25457 max_load_reduction_pct=0.776886 moved_to_echo=15753 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 888, 1, 1, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 3, 81, 0, 0, 3435, 866, 0, 0, 0, 0, 0, 0, 647, 4, 0, 3926, 1, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 7, 1008, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 273, 14, 1248, 945, 131, 0, 0, 0, 0, 0, 5, 0, 0, 0, 46, 1, 0, 236, 2324, 1212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1615, 0, 0, 246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2891, 0, 2443, 28, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4951, 4395, 4586, 5112, 2619, 3824, 1861, 5420] avg=4096.0000 max=5420 max_rank=7 min=1861 min_rank=6 imbalance_ratio=1.323242 max_load_reduction=27348 max_load_reduction_pct=0.834595 moved_to_echo=13724 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 12, 0, 889, 0, 105, 3014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2419, 867, 0, 0, 0, 0, 0, 0, 456, 0, 0, 0, 794, 23, 9, 482, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 124, 0, 3554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 4028, 662, 835, 462, 0, 0, 0, 0, 0, 0, 79, 0, 0, 0, 0, 23, 0, 2328, 643, 0, 0, 0, 0, 0, 0, 882, 0, 0, 858, 3787, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3219, 0, 2159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4020, 3286, 1764, 3680, 6035, 3073, 5532, 5378] avg=4096.0000 max=6035 max_rank=4 min=1764 min_rank=2 imbalance_ratio=1.473389 max_load_reduction=26733 max_load_reduction_pct=0.815826 moved_to_echo=20018 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 880, 0, 0, 403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2847, 857, 0, 0, 0, 0, 0, 0, 536, 0, 0, 3943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 451, 0, 918, 0, 0, 999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 0, 89, 8, 22, 228, 4094, 1056, 1112, 119, 0, 0, 0, 0, 0, 0, 0, 0, 609, 0, 0, 0, 0, 2303, 1025, 0, 0, 0, 0, 0, 0, 1752, 0, 3, 1368, 1114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 0, 2876, 6, 0, 1541, 5, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1283, 3708, 4479, 2368, 7264, 3937, 4237, 5492] avg=4096.0000 max=7264 max_rank=4 min=1283 min_rank=0 imbalance_ratio=1.773438 max_load_reduction=25504 max_load_reduction_pct=0.778320 moved_to_echo=20930 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 879, 0, 0, 3541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3417, 869, 0, 0, 0, 0, 0, 0, 640, 2, 0, 4094, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2136, 0, 3609, 11, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3291, 0, 0, 98, 1256, 108, 475, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 522, 2327, 1220, 0, 0, 0, 0, 0, 0, 34, 0, 0, 1614, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2251, 0, 265, 0, 0, 23, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4420, 4288, 4740, 5759, 5229, 4072, 1720, 2540] avg=4096.0000 max=5759 max_rank=3 min=1720 min_rank=6 imbalance_ratio=1.406006 max_load_reduction=27009 max_load_reduction_pct=0.824249 moved_to_echo=13561 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 768, 128, 128, 3840, 128, 1152, 10496, 9216, 4992, 2432, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 640, 128, 128, 128, 4736, 18560, 8960, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8576, 0, 128, 11776, 12160, 0, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 17536, 640, 12800, 128, 128, 1920, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 0, 7168, 128, 128, 25600, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 256, 128, 128, 128, 128, 25600, 6912, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[640, 128, 4224, 128, 18432, 128, 128, 9600, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4864, 128, 0, 26624, 896, 128, 128, 640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 122, 93, 0, 0, 483, 0, 0, 0, 0, 0, 0, 0, 4096, 10, 4095, 0, 0, 0, 0, 0, 0, 4088, 2, 0, 881, 0, 0, 0, 0, 15, 0, 0, 0, 0, 1925, 3512, 0, 0, 325, 988, 16, 10, 1, 0, 4045, 0, 0, 42, 0, 7, 0, 0, 0, 4089, 0, 67, 0, 0, 0, 0, 0, 0, 3856] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[16, 1, 0, 3, 110, 4053, 4, 0, 2, 0, 42, 35, 0, 4045, 1, 4051, 0, 0, 467, 0, 0, 0, 2448, 0, 0, 3, 0, 0, 0, 115, 0, 0, 0, 0, 1, 49, 2260, 0, 1, 14, 0, 0, 179, 0, 3286, 3806, 1, 0, 0, 0, 3656, 0, 0, 2, 4096, 0, 0, 0, 1, 0, 0, 1, 1, 18] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[4, 0, 0, 0, 0, 3029, 0, 0, 0, 0, 0, 72, 6, 4020, 1, 4015, 0, 0, 0, 0, 0, 3, 1861, 5, 0, 0, 7, 17, 0, 0, 1, 0, 0, 24, 0, 229, 24, 0, 0, 3975, 0, 0, 1781, 0, 0, 4080, 7, 0, 0, 0, 65, 0, 0, 8, 4096, 0, 1644, 0, 0, 1, 0, 0, 0, 3793] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[52, 3324, 0, 0, 0, 2316, 0, 0, 0, 22, 0, 0, 0, 4091, 0, 4074, 0, 0, 0, 0, 0, 0, 4032, 0, 0, 1343, 474, 0, 0, 264, 0, 0, 1, 0, 0, 6, 0, 0, 0, 3476, 859, 0, 173, 0, 50, 4085, 1, 0, 0, 0, 0, 0, 2, 12, 4017, 0, 33, 0, 0, 0, 0, 43, 0, 18] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 424, 5, 0, 3966, 154, 0, 2, 0, 1, 0, 123, 3952, 0, 3826, 0, 12, 7, 0, 98, 1, 3964, 1, 0, 3574, 0, 0, 0, 32, 166, 0, 0, 0, 0, 1821, 4029, 0, 0, 0, 0, 163, 0, 0, 3283, 2730, 0, 0, 3, 0, 0, 0, 19, 0, 2, 0, 339, 2, 0, 0, 0, 56, 0, 13] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 4, 1, 0, 6, 148, 208, 0, 0, 2, 0, 0, 8, 3881, 97, 3846, 0, 0, 0, 0, 0, 0, 4093, 0, 2, 2460, 4, 0, 0, 0, 3, 0, 0, 0, 35, 2971, 213, 1, 2, 1106, 0, 0, 358, 4, 0, 3898, 1, 0, 17, 0, 1060, 31, 0, 228, 4079, 0, 0, 0, 0, 0, 61, 0, 0, 3940] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[554, 100, 0, 6, 0, 11, 0, 0, 0, 0, 13, 0, 182, 3522, 0, 4029, 0, 0, 0, 22, 0, 0, 1708, 828, 0, 3843, 0, 0, 0, 1, 1404, 0, 0, 0, 78, 2465, 769, 0, 0, 4084, 29, 0, 3, 0, 3, 3867, 27, 0, 21, 6, 1223, 0, 0, 12, 3906, 0, 0, 34, 0, 0, 0, 1, 17, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 0, 22, 0, 0, 2, 4096, 3, 4069, 0, 0, 0, 0, 1, 2, 4062, 10, 0, 433, 0, 0, 0, 0, 2782, 1, 0, 0, 0, 0, 3111, 0, 2, 1956, 347, 0, 1, 0, 11, 3274, 5, 0, 0, 0, 436, 0, 0, 0, 4021, 0, 16, 0, 4, 0, 0, 5, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 122, 93, 0, 0, 483, 0, 0, 1168, 65, 247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 10, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 2, 57, 344, 0, 0, 0, 0, 0, 0, 0, 881, 0, 0, 0, 0, 15, 0, 1911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1925, 3512, 0, 0, 203, 0, 0, 0, 0, 0, 0, 0, 0, 988, 16, 10, 1, 0, 2877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 7, 0, 0, 0, 3745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 0, 0, 0, 3856, 1881, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2178, 4162, 4491, 2807, 5640, 3892, 3794, 5804] avg=4096.0000 max=5804 max_rank=7 min=2178 min_rank=0 imbalance_ratio=1.416992 max_load_reduction=26964 max_load_reduction_pct=0.822876 moved_to_echo=19130 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[16, 1, 0, 3, 110, 4053, 4, 0, 1099, 2, 244, 0, 0, 0, 0, 0, 2, 0, 42, 35, 0, 4045, 1, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 0, 0, 0, 2448, 0, 2, 345, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 115, 0, 0, 1891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 49, 2260, 0, 1, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 0, 3286, 2707, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3656, 0, 0, 2, 3751, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 18, 1861, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5532, 4180, 3262, 2009, 2321, 6173, 7409, 1882] avg=4096.0000 max=7409 max_rank=6 min=1882 min_rank=7 imbalance_ratio=1.808838 max_load_reduction=25359 max_load_reduction_pct=0.773895 moved_to_echo=17785 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 0, 0, 0, 3029, 0, 0, 1178, 806, 242, 0, 0, 0, 0, 0, 0, 0, 0, 72, 6, 4020, 1, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1861, 5, 709, 345, 0, 0, 0, 0, 0, 0, 0, 0, 7, 17, 0, 0, 1, 0, 1874, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 229, 24, 0, 0, 2460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1781, 0, 0, 2902, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 8, 3751, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1644, 0, 0, 1, 0, 0, 0, 3793, 1845, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5259, 4153, 2923, 1899, 2737, 4690, 3824, 7283] avg=4096.0000 max=7283 max_rank=7 min=1899 min_rank=3 imbalance_ratio=1.778076 max_load_reduction=25485 max_load_reduction_pct=0.777740 moved_to_echo=18534 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[52, 3324, 0, 0, 0, 2316, 0, 0, 1180, 705, 246, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 4091, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 0, 620, 338, 0, 0, 0, 0, 0, 0, 0, 1343, 474, 0, 0, 264, 0, 0, 1902, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6, 0, 0, 0, 2151, 0, 0, 0, 0, 0, 0, 0, 0, 859, 0, 173, 0, 50, 2905, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 12, 3679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 43, 0, 18, 1872, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7823, 4167, 4990, 3983, 2158, 3988, 3693, 1966] avg=4096.0000 max=7823 max_rank=0 min=1966 min_rank=7 imbalance_ratio=1.909912 max_load_reduction=24945 max_load_reduction_pct=0.761261 moved_to_echo=11805 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[554, 100, 0, 6, 0, 11, 0, 0, 1117, 829, 243, 0, 0, 0, 0, 0, 0, 0, 13, 0, 182, 3522, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 1708, 828, 728, 329, 0, 0, 0, 0, 0, 0, 0, 3843, 0, 0, 0, 1, 1404, 0, 1881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 2465, 769, 0, 0, 2527, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 3, 0, 3, 2750, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 6, 1223, 0, 0, 12, 3577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 1, 17, 0, 1851, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2860, 3771, 3615, 7129, 5839, 2812, 4839, 1903] avg=4096.0000 max=7129 max_rank=3 min=1903 min_rank=7 imbalance_ratio=1.740479 max_load_reduction=25639 max_load_reduction_pct=0.782440 moved_to_echo=15393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 424, 5, 0, 3966, 154, 0, 788, 0, 231, 0, 0, 0, 0, 0, 2, 0, 1, 0, 123, 3952, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 7, 0, 98, 1, 3964, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3574, 0, 0, 0, 32, 166, 0, 1786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1821, 4029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 163, 0, 0, 3283, 1942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 19, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 339, 2, 0, 0, 0, 56, 0, 13, 1758, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5568, 4129, 4083, 5558, 5850, 5388, 24, 2168] avg=4096.0000 max=5850 max_rank=4 min=24 min_rank=6 imbalance_ratio=1.428223 max_load_reduction=26918 max_load_reduction_pct=0.821472 moved_to_echo=13430 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 1, 0, 6, 148, 208, 0, 1126, 224, 232, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8, 3881, 97, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 197, 344, 0, 0, 0, 0, 0, 0, 2, 2460, 4, 0, 0, 0, 3, 0, 1795, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 2971, 213, 1, 2, 685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 358, 4, 0, 2772, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 1060, 31, 0, 228, 3735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 3940, 1767, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1949, 4040, 4634, 4264, 3907, 3135, 5071, 5768] avg=4096.0000 max=5768 max_rank=7 min=1949 min_rank=0 imbalance_ratio=1.408203 max_load_reduction=27000 max_load_reduction_pct=0.823975 moved_to_echo=17881 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 948, 401, 249, 0, 0, 0, 0, 0, 0, 22, 0, 0, 2, 4096, 3, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 4062, 10, 352, 343, 0, 0, 0, 0, 0, 0, 0, 433, 0, 0, 0, 0, 2782, 1, 1903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3111, 0, 2, 1203, 0, 0, 0, 0, 0, 0, 0, 0, 347, 0, 1, 0, 11, 2326, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 436, 0, 0, 0, 3678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 4, 0, 0, 5, 0, 4095, 1874, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1599, 4166, 4770, 5119, 4316, 2690, 4114, 5994] avg=4096.0000 max=5994 max_rank=7 min=1599 min_rank=0 imbalance_ratio=1.463379 max_load_reduction=26774 max_load_reduction_pct=0.817078 moved_to_echo=17114 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 12544, 512, 128, 0, 512, 4480, 128, 14976, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[640, 3584, 640, 128, 128, 14080, 384, 0, 8704, 3072, 2048, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 9472, 13952, 128, 128, 9344, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[2176, 128, 128, 128, 128, 128, 128, 15744, 14720, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[2304, 256, 2560, 128, 6656, 21248, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 6528, 128, 128, 384, 25984, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 512, 128, 128, 128, 26368, 896, 2688, 2432, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 384, 31744, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 4089, 0, 1, 983, 2, 1696, 0, 0, 4049, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 2466, 15, 6, 0, 0, 0, 0, 0, 0, 0, 0, 3416, 48, 1, 0, 721, 8, 0, 0, 0, 4093, 2, 936, 5, 28, 2436, 0, 32, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 4002, 1198, 1, 2435] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 2406, 0, 0, 216, 0, 465, 0, 0, 4069, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 2266, 0, 28, 4096, 0, 1, 0, 181, 0, 1, 126, 0, 3125, 3, 0, 0, 2, 16, 2, 0, 0, 0, 0, 0, 0, 26, 0, 1, 0, 0, 3960, 3898, 0, 3775] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 4009, 185, 0, 1666, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 38, 3357, 0, 0, 0, 0, 0, 0, 2710, 0, 0, 1721, 4076, 0, 0, 0, 98, 0, 0, 806, 16, 897, 20, 10, 1568, 0, 0, 0, 194, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 2430, 133, 592] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 1310, 0, 2, 0, 0, 407, 0, 0, 4072, 0, 0, 0, 0, 2, 0, 0, 16, 0, 0, 0, 0, 3553, 0, 0, 0, 0, 5, 1, 0, 911, 0, 13, 3933, 1, 0, 6, 0, 0, 1, 0, 0, 3502, 0, 703, 0, 0, 194, 0, 0, 1072, 0, 0, 0, 0, 1495, 0, 0, 20, 1, 4095, 3450, 0, 4003] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 4070, 884, 0, 2188, 0, 976, 1, 2, 3963, 0, 0, 0, 0, 0, 0, 0, 29, 8, 0, 0, 1, 1736, 3, 0, 26, 0, 0, 1, 0, 2054, 6, 107, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 3379, 0, 0, 1, 0, 0, 0, 0, 2, 22, 0, 76, 0, 0, 0, 0, 0, 25, 3609, 1451, 0, 4079] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 4095, 27, 0, 3188, 0, 1337, 475, 0, 3521, 0, 6, 0, 0, 0, 0, 0, 26, 1, 40, 0, 0, 1680, 0, 1, 0, 2, 0, 3, 369, 0, 0, 0, 3339, 0, 0, 0, 388, 1424, 0, 0, 0, 142, 1, 1925, 0, 0, 0, 0, 0, 85, 0, 0, 41, 0, 1003, 0, 1, 0, 1, 4079, 4090, 0, 1478] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 3997, 0, 53, 0, 0, 9, 0, 0, 4094, 0, 2, 0, 0, 0, 0, 0, 0, 457, 0, 0, 0, 4096, 0, 52, 0, 0, 0, 90, 59, 0, 0, 69, 3942, 0, 0, 0, 4, 11, 0, 0, 0, 4086, 0, 1460, 0, 0, 0, 98, 0, 190, 0, 0, 0, 0, 101, 0, 0, 21, 0, 3687, 2428, 41, 3721] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 4089, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 1, 0, 2, 4094, 0, 25, 0, 15, 5, 0, 1, 0, 3788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4030, 6, 0, 0, 4, 4068, 4074, 6, 366] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1545, 0, 0, 216, 0, 465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 0, 35, 0, 68, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 363, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2266, 0, 3037, 0, 0, 0, 0, 0, 0, 0, 28, 3690, 0, 1, 0, 181, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 126, 0, 3057, 3, 0, 0, 2, 16, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 26, 1856, 923, 498, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2042, 0, 3775, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2226, 4203, 4808, 5303, 3901, 3204, 3305, 5818] avg=4096.0000 max=5818 max_rank=7 min=2226 min_rank=0 imbalance_ratio=1.420410 max_load_reduction=26950 max_load_reduction_pct=0.822449 moved_to_echo=16228 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 2613, 884, 0, 2188, 0, 976, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3963, 0, 0, 0, 0, 0, 0, 73, 31, 0, 0, 0, 0, 0, 0, 0, 29, 8, 0, 0, 1, 1736, 3, 614, 373, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 1, 0, 2054, 6, 2767, 0, 0, 0, 0, 0, 0, 0, 107, 3665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3306, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 22, 0, 76, 0, 0, 690, 842, 843, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 761, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6662, 4069, 2764, 4854, 3772, 3307, 2475, 4865] avg=4096.0000 max=6662 max_rank=0 min=2475 min_rank=6 imbalance_ratio=1.626465 max_load_reduction=26106 max_load_reduction_pct=0.796692 moved_to_echo=14419 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 2625, 0, 1, 983, 2, 1696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 0, 0, 0, 0, 28, 89, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2466, 15, 617, 313, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 3070, 0, 0, 0, 0, 0, 0, 0, 0, 3077, 48, 1, 0, 721, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4004, 2, 936, 5, 28, 2436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 71, 0, 0, 0, 0, 570, 932, 847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 628, 1, 2435, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5307, 4192, 3411, 3076, 3855, 7411, 2452, 3064] avg=4096.0000 max=7411 max_rank=5 min=2452 min_rank=6 imbalance_ratio=1.809326 max_load_reduction=25357 max_load_reduction_pct=0.773834 moved_to_echo=16782 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 2573, 185, 0, 1666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 16, 19, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 3357, 0, 605, 373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2710, 0, 0, 3114, 0, 0, 0, 0, 0, 0, 0, 1721, 3672, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 806, 16, 878, 20, 10, 1568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 194, 92, 0, 0, 0, 0, 0, 1157, 945, 831, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1273, 133, 592, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4424, 4141, 4373, 5824, 5491, 3298, 3219, 1998] avg=4096.0000 max=5824 max_rank=3 min=1998 min_rank=7 imbalance_ratio=1.421875 max_load_reduction=26944 max_load_reduction_pct=0.822266 moved_to_echo=14006 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 842, 0, 2, 0, 0, 407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 2, 0, 76, 29, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 3553, 0, 197, 360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 911, 0, 3140, 0, 0, 0, 0, 0, 0, 0, 13, 3544, 1, 0, 6, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3426, 0, 703, 0, 0, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1072, 0, 0, 0, 0, 1495, 1642, 955, 271, 0, 0, 0, 0, 0, 0, 0, 20, 1, 0, 1808, 0, 4003, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1251, 4179, 4126, 4057, 3565, 4323, 5435, 5832] avg=4096.0000 max=5832 max_rank=7 min=1251 min_rank=0 imbalance_ratio=1.423828 max_load_reduction=26936 max_load_reduction_pct=0.822021 moved_to_echo=19155 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 2629, 27, 0, 3188, 0, 1337, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3521, 0, 6, 0, 0, 0, 0, 3, 25, 0, 0, 0, 0, 0, 0, 0, 26, 1, 40, 0, 0, 1680, 0, 618, 306, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 3, 369, 0, 0, 3128, 0, 0, 0, 0, 0, 0, 0, 0, 3008, 0, 0, 0, 388, 1424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 1, 1925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 41, 0, 1003, 1947, 951, 848, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 2143, 0, 1478, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7656, 3555, 2671, 3503, 4820, 2065, 4875, 3623] avg=4096.0000 max=7656 max_rank=0 min=2065 min_rank=5 imbalance_ratio=1.869141 max_load_reduction=25112 max_load_reduction_pct=0.766357 moved_to_echo=15383 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 2566, 0, 53, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2, 0, 0, 0, 0, 89, 30, 0, 0, 0, 0, 0, 0, 0, 0, 457, 0, 0, 0, 4096, 0, 603, 361, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 90, 59, 0, 0, 2828, 0, 0, 0, 0, 0, 0, 0, 69, 3551, 0, 0, 0, 4, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3997, 0, 1460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 190, 0, 0, 0, 0, 101, 1156, 859, 828, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 1272, 41, 3721, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2628, 4215, 5517, 3029, 3635, 5457, 3232, 5055] avg=4096.0000 max=5517 max_rank=2 min=2628 min_rank=0 imbalance_ratio=1.346924 max_load_reduction=27251 max_load_reduction_pct=0.831635 moved_to_echo=17379 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 2614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 1, 0, 86, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 623, 379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 3120, 0, 0, 0, 0, 0, 0, 0, 2, 3682, 0, 25, 0, 15, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4030, 1943, 948, 852, 0, 0, 0, 0, 0, 6, 0, 0, 4, 0, 2131, 6, 366, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2614, 4214, 5098, 3122, 3729, 3703, 7775, 2513] avg=4096.0000 max=7775 max_rank=6 min=2513 min_rank=7 imbalance_ratio=1.898193 max_load_reduction=24993 max_load_reduction_pct=0.762726 moved_to_echo=17720 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 18048, 1152, 128, 8320, 128, 4992, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 0, 12160, 256, 20480, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 256, 1536, 128, 0, 128, 0, 6656, 11008, 7424, 5888, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1024, 128, 22528, 128, 5120, 1664, 128, 2688, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 3200, 5248, 128, 24320, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 512, 128, 0, 128, 25088, 128, 4352, 2944, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[2048, 27904, 128, 128, 128, 1408, 1536, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32000, 0, 128, 0, 0, 128, 128, 512, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1453, 3710, 36, 0, 4056, 0, 45, 9, 0, 4013, 0, 0, 0, 1, 0, 1, 0, 1487, 0, 0, 0, 26, 147, 135, 0, 0, 0, 0, 0, 0, 3930, 2, 3238, 0, 711, 0, 0, 0, 0, 0, 0, 0, 4068, 326, 0, 0, 0, 0, 0, 31, 2335, 32, 0, 0, 64, 0, 239, 1230, 0, 140, 0, 3, 0, 1300] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 1, 129, 0, 4083, 0, 0, 0, 0, 4093, 0, 0, 2, 2892, 0, 0, 764, 2721, 0, 0, 0, 3687, 1366, 0, 0, 0, 23, 0, 0, 0, 1, 1, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 3688, 0, 0, 0, 0, 0, 0, 26, 0, 27, 0, 0, 0, 3, 0, 0, 164, 1, 0, 0, 904] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 88, 4090, 202, 45, 1, 1, 4094, 0, 0, 5, 4048, 0, 1, 0, 0, 0, 1, 1, 1003, 4013, 0, 16, 0, 7, 55, 0, 0, 3525, 0, 4007, 0, 0, 0, 0, 0, 0, 0, 5, 2, 4035, 0, 0, 0, 0, 0, 0, 405, 20, 0, 0, 0, 74, 0, 1, 337, 0, 198, 0, 0, 0, 2488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 7, 0, 20, 3240, 0, 322, 0, 0, 4009, 0, 2, 1, 3659, 0, 0, 20, 0, 1, 4, 0, 2, 1847, 0, 0, 3, 0, 0, 0, 5, 1253, 1, 3713, 0, 51, 0, 0, 0, 0, 21, 3142, 0, 4059, 3869, 73, 0, 0, 0, 0, 10, 1044, 14, 0, 0, 0, 937, 0, 0, 0, 783, 0, 0, 0, 656] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 2, 4094, 1, 4090, 0, 0, 0, 0, 3929, 0, 0, 0, 3760, 0, 1, 0, 0, 0, 0, 29, 3953, 1348, 0, 0, 0, 0, 0, 186, 0, 0, 0, 4082, 2, 1, 0, 2, 0, 6, 0, 0, 0, 4063, 0, 0, 0, 0, 4, 16, 102, 0, 0, 0, 0, 0, 5, 0, 2402, 0, 14, 0, 0, 4, 672] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 1, 279, 0, 4092, 9, 0, 0, 130, 2297, 53, 0, 19, 0, 0, 3, 2677, 0, 0, 34, 11, 4083, 242, 0, 0, 0, 0, 0, 1, 0, 3998, 0, 4070, 0, 0, 0, 0, 7, 0, 8, 0, 0, 3366, 367, 0, 0, 0, 68, 672, 0, 0, 0, 0, 0, 0, 1778, 0, 116, 0, 4024, 0, 0, 0, 363] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[252, 5, 4068, 0, 3749, 0, 0, 119, 0, 3409, 0, 0, 0, 1, 9, 0, 12, 2, 0, 203, 0, 3532, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4073, 0, 88, 0, 0, 2, 0, 0, 0, 0, 3938, 0, 5, 0, 0, 0, 0, 714, 4030, 305, 0, 0, 87, 179, 0, 3954, 0, 25, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 3181, 11, 4089, 13, 0, 0, 0, 4072, 0, 0, 0, 4081, 0, 0, 89, 0, 0, 0, 0, 1955, 2605, 0, 0, 0, 1090, 0, 4, 0, 10, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3369, 13, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 129, 0, 1902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1930, 0, 0, 2, 2892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 764, 2721, 0, 0, 0, 2630, 1366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 1, 1, 1748, 757, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2741, 3688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 27, 0, 0, 0, 2181, 415, 0, 0, 0, 0, 0, 0, 3, 0, 0, 164, 1, 0, 0, 904, 956, 598, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2032, 4824, 7481, 2530, 4197, 6429, 2649, 2626] avg=4096.0000 max=7481 max_rank=2 min=2032 min_rank=0 imbalance_ratio=1.826416 max_load_reduction=25287 max_load_reduction_pct=0.771698 moved_to_echo=15901 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1453, 3710, 36, 0, 1889, 0, 45, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1892, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1487, 0, 0, 0, 20, 147, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3930, 2, 1714, 752, 0, 0, 0, 0, 0, 0, 3238, 0, 711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2722, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 2335, 32, 0, 0, 64, 0, 2167, 407, 0, 0, 0, 0, 0, 0, 239, 1230, 0, 140, 0, 3, 0, 1300, 6, 594, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7142, 1894, 1789, 6398, 3949, 3048, 5036, 3512] avg=4096.0000 max=7142 max_rank=0 min=1789 min_rank=2 imbalance_ratio=1.743652 max_load_reduction=25626 max_load_reduction_pct=0.782043 moved_to_echo=15545 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4094, 1, 1905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1853, 0, 0, 0, 3760, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 2820, 1348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 0, 0, 0, 1678, 751, 0, 0, 0, 0, 0, 0, 4082, 2, 1, 0, 2, 0, 6, 0, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2719, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 16, 102, 0, 0, 0, 0, 0, 5, 2185, 398, 0, 0, 0, 0, 0, 0, 0, 2402, 0, 14, 0, 0, 4, 672, 1025, 593, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6002, 5614, 4197, 2615, 4201, 2723, 2706, 4710] avg=4096.0000 max=6002 max_rank=0 min=2615 min_rank=3 imbalance_ratio=1.465332 max_load_reduction=26766 max_load_reduction_pct=0.816833 moved_to_echo=14340 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 0, 20, 1509, 0, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1891, 0, 2, 1, 3659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 1, 4, 0, 2, 1847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 5, 1253, 1, 1712, 750, 0, 0, 0, 0, 0, 0, 3713, 0, 51, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 3142, 0, 2716, 3869, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1044, 14, 0, 0, 0, 937, 1731, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 783, 0, 0, 0, 656, 0, 593, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1858, 5553, 1874, 3724, 3785, 9800, 4142, 2032] avg=4096.0000 max=9800 max_rank=5 min=1858 min_rank=0 imbalance_ratio=2.392578 max_load_reduction=22968 max_load_reduction_pct=0.700928 moved_to_echo=19759 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 88, 1905, 202, 45, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1930, 0, 0, 5, 4048, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 716, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 7, 55, 0, 0, 3525, 0, 1749, 746, 0, 0, 0, 0, 0, 0, 4007, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 5, 2, 2700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 405, 20, 0, 0, 0, 74, 0, 2185, 415, 0, 0, 0, 0, 0, 0, 1, 337, 0, 198, 0, 0, 0, 2488, 260, 589, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2241, 5985, 4731, 6098, 4034, 2707, 3099, 3873] avg=4096.0000 max=6098 max_rank=3 min=2241 min_rank=0 imbalance_ratio=1.488770 max_load_reduction=26670 max_load_reduction_pct=0.813904 moved_to_echo=13713 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 279, 0, 1906, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 1083, 53, 0, 19, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2677, 0, 0, 34, 11, 2913, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3998, 0, 981, 622, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 7, 0, 8, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2253, 367, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 672, 0, 0, 0, 0, 0, 0, 1778, 2186, 233, 0, 0, 0, 0, 0, 0, 0, 116, 0, 4024, 0, 0, 0, 363, 1058, 491, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2195, 1288, 5877, 5602, 4197, 2688, 4869, 6052] avg=4096.0000 max=6052 max_rank=7 min=1288 min_rank=1 imbalance_ratio=1.477539 max_load_reduction=26716 max_load_reduction_pct=0.815308 moved_to_echo=17806 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[252, 5, 4068, 0, 1746, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1607, 0, 0, 0, 1, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 2, 0, 203, 0, 2520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 1456, 728, 0, 0, 0, 0, 0, 0, 4073, 0, 88, 0, 0, 2, 0, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2635, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 714, 4030, 305, 0, 0, 87, 179, 2003, 346, 0, 0, 0, 0, 0, 0, 0, 3954, 0, 25, 0, 0, 0, 0, 915, 575, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6190, 1617, 2737, 2191, 4260, 2640, 7664, 5469] avg=4096.0000 max=7664 max_rank=6 min=1617 min_rank=1 imbalance_ratio=1.871094 max_load_reduction=25104 max_load_reduction_pct=0.766113 moved_to_echo=20033 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3181, 11, 1903, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1912, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 1388, 2605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1090, 0, 4, 0, 10, 0, 1743, 763, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2186, 417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3369, 13, 0, 0, 0, 510, 602, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5108, 5993, 4082, 3610, 4145, 2733, 2603, 4494] avg=4096.0000 max=5993 max_rank=1 min=2603 min_rank=6 imbalance_ratio=1.463135 max_load_reduction=26775 max_load_reduction_pct=0.817108 moved_to_echo=13975 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[768, 1280, 7552, 384, 128, 0, 256, 2944, 16896, 3072, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 1152, 128, 256, 128, 12800, 128, 12800, 5888, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[256, 14208, 128, 128, 128, 18560, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1792, 3840, 11904, 128, 14720, 256, 512, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3200, 128, 21248, 8320, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[256, 8064, 0, 8832, 128, 128, 128, 6400, 4736, 4736, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3584, 4224, 128, 256, 128, 13056, 11648, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[31488, 128, 896, 128, 128, 128, 128, 128, 512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 4061, 3756, 0, 101, 821, 4043, 0, 3956, 0, 4096, 0, 0, 4025, 0, 0, 0, 0, 0, 0, 8, 29, 0, 259, 0, 135, 0, 0, 0, 0, 0, 0, 1, 0, 13, 1, 0, 0, 0, 0, 0, 0, 3307, 0, 3822, 19, 306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[328, 0, 1, 4080, 3901, 0, 143, 0, 4094, 4069, 0, 0, 3730, 0, 0, 3939, 0, 0, 0, 0, 0, 330, 0, 0, 26, 0, 0, 151, 0, 0, 11, 0, 3, 0, 0, 3610, 0, 0, 40, 0, 0, 2, 0, 0, 4048, 0, 115, 0, 0, 14, 5, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 6, 0, 17] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 29, 0, 4092, 4050, 0, 0, 0, 4096, 5, 137, 0, 4013, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 4066, 0, 0, 442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3642, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[9, 131, 172, 4046, 3974, 0, 38, 280, 3398, 706, 6, 481, 4052, 0, 0, 3173, 0, 0, 0, 0, 0, 0, 0, 0, 3367, 94, 26, 118, 0, 219, 1, 1, 0, 0, 0, 2008, 0, 23, 7, 0, 0, 0, 45, 0, 3059, 619, 1324, 0, 1303, 0, 0, 0, 0, 19, 34, 32, 1, 0, 0, 0, 2, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 33, 0, 4044, 3644, 0, 0, 19, 71, 503, 0, 0, 4042, 3426, 4053, 4096, 0, 31, 0, 0, 0, 0, 0, 1, 1087, 0, 18, 2, 0, 0, 0, 0, 0, 47, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 3952, 0, 3477, 138, 0, 1, 0, 0, 0, 0, 0, 4, 0, 11, 0, 7, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4055, 0, 0, 0, 3729, 0, 2984, 46, 309, 357, 807, 4084, 0, 0, 1, 0, 2, 0, 0, 0, 765, 0, 8, 0, 2, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4065, 0, 3327, 24, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4074, 0, 336, 0, 4084, 1007, 0, 0, 3501, 0, 156, 4082, 0, 0, 0, 1, 34, 0, 11, 461, 4, 0, 0, 0, 0, 2410, 0, 0, 180, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4086, 21, 3995, 10, 0, 4, 211, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 4, 0, 4085, 4082, 0, 0, 0, 4095, 0, 993, 3, 4017, 4019, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 24, 0, 0, 0, 0, 0, 118, 1, 0, 76, 0, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 3052, 0, 3846, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 29, 0, 0, 3895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 751, 5, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 30, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4066, 0, 0, 1816, 155, 0, 0, 0, 0, 0, 0, 442, 0, 0, 0, 0, 0, 0, 0, 2875, 842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 984, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3983, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4088, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3924, 893, 4132, 6038, 4159, 5080, 4453, 4089] avg=4096.0000 max=6038 max_rank=3 min=893 min_rank=1 imbalance_ratio=1.474121 max_load_reduction=26730 max_load_reduction_pct=0.815735 moved_to_echo=17781 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[9, 131, 172, 0, 3822, 0, 38, 280, 0, 0, 0, 0, 0, 0, 0, 0, 623, 706, 6, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3173, 30, 4, 0, 0, 0, 0, 0, 3367, 94, 26, 118, 0, 219, 1, 1, 1525, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2008, 0, 23, 7, 0, 2385, 708, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 826, 619, 1324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1303, 0, 0, 0, 0, 19, 34, 32, 4022, 390, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 4042, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4452, 1816, 3207, 5503, 5131, 2814, 5800, 4045] avg=4096.0000 max=5800 max_rank=6 min=1816 min_rank=1 imbalance_ratio=1.416016 max_load_reduction=26968 max_load_reduction_pct=0.822998 moved_to_echo=17790 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3612, 0, 101, 821, 0, 0, 0, 0, 0, 0, 0, 0, 741, 0, 3956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 29, 4025, 31, 4, 0, 0, 0, 0, 0, 0, 259, 0, 135, 0, 0, 0, 0, 1649, 144, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 13, 1, 0, 0, 2838, 765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 893, 0, 3822, 19, 0, 0, 0, 0, 0, 0, 0, 0, 306, 0, 0, 0, 0, 0, 0, 0, 4065, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 4, 4057, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4534, 4697, 4097, 2187, 3618, 4734, 4835, 4066] avg=4096.0000 max=4835 max_rank=6 min=2187 min_rank=3 imbalance_ratio=1.180420 max_load_reduction=27933 max_load_reduction_pct=0.852448 moved_to_echo=17253 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3918, 0, 336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 748, 1007, 0, 0, 0, 0, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 34, 0, 11, 461, 4082, 26, 4, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 2410, 0, 0, 2037, 156, 0, 0, 0, 0, 0, 0, 180, 0, 1, 0, 0, 0, 1, 0, 2867, 945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1104, 21, 3995, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 211, 1, 0, 0, 0, 0, 3475, 469, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4254, 1911, 4619, 4607, 3994, 5130, 4160, 4093] avg=4096.0000 max=5130 max_rank=5 min=1911 min_rank=1 imbalance_ratio=1.252441 max_load_reduction=27638 max_load_reduction_pct=0.843445 moved_to_echo=17377 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[328, 0, 1, 0, 3751, 0, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 750, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 330, 0, 0, 3939, 28, 4, 0, 0, 0, 0, 0, 26, 0, 0, 151, 0, 0, 11, 0, 2018, 150, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3610, 0, 0, 40, 0, 2874, 936, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1094, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 5, 0, 0, 0, 105, 0, 3702, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 17, 4076, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4223, 4819, 4301, 2356, 7463, 1211, 4296, 4099] avg=4096.0000 max=7463 max_rank=4 min=1211 min_rank=5 imbalance_ratio=1.822021 max_load_reduction=25305 max_load_reduction_pct=0.772247 moved_to_echo=17069 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 684, 0, 2984, 46, 0, 357, 807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 4084, 2, 4, 0, 0, 0, 0, 0, 765, 0, 8, 0, 2, 4096, 0, 1, 2027, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2617, 940, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 1098, 0, 3327, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 307, 428, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3900, 4878, 4093, 7054, 3557, 4455, 735, 4096] avg=4096.0000 max=7054 max_rank=3 min=735 min_rank=6 imbalance_ratio=1.722168 max_load_reduction=25714 max_load_reduction_pct=0.784729 moved_to_echo=12843 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 33, 0, 0, 3504, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 14, 503, 0, 0, 0, 3426, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 1, 4096, 30, 4, 0, 0, 0, 0, 0, 1087, 0, 18, 2, 0, 0, 0, 0, 1970, 140, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 61, 0, 0, 0, 49, 914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 0, 3477, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 4012, 8, 0, 0, 0, 0, 0, 0, 0, 11, 0, 7, 0, 0, 0, 0, 4040, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3556, 7996, 4162, 3217, 1071, 4683, 4025, 4058] avg=4096.0000 max=7996 max_rank=1 min=1071 min_rank=4 imbalance_ratio=1.952148 max_load_reduction=24772 max_load_reduction_pct=0.755981 moved_to_echo=13837 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 3921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 743, 0, 993, 3, 0, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 24, 4096, 31, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 1, 0, 1526, 161, 0, 0, 0, 0, 0, 0, 76, 0, 0, 110, 0, 0, 0, 0, 2878, 711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 815, 0, 3846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 3986, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 4080, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3925, 5758, 4157, 1806, 3775, 4661, 4464, 4222] avg=4096.0000 max=5758 max_rank=1 min=1806 min_rank=3 imbalance_ratio=1.405762 max_load_reduction=27010 max_load_reduction_pct=0.824280 moved_to_echo=17122 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[768, 128, 128, 5760, 128, 128, 128, 0, 19456, 6784, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[5376, 384, 128, 512, 128, 11008, 128, 128, 14592, 1280, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[5120, 6400, 8192, 640, 0, 7808, 5120, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 0, 7936, 640, 24064, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[1664, 128, 256, 128, 0, 128, 256, 128, 27648, 3200, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[384, 256, 256, 0, 30336, 0, 640, 1152, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[0, 128, 128, 128, 128, 384, 128, 640, 31616, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 256, 32640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33792 slack_slots=31744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 0, 0, 0, 0, 221, 0, 404, 18, 0, 0, 0, 0, 4, 0, 0, 182, 4036, 29, 26, 0, 1758, 0, 1, 0, 4007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 4, 82, 6, 0, 27, 0, 1439, 0, 421, 4065, 51, 4050, 466, 0, 0, 0, 3291, 0, 4086, 0, 0, 4060] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 2, 1, 0, 1, 4040, 0, 1, 0, 0, 3, 0, 49, 2, 11, 4070, 223, 805, 0, 4084, 0, 0, 0, 4087, 0, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4085, 0, 3897, 0, 0, 0, 11, 2982, 0, 4096, 0, 0, 296] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 128, 0, 0, 0, 4046, 0, 0, 0, 0, 0, 188, 32, 0, 3512, 24, 0, 4096, 0, 0, 2, 378, 55, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 19, 19, 0, 4, 119, 0, 0, 0, 0, 24, 0, 0, 0, 534, 7, 4055, 0, 3944, 0, 0, 0, 0, 3423, 0, 4093, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 26, 3919, 0, 2985, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1839, 0, 3059, 0, 4096, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 40, 17, 0, 0, 0, 0, 12, 29, 8, 0, 33, 0, 0, 0, 0, 5, 33, 0, 2491, 2262, 0, 1651, 0, 6, 0, 0, 208, 4058, 0, 0, 0, 9, 0, 0, 0, 1, 0, 6, 0, 120, 0, 0, 0, 0, 0, 2, 0, 2302, 1340, 1617, 4087, 0, 3850, 19, 0, 0, 0, 3871, 0, 4056, 0, 0, 635] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 1, 9, 0, 0, 2, 0, 101, 0, 653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 1, 456, 0, 2563, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 4, 4002, 0, 1, 0, 0, 4, 0, 0, 4, 0, 0, 0, 0, 717, 0, 4055, 0, 3681, 83, 0, 0, 0, 4081, 0, 4094, 0, 0, 103] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 7, 0, 125, 0, 37, 0, 0, 0, 867, 143, 0, 0, 3, 0, 0, 0, 147, 1, 2, 4096, 0, 3846, 0, 1, 61, 2, 0, 4090, 26, 257, 2, 0, 0, 0, 0, 1, 0, 0, 50, 3, 1, 0, 0, 0, 11, 0, 0, 0, 3230, 0, 4096, 0, 3684, 0, 0, 1, 0, 3941, 0, 4037, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 0, 0, 4087, 0, 390, 0, 3047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 5, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 293, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 443, 5, 4096, 0, 4096, 0, 0, 0, 0, 4007, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 0, 0, 0, 0, 2031, 1190, 459, 0, 0, 0, 0, 0, 221, 0, 404, 18, 0, 0, 0, 0, 1501, 498, 224, 0, 0, 0, 0, 0, 4, 0, 0, 182, 2005, 29, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1758, 0, 1, 0, 2506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 3606, 37, 0, 0, 0, 0, 0, 0, 0, 0, 4, 82, 6, 0, 27, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 1439, 0, 421, 0, 51, 2362, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3067, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3682, 2866, 2246, 4265, 3675, 4168, 4739, 7127] avg=4096.0000 max=7127 max_rank=7 min=2246 min_rank=2 imbalance_ratio=1.739990 max_load_reduction=25641 max_load_reduction_pct=0.782501 moved_to_echo=19709 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 2061, 1203, 345, 0, 0, 0, 0, 0, 79, 0, 375, 0, 0, 0, 0, 0, 1534, 504, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2035, 26, 3919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2985, 0, 0, 0, 2562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 2714, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 1839, 0, 0, 0, 2389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3609, 2771, 5980, 5547, 2757, 4060, 4228, 3816] avg=4096.0000 max=5980 max_rank=2 min=2757 min_rank=4 imbalance_ratio=1.459961 max_load_reduction=26788 max_load_reduction_pct=0.817505 moved_to_echo=14861 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 128, 0, 0, 0, 2061, 1159, 459, 0, 0, 0, 0, 0, 4046, 0, 0, 0, 0, 0, 188, 32, 1522, 485, 233, 0, 0, 0, 0, 0, 0, 3512, 24, 0, 2035, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 378, 55, 0, 0, 2542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 19, 0, 4, 3596, 38, 0, 0, 0, 0, 0, 0, 119, 0, 0, 0, 0, 24, 0, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 534, 7, 0, 0, 2300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3809, 6506, 5573, 2975, 3676, 4198, 2841, 3190] avg=4096.0000 max=6506 max_rank=1 min=2841 min_rank=6 imbalance_ratio=1.588379 max_load_reduction=26262 max_load_reduction_pct=0.801453 moved_to_echo=13905 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 2, 1, 2048, 1145, 463, 0, 0, 0, 0, 0, 0, 1, 4040, 0, 1, 0, 0, 3, 1531, 479, 203, 0, 0, 0, 0, 0, 0, 49, 2, 11, 2022, 223, 805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 2556, 0, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3622, 37, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 2273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 2779, 0, 0, 0, 0, 296, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3659, 6258, 3112, 6651, 3659, 4061, 2282, 3086] avg=4096.0000 max=6651 max_rank=3 min=2282 min_rank=6 imbalance_ratio=1.623779 max_load_reduction=26117 max_load_reduction_pct=0.797028 moved_to_echo=13088 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 9, 0, 0, 2, 0, 2054, 1081, 458, 0, 0, 0, 0, 0, 101, 0, 653, 0, 0, 0, 0, 0, 1525, 453, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2028, 1, 456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2563, 0, 0, 0, 2546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4002, 0, 1, 0, 3597, 37, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 0, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 717, 0, 0, 0, 2147, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3803, 0, 0, 0, 0, 103, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3605, 3010, 2485, 5109, 7641, 4065, 2947, 3906] avg=4096.0000 max=7641 max_rank=4 min=2485 min_rank=2 imbalance_ratio=1.865479 max_load_reduction=25127 max_load_reduction_pct=0.766815 moved_to_echo=18559 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 40, 17, 0, 0, 0, 0, 12, 1138, 1131, 463, 0, 0, 0, 0, 0, 29, 8, 0, 33, 0, 0, 0, 0, 1520, 473, 264, 0, 0, 0, 0, 0, 5, 33, 0, 2491, 1124, 0, 1651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 208, 2538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 1, 0, 6, 0, 3624, 36, 0, 0, 0, 0, 0, 0, 120, 0, 0, 0, 0, 0, 2, 0, 4020, 0, 0, 0, 0, 0, 0, 0, 2302, 1340, 1617, 0, 0, 2246, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3607, 0, 0, 0, 0, 635, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2801, 2327, 5304, 2752, 3676, 4142, 7524, 4242] avg=4096.0000 max=7524 max_rank=6 min=2327 min_rank=1 imbalance_ratio=1.836914 max_load_reduction=25244 max_load_reduction_pct=0.770386 moved_to_echo=19584 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 0, 125, 0, 37, 0, 0, 2061, 1082, 463, 0, 0, 0, 0, 0, 0, 867, 143, 0, 0, 3, 0, 0, 1532, 453, 269, 0, 0, 0, 0, 0, 0, 147, 1, 2, 2035, 0, 3846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 61, 2, 0, 2558, 26, 257, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 50, 3633, 36, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 11, 0, 0, 4001, 0, 0, 0, 0, 0, 0, 0, 0, 3230, 0, 0, 0, 2149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3775, 3267, 6031, 2907, 3720, 4016, 5379, 3673] avg=4096.0000 max=6031 max_rank=2 min=2907 min_rank=3 imbalance_ratio=1.472412 max_load_reduction=26737 max_load_reduction_pct=0.815948 moved_to_echo=16788 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 0, 0, 4087, 0, 2063, 1208, 464, 0, 0, 0, 0, 0, 390, 0, 3047, 0, 0, 0, 0, 0, 1539, 508, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2033, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 2557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 1, 0, 0, 3632, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 443, 5, 0, 0, 2380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7828, 5763, 2037, 2562, 3964, 4058, 2828, 3728] avg=4096.0000 max=7828 max_rank=0 min=2037 min_rank=2 imbalance_ratio=1.911133 max_load_reduction=24940 max_load_reduction_pct=0.761108 moved_to_echo=14578 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 256, 128, 128, 4096, 128, 15616, 9216, 3584, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4992, 896, 8704, 128, 128, 128, 256, 128, 12288, 3968, 2048, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[11904, 128, 128, 256, 20480, 128, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 3840, 128, 2688, 15360, 384, 10752, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3840, 8192, 2176, 0, 128, 18304, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 4352, 128, 128, 128, 28032, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 27776, 0, 0, 0, 0, 5120, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[256, 128, 128, 128, 128, 128, 128, 0, 32384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 1281, 0, 0, 2374, 1, 2810, 53, 3985, 1, 2758, 0, 989, 0, 0, 0, 0, 6, 0, 0, 9, 29, 0, 0, 1250, 3, 0, 72, 3941, 0, 0, 348, 21, 0, 3, 0, 0, 860, 26, 7, 0, 840, 0, 0, 0, 0, 4044, 2930, 4, 0, 0, 0, 2, 0, 19, 0, 0, 0, 0, 0, 0, 6, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 4060, 0, 1524, 0, 4094, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3918, 4070, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 19, 0, 1793, 0, 1150, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 3701] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 2, 0, 5, 3788, 0, 0, 0, 4092, 0, 193, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 43, 0, 9, 0, 0, 32, 6, 4066, 0, 0, 3964, 4095, 1547, 0, 5, 0, 4061, 0, 0, 0, 1, 1, 0, 0, 561, 0, 2258, 0, 0, 0, 0, 0, 1, 1, 0, 1, 7, 0, 0, 0, 224, 0, 3766] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 14, 0, 0, 4010, 0, 1953, 0, 3885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3288, 0, 36, 1, 0, 59, 0, 35, 0, 4092, 0, 45, 3002, 589, 622, 0, 294, 0, 0, 0, 0, 0, 0, 4, 106, 0, 0, 146, 3920, 0, 0, 0, 0, 0, 2563, 1, 2, 0, 1, 0, 0, 0, 40, 0, 4060] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 1, 0, 70, 3952, 0, 3646, 1, 4032, 0, 2709, 0, 1, 0, 0, 0, 0, 0, 0, 38, 0, 97, 0, 0, 0, 0, 0, 0, 4012, 15, 149, 0, 663, 0, 0, 0, 0, 3847, 0, 0, 7, 2, 0, 0, 120, 241, 3, 4075, 0, 0, 3, 0, 0, 0, 6, 979, 0, 0, 0, 0, 0, 0, 4, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 287, 4059, 0, 4060, 0, 3738, 2, 0, 0, 102, 0, 0, 0, 0, 0, 1, 0, 0, 3, 1, 0, 3, 3, 0, 0, 4074, 35, 0, 3555, 2755, 30, 0, 0, 100, 85, 9, 0, 0, 23, 0, 1204, 0, 3771, 21, 542, 0, 0, 0, 0, 0, 51, 179, 0, 0, 0, 15, 44, 0, 0, 0, 4016] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 2560, 0, 12, 2144, 0, 0, 0, 3565, 0, 183, 0, 3987, 10, 3, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 32, 1255, 0, 4096, 0, 0, 4066, 4096, 0, 0, 214, 0, 0, 0, 0, 0, 16, 214, 0, 0, 4048, 0, 2068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 14, 0, 0, 3, 173] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 3719, 0, 0, 4096, 4, 176, 0, 847, 0, 1155, 0, 3633, 0, 0, 0, 0, 0, 0, 1880, 11, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 78, 4079, 1612, 0, 0, 785, 16, 0, 0, 0, 0, 0, 0, 0, 747, 0, 1733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1281, 0, 0, 883, 1, 2810, 53, 0, 0, 0, 0, 0, 0, 0, 0, 2390, 1, 2758, 0, 989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 9, 29, 0, 0, 1595, 881, 5, 0, 0, 0, 0, 0, 1250, 3, 0, 72, 1316, 0, 0, 348, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 3, 0, 0, 860, 26, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 840, 0, 0, 0, 0, 4044, 2328, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 2, 0, 19, 0, 2625, 610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 602, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5028, 6138, 2525, 2989, 912, 7212, 3260, 4704] avg=4096.0000 max=7212 max_rank=5 min=912 min_rank=4 imbalance_ratio=1.760742 max_load_reduction=25556 max_load_reduction_pct=0.779907 moved_to_echo=16088 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 1510, 0, 1524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2455, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 0, 1639, 1507, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1368, 0, 0, 3918, 0, 0, 0, 0, 0, 0, 0, 0, 2907, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 1793, 0, 914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 2728, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 3701, 236, 80, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3037, 2528, 4234, 5286, 7009, 2726, 3775, 4173] avg=4096.0000 max=7009 max_rank=4 min=2528 min_rank=1 imbalance_ratio=1.711182 max_load_reduction=25759 max_load_reduction_pct=0.786102 moved_to_echo=17683 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 5, 1409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2454, 0, 193, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 9, 1638, 1406, 1089, 0, 0, 0, 0, 0, 0, 0, 32, 6, 1358, 0, 0, 3964, 0, 0, 0, 0, 0, 0, 0, 0, 2925, 1547, 0, 5, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 561, 0, 1794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 2708, 973, 0, 0, 0, 0, 0, 0, 1, 7, 0, 0, 0, 224, 0, 3766, 464, 81, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1416, 2686, 4185, 5360, 8538, 2357, 3683, 4543] avg=4096.0000 max=8538 max_rank=4 min=1416 min_rank=0 imbalance_ratio=2.084473 max_load_reduction=24230 max_load_reduction_pct=0.739441 moved_to_echo=19121 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 2560, 0, 12, 798, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2138, 0, 183, 0, 3987, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 1427, 796, 1089, 0, 0, 0, 0, 0, 0, 32, 1255, 0, 1368, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 2926, 0, 0, 214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 214, 0, 0, 4048, 0, 1643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2728, 550, 0, 0, 0, 0, 0, 0, 0, 2, 1, 14, 0, 0, 3, 173, 425, 81, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3370, 6321, 3318, 6721, 3140, 5921, 3278, 699] avg=4096.0000 max=6721 max_rank=3 min=699 min_rank=7 imbalance_ratio=1.640869 max_load_reduction=26047 max_load_reduction_pct=0.794891 moved_to_echo=13038 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 14, 0, 0, 1491, 0, 1953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3288, 0, 36, 1, 0, 1555, 1489, 156, 0, 0, 0, 0, 0, 59, 0, 35, 0, 1366, 0, 45, 3002, 0, 0, 0, 0, 0, 0, 0, 0, 422, 622, 0, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 106, 0, 0, 146, 3115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2563, 1, 2, 2726, 1030, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 40, 0, 4060, 805, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3458, 2330, 6525, 4507, 1338, 3371, 6322, 4917] avg=4096.0000 max=6525 max_rank=2 min=1338 min_rank=4 imbalance_ratio=1.593018 max_load_reduction=26243 max_load_reduction_pct=0.800873 moved_to_echo=15948 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 287, 1509, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2242, 2, 0, 0, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 1, 0, 1496, 1507, 733, 0, 0, 0, 0, 0, 3, 3, 0, 0, 1360, 35, 0, 3555, 0, 0, 0, 0, 0, 0, 0, 0, 1968, 30, 0, 0, 100, 85, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 1204, 0, 3771, 21, 431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 179, 0, 2714, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 15, 44, 0, 0, 0, 4016, 111, 54, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5856, 2346, 3741, 4956, 2192, 5450, 3987, 4240] avg=4096.0000 max=5856 max_rank=0 min=2192 min_rank=4 imbalance_ratio=1.429688 max_load_reduction=26912 max_load_reduction_pct=0.821289 moved_to_echo=15869 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 70, 1470, 0, 3646, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2418, 0, 2709, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 97, 0, 0, 1614, 1467, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1340, 15, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 0, 0, 0, 0, 3847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2, 0, 0, 120, 241, 3, 3238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 6, 979, 2672, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4095, 837, 13, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5188, 5128, 3392, 1504, 4321, 3611, 4675, 4949] avg=4096.0000 max=5188 max_rank=0 min=1504 min_rank=3 imbalance_ratio=1.266602 max_load_reduction=27580 max_load_reduction_pct=0.841675 moved_to_echo=17556 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 3719, 0, 0, 1516, 4, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 0, 1155, 0, 3633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1880, 11, 0, 0, 0, 344, 1524, 1089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1364, 0, 3, 78, 0, 0, 0, 0, 0, 0, 0, 0, 2905, 1612, 0, 0, 785, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 747, 0, 1373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2732, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 360, 85, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5415, 5291, 4848, 1445, 5318, 2120, 3788, 4543] avg=4096.0000 max=5415 max_rank=0 min=1445 min_rank=3 imbalance_ratio=1.322021 max_load_reduction=27353 max_load_reduction_pct=0.834747 moved_to_echo=15769 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[0, 128, 128, 5248, 128, 256, 128, 128, 11392, 10624, 5504, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[14592, 3840, 128, 640, 896, 13056, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 0, 128, 128, 128, 2688, 256, 1024, 21760, 7424, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1408, 128, 1408, 128, 10880, 128, 256, 18944, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 7680, 0, 384, 10624, 128, 14208, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[17024, 128, 7040, 0, 8832, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 896, 256, 1408, 128, 11264, 4224, 14848, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 0, 512, 128, 28032, 3840, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[2670, 0, 3857, 0, 0, 161, 3894, 0, 0, 0, 0, 3049, 9, 0, 0, 0, 1, 0, 4, 0, 0, 1, 0, 4053, 10, 0, 304, 1, 0, 4069, 0, 2485, 0, 0, 0, 0, 0, 0, 0, 0, 150, 7, 66, 3935, 0, 0, 351, 55, 0, 12, 0, 0, 1, 0, 200, 27, 2, 3309, 0, 0, 0, 0, 84, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[30, 0, 43, 0, 0, 0, 4092, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4001, 0, 3950, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 342, 0, 0, 0, 0, 0, 0, 0, 0, 3977, 3971, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[3608, 0, 3195, 1, 0, 0, 4096, 0, 0, 0, 0, 3799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 556, 0, 0, 24, 3424, 0, 0, 0, 0, 0, 0, 0, 0, 223, 5, 0, 2, 0, 4096, 0, 1614, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 4020, 2, 3, 27, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[2487, 0, 4096, 0, 203, 421, 4027, 0, 0, 0, 10, 9, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4084, 0, 0, 2, 0, 0, 946, 0, 4096, 0, 0, 0, 60, 0, 0, 21, 0, 2, 65, 0, 4019, 0, 2265, 0, 0, 0, 1, 0, 0, 0, 11, 0, 0, 3359, 859, 0, 184, 0, 33, 1505, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[4094, 0, 4054, 0, 0, 0, 2764, 0, 0, 0, 0, 3949, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 65, 0, 0, 64, 0, 42, 0, 689, 0, 0, 0, 0, 0, 0, 19, 0, 0, 4096, 37, 601, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[2961, 0, 3912, 0, 516, 0, 3763, 294, 4, 0, 5, 0, 7, 0, 0, 0, 0, 4043, 0, 2, 8, 0, 0, 4095, 3, 233, 139, 0, 0, 3115, 2, 52, 0, 1641, 25, 0, 0, 0, 0, 0, 19, 2570, 0, 4077, 0, 13, 0, 608, 0, 4, 0, 0, 0, 0, 0, 0, 131, 277, 0, 0, 79, 1, 169, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[899, 0, 4065, 0, 0, 290, 3386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 3967, 0, 0, 0, 0, 0, 4044, 0, 0, 4, 0, 0, 3196, 0, 2, 0, 0, 366, 120, 91, 0, 0, 0, 0, 0, 0, 838, 0, 3943, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 3476, 3994, 0, 0, 0, 0, 4, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[8, 0, 4090, 53, 0, 0, 1210, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 3969, 5, 26, 52, 0, 7, 4069, 6, 163, 3, 0, 0, 0, 0, 3, 0, 0, 0, 521, 0, 4061, 0, 3053, 0, 3286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 3, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[30, 0, 0, 0, 0, 0, 2115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 982, 584, 409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3592, 0, 3950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 1977, 1529, 865, 0, 0, 0, 0, 0, 0, 0, 0, 1848, 0, 0, 5, 342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 719, 0, 0, 0, 0, 0, 0, 3977, 2522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2145, 6057, 3114, 7542, 4454, 2195, 762, 6499] avg=4096.0000 max=7542 max_rank=3 min=762 min_rank=6 imbalance_ratio=1.841309 max_load_reduction=25226 max_load_reduction_pct=0.769836 moved_to_echo=13910 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2670, 0, 0, 0, 0, 161, 2013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3049, 9, 0, 0, 0, 972, 486, 416, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 1, 0, 3081, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 304, 1, 0, 3653, 0, 2485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1881, 1469, 721, 0, 0, 0, 0, 0, 150, 7, 66, 1775, 0, 0, 351, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 1, 0, 200, 27, 3857, 691, 0, 0, 0, 0, 0, 0, 2, 2102, 0, 0, 0, 0, 84, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4844, 4932, 3087, 6453, 4071, 2404, 4788, 2189] avg=4096.0000 max=6453 max_rank=3 min=2189 min_rank=7 imbalance_ratio=1.575439 max_load_reduction=26315 max_load_reduction_pct=0.803070 moved_to_echo=13452 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2487, 0, 0, 0, 203, 421, 2081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 9, 0, 0, 0, 0, 979, 126, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 850, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 21, 0, 1946, 1500, 187, 0, 0, 0, 0, 0, 2, 65, 0, 1814, 0, 2265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 11, 0, 0, 4096, 705, 0, 0, 0, 0, 0, 0, 3359, 546, 0, 184, 0, 33, 1505, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5192, 1220, 3108, 4948, 3714, 4146, 4813, 5627] avg=4096.0000 max=5627 max_rank=7 min=1220 min_rank=1 imbalance_ratio=1.373779 max_load_reduction=27141 max_load_reduction_pct=0.828278 moved_to_echo=18300 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3608, 0, 0, 1, 0, 0, 2117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3799, 0, 0, 0, 0, 974, 591, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 0, 0, 24, 3074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 223, 5, 1979, 1529, 876, 0, 0, 0, 0, 0, 0, 2, 0, 1848, 0, 1614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 3195, 719, 0, 0, 0, 0, 0, 0, 0, 2553, 2, 3, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5726, 5714, 3091, 3654, 4612, 3464, 3922, 2585] avg=4096.0000 max=5726 max_rank=0 min=2585 min_rank=7 imbalance_ratio=1.397949 max_load_reduction=27042 max_load_reduction_pct=0.825256 moved_to_echo=14583 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2961, 0, 0, 0, 516, 0, 1945, 294, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 5, 0, 7, 0, 0, 0, 982, 40, 319, 0, 0, 0, 0, 0, 0, 4043, 0, 2, 8, 0, 0, 3113, 0, 0, 0, 0, 0, 0, 0, 0, 3, 233, 139, 0, 0, 2796, 2, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1641, 25, 0, 0, 0, 0, 0, 1818, 1522, 60, 0, 0, 0, 0, 0, 19, 2570, 0, 1839, 0, 13, 0, 608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 3912, 716, 0, 0, 0, 0, 0, 0, 131, 177, 0, 0, 79, 1, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5716, 1357, 7166, 3225, 5066, 5049, 4632, 557] avg=4096.0000 max=7166 max_rank=2 min=557 min_rank=7 imbalance_ratio=1.749512 max_load_reduction=25602 max_load_reduction_pct=0.781311 moved_to_echo=15304 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[899, 0, 0, 0, 0, 290, 1750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 969, 587, 327, 0, 0, 0, 0, 0, 25, 3967, 0, 0, 0, 0, 0, 3075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 2869, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 366, 120, 91, 0, 0, 0, 1636, 312, 870, 0, 0, 0, 0, 0, 0, 0, 0, 379, 0, 3943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 4065, 147, 0, 0, 0, 0, 0, 0, 3476, 2537, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2939, 1883, 7067, 2875, 3395, 4322, 4270, 6017] avg=4096.0000 max=7067 max_rank=2 min=1883 min_rank=1 imbalance_ratio=1.725342 max_load_reduction=25701 max_load_reduction_pct=0.784332 moved_to_echo=18004 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 0, 0, 1425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3949, 0, 0, 4, 0, 986, 607, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 52, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 689, 0, 0, 0, 0, 0, 0, 1339, 1535, 895, 0, 0, 0, 0, 0, 19, 0, 0, 1839, 37, 601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4054, 722, 0, 0, 0, 0, 0, 0, 4096, 2592, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5519, 5558, 3110, 159, 4458, 2496, 4778, 6690] avg=4096.0000 max=6690 max_rank=7 min=159 min_rank=3 imbalance_ratio=1.633301 max_load_reduction=26078 max_load_reduction_pct=0.795837 moved_to_echo=18422 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 0, 53, 0, 0, 626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 951, 602, 416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 3018, 0, 0, 0, 0, 0, 0, 0, 0, 5, 26, 52, 0, 7, 3653, 6, 163, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3, 0, 0, 584, 1516, 892, 0, 0, 0, 0, 0, 0, 521, 0, 1832, 0, 3053, 0, 3286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 713, 0, 0, 0, 0, 0, 0, 0, 2601, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[687, 6047, 3025, 3912, 2998, 8692, 4803, 2604] avg=4096.0000 max=8692 max_rank=5 min=687 min_rank=0 imbalance_ratio=2.122070 max_load_reduction=24076 max_load_reduction_pct=0.734741 moved_to_echo=19097 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 2432, 512, 256, 128, 128, 256, 128, 13184, 11008, 5376, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[16768, 0, 0, 128, 768, 896, 14080, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 0, 128, 19072, 128, 0, 128, 0, 7808, 3712, 2432, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[15104, 15744, 128, 256, 128, 128, 1792, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 3200, 128, 13184, 128, 11520, 384, 4352, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 896, 640, 128, 128, 20608, 128, 10880, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 8064, 128, 128, 128, 128, 128, 24832, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 0, 128, 128, 128, 256, 128, 27392, 5248, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3135, 0, 0, 29, 0, 0, 63, 0, 299, 33, 0, 0, 4054, 8, 2397, 0, 2, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 24, 3510, 0, 0, 3903, 1, 0, 0, 0, 3919, 0, 0, 722, 3, 285, 0, 13, 4096, 4050, 198, 0, 0, 0, 0, 1648, 67, 0, 0, 125, 102, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 3972, 0, 2, 0, 0, 0, 1791, 0, 0, 2, 19, 0, 4086, 0, 3915, 0, 0, 0, 794, 0, 4, 0, 0, 0, 0, 79, 37, 0, 4088, 774, 0, 4092, 0, 0, 1, 7, 3480, 0, 0, 0, 0, 0, 0, 7, 105, 1462, 0, 0, 45, 0, 0, 3985, 0, 0, 0, 0, 0, 8, 0, 11] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4035, 0, 0, 0, 0, 0, 0, 0, 2790, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2049, 0, 4096, 0, 0, 0, 0, 3144, 3, 0, 0, 0, 0, 0, 7, 2000, 1710, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 10, 0, 13, 35, 0, 0, 296, 0, 3869, 0, 0, 0, 2337, 0, 3995, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 17, 0, 2, 3703, 0, 0, 3962, 0, 0, 0, 2, 0, 0, 0, 2775, 36, 0, 0, 134, 4096, 3784, 5, 1, 0, 0, 0, 3672, 13, 2, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 4088, 0, 0, 0, 0, 0, 1095, 0, 3662, 0, 0, 0, 32, 0, 3467, 0, 0, 0, 3958, 0, 0, 0, 0, 7, 1, 0, 1, 0, 4096, 0, 1, 4034, 0, 0, 0, 0, 8, 0, 0, 573, 0, 253, 0, 0, 2052, 2444, 0, 0, 0, 0, 0, 2414, 0, 486, 0, 0, 0, 93, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 267, 0, 1, 13, 25, 0, 46, 0, 0, 124, 0, 3150, 0, 2173, 0, 0, 15, 0, 1, 0, 102, 5, 0, 3, 812, 0, 0, 3736, 0, 10, 4082, 1, 0, 0, 0, 38, 0, 0, 2217, 0, 5, 0, 0, 3962, 487, 0, 0, 0, 0, 3, 4089, 0, 0, 0, 0, 0, 0, 312, 2993] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 317, 0, 128, 0, 0, 0, 4095, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 4096, 0, 0, 0, 0, 3310, 150, 0, 0, 0, 0, 0, 42, 3269, 3899, 0, 0, 0, 0, 0, 516, 0, 707, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 23, 2335, 0, 0, 0, 0, 0, 0, 0, 135, 0, 1, 0, 90, 0, 4082, 0, 0, 0, 3448, 40, 0, 0, 0, 0, 0, 0, 0, 2, 3869, 181, 0, 4085, 0, 15, 0, 0, 4094, 0, 0, 0, 0, 11, 0, 0, 3977, 4096, 0, 2, 0, 0, 0, 136, 0, 2139, 0, 0, 0, 0, 0, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3135, 0, 0, 29, 1180, 724, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 299, 33, 0, 0, 2916, 110, 0, 0, 0, 0, 0, 0, 4054, 8, 165, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 24, 3811, 0, 0, 0, 0, 0, 0, 0, 2885, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3919, 0, 0, 722, 3, 285, 0, 13, 625, 365, 0, 0, 0, 0, 0, 0, 0, 2961, 198, 0, 0, 0, 0, 1648, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 125, 102, 0, 0, 0, 2122, 92, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5068, 3421, 4229, 3917, 2886, 5932, 4807, 2508] avg=4096.0000 max=5932 max_rank=5 min=2508 min_rank=7 imbalance_ratio=1.448242 max_load_reduction=26836 max_load_reduction_pct=0.818970 moved_to_echo=16133 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4035, 0, 0, 0, 576, 306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2790, 0, 0, 0, 1424, 188, 0, 0, 0, 0, 0, 0, 4093, 0, 282, 0, 0, 0, 624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4000, 0, 0, 0, 0, 0, 0, 0, 3366, 2049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3144, 3, 0, 0, 0, 0, 0, 7, 730, 154, 0, 0, 0, 0, 0, 0, 0, 1250, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3626, 96, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4917, 4402, 4999, 4000, 5415, 4038, 1275, 3722] avg=4096.0000 max=5415 max_rank=4 min=1275 min_rank=6 imbalance_ratio=1.322021 max_load_reduction=27353 max_load_reduction_pct=0.834747 moved_to_echo=14450 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 3972, 0, 2, 0, 31, 261, 0, 0, 0, 0, 0, 0, 0, 0, 1791, 0, 0, 2, 19, 0, 74, 179, 0, 0, 0, 0, 0, 0, 4086, 0, 271, 0, 0, 0, 794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 79, 37, 0, 3996, 0, 0, 0, 0, 0, 0, 0, 3360, 774, 0, 0, 0, 0, 1, 7, 0, 0, 0, 0, 0, 0, 0, 0, 3480, 0, 0, 0, 0, 0, 0, 7, 728, 131, 0, 0, 0, 0, 0, 0, 0, 1070, 0, 0, 45, 0, 0, 3985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 11, 3465, 96, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4268, 2065, 5151, 4116, 4142, 4346, 5100, 3580] avg=4096.0000 max=5151 max_rank=2 min=2065 min_rank=1 imbalance_ratio=1.257568 max_load_reduction=27617 max_load_reduction_pct=0.842804 moved_to_echo=17168 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 4088, 0, 0, 0, 591, 437, 0, 0, 0, 0, 0, 0, 0, 0, 1095, 0, 3662, 0, 0, 0, 1461, 159, 0, 0, 0, 0, 0, 0, 32, 0, 239, 0, 0, 0, 3958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1, 0, 1, 0, 3938, 0, 0, 0, 0, 0, 0, 0, 3366, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 573, 0, 253, 0, 0, 730, 220, 0, 0, 0, 0, 0, 0, 0, 1787, 0, 0, 0, 0, 0, 2414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 0, 0, 0, 93, 0, 0, 3069, 96, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5119, 6377, 4229, 3947, 3367, 1784, 4201, 3744] avg=4096.0000 max=6377 max_rank=1 min=1784 min_rank=5 imbalance_ratio=1.556885 max_load_reduction=26391 max_load_reduction_pct=0.805389 moved_to_echo=13096 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 267, 0, 1, 1142, 87, 0, 0, 0, 0, 0, 0, 13, 25, 0, 46, 0, 0, 124, 0, 2820, 99, 0, 0, 0, 0, 0, 0, 3150, 0, 151, 0, 0, 15, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 5, 0, 3, 812, 0, 0, 3985, 0, 0, 0, 0, 0, 0, 0, 3070, 0, 10, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 2217, 0, 5, 0, 0, 666, 43, 0, 0, 0, 0, 0, 0, 0, 357, 0, 0, 0, 0, 3, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 312, 2993, 1923, 97, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5593, 3127, 3317, 4907, 3081, 2969, 4449, 5325] avg=4096.0000 max=5593 max_rank=0 min=2969 min_rank=5 imbalance_ratio=1.365479 max_load_reduction=27175 max_load_reduction_pct=0.829315 moved_to_echo=15824 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 10, 0, 13, 35, 1181, 677, 0, 0, 0, 0, 0, 0, 0, 0, 296, 0, 3869, 0, 0, 0, 2915, 183, 0, 0, 0, 0, 0, 0, 2337, 0, 276, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 17, 0, 2, 3869, 0, 0, 0, 0, 0, 0, 0, 3043, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2775, 36, 0, 0, 134, 660, 341, 0, 0, 0, 0, 0, 0, 0, 2766, 5, 1, 0, 0, 0, 3672, 0, 0, 0, 0, 0, 0, 0, 0, 13, 2, 0, 0, 0, 0, 0, 0, 3536, 93, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1920, 7263, 2616, 3890, 3045, 3946, 6444, 3644] avg=4096.0000 max=7263 max_rank=1 min=1920 min_rank=0 imbalance_ratio=1.773193 max_load_reduction=25505 max_load_reduction_pct=0.778351 moved_to_echo=17079 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 943, 701, 0, 0, 0, 0, 0, 0, 0, 0, 317, 0, 128, 0, 0, 0, 2326, 189, 0, 0, 0, 0, 0, 0, 4095, 0, 272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4000, 0, 0, 0, 0, 0, 0, 0, 3362, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3310, 150, 0, 0, 0, 0, 0, 42, 734, 354, 0, 0, 0, 0, 0, 0, 0, 2844, 0, 0, 0, 0, 0, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 707, 0, 0, 0, 0, 0, 0, 3587, 96, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1645, 2960, 4367, 4000, 7456, 4590, 3360, 4390] avg=4096.0000 max=7456 max_rank=4 min=1645 min_rank=0 imbalance_ratio=1.820312 max_load_reduction=25312 max_load_reduction_pct=0.772461 moved_to_echo=19796 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 23, 2335, 0, 0, 0, 1147, 733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 0, 1, 0, 2830, 187, 0, 0, 0, 0, 0, 0, 90, 0, 282, 0, 0, 0, 3448, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3989, 0, 0, 0, 0, 0, 0, 0, 3180, 181, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 11, 0, 0, 689, 369, 0, 0, 0, 0, 0, 0, 0, 2994, 0, 2, 0, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2139, 0, 0, 0, 0, 0, 7, 3613, 96, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4238, 3153, 3860, 3991, 3376, 5163, 3132, 5855] avg=4096.0000 max=5855 max_rank=7 min=3132 min_rank=6 imbalance_ratio=1.429443 max_load_reduction=26913 max_load_reduction_pct=0.821320 moved_to_echo=17526 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[0, 128, 128, 128, 21760, 384, 128, 128, 6912, 3968, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 3584, 128, 11008, 128, 256, 0, 16768, 1408, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 16128, 256, 128, 128, 0, 128, 16512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[25728, 7168, 128, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[18048, 256, 0, 6400, 128, 640, 0, 256, 5632, 2048, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[22016, 128, 2048, 0, 128, 128, 8832, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 3456, 0, 128, 128, 128, 384, 3072, 24960, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 256, 128, 128, 128, 1024, 128, 128, 31616, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 3927, 3906, 0, 1, 0, 1, 2, 1, 3, 29, 0, 0, 8, 4069, 0, 0, 0, 0, 7, 16, 0, 23, 0, 32, 132, 0, 11, 0, 0, 169, 479, 0, 31, 266, 3712, 1, 0, 0, 54, 0, 4032, 4, 0, 0, 0, 4087, 41, 4047, 3581, 0, 0, 0, 11, 0, 7, 0, 67, 10, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 3453, 0, 0, 2, 0, 0, 0, 0, 1970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2112, 0, 0, 2, 0, 0, 0, 0, 0, 168, 9, 1100, 0, 0, 3, 109, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4035, 0, 3649, 0, 0, 0, 3868, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 3176, 3742, 0, 0, 0, 397, 0, 0, 0, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 18, 0, 0, 8, 0, 0, 0, 0, 2300, 21, 4091, 0, 12, 0, 0, 0, 2096, 0, 0, 0, 3498, 42, 0, 60, 0, 3726, 0, 1743, 0, 0, 589, 90, 0, 0, 9, 0, 0, 1961, 90, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 36, 4024, 3863, 0, 3, 0, 0, 59, 0, 0, 0, 0, 0, 0, 1127, 0, 0, 5, 12, 0, 3989, 0, 0, 0, 4, 51, 0, 0, 38, 3953, 0, 4, 142, 1, 4, 232, 0, 3979, 0, 0, 140, 3854, 0, 142, 0, 0, 689, 0, 2653, 0, 0, 68, 3639, 0, 0, 20, 0, 0, 0, 34, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4069, 4091, 0, 179, 6, 1, 0, 0, 0, 1374, 0, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 820, 2353, 0, 0, 45, 0, 0, 3819, 0, 0, 1, 4068, 2, 0, 0, 0, 4073, 1, 3728, 0, 0, 0, 3907, 0, 0, 0, 0, 0, 85, 0, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[38, 0, 0, 0, 4038, 4096, 0, 313, 0, 0, 91, 0, 0, 1220, 0, 0, 0, 4096, 0, 0, 0, 0, 390, 1442, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4005, 3, 164, 0, 16, 0, 4064, 0, 0, 0, 3920, 0, 0, 0, 0, 84, 0, 393, 0, 0, 26, 3719, 0, 0, 0, 0, 5, 625, 0, 19] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 2260, 0, 0, 0, 2074, 0, 0, 0, 3, 0, 3902, 0, 0, 0, 0, 0, 0, 0, 19, 191, 0, 2087, 0, 0, 0, 4096, 0, 4089, 0, 0, 0, 4089, 0, 0, 0, 0, 1221, 0, 0, 0, 0, 0, 477, 0, 0, 0, 0, 2, 67, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 11, 4092, 4051, 0, 0, 0, 0, 0, 0, 0, 265, 0, 0, 2, 5, 0, 0, 0, 0, 0, 170, 2, 0, 22, 0, 0, 0, 1, 0, 0, 49, 2, 2, 81, 8, 4094, 0, 1853, 0, 0, 2268, 4061, 1, 0, 0, 0, 3920, 0, 3881, 29, 0, 257, 3641, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 116, 3453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1970, 0, 0, 2938, 1486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2112, 558, 160, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 3980, 226, 0, 0, 0, 0, 0, 0, 168, 9, 940, 0, 0, 3, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1605, 0, 0, 0, 3868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3085, 1097, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3569, 6396, 2830, 4208, 1229, 4881, 5473, 4182] avg=4096.0000 max=6396 max_rank=1 min=1229 min_rank=4 imbalance_ratio=1.561523 max_load_reduction=26372 max_load_reduction_pct=0.804810 moved_to_echo=15765 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 111, 3906, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 1, 3, 29, 0, 0, 2976, 1648, 0, 0, 0, 0, 0, 0, 8, 4069, 0, 0, 0, 0, 7, 16, 619, 69, 0, 0, 0, 0, 0, 0, 0, 23, 0, 32, 132, 0, 11, 0, 3816, 222, 0, 0, 0, 0, 0, 0, 0, 169, 410, 0, 31, 266, 3712, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 774, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 1780, 3581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 7, 0, 67, 10, 0, 0, 3036, 1111, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4019, 4660, 4788, 4236, 4589, 832, 5402, 4242] avg=4096.0000 max=5402 max_rank=6 min=832 min_rank=5 imbalance_ratio=1.318848 max_load_reduction=27366 max_load_reduction_pct=0.835144 moved_to_echo=15065 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 90, 3742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 397, 0, 0, 0, 1001, 0, 0, 2713, 709, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 266, 595, 0, 0, 0, 0, 0, 0, 18, 0, 0, 8, 0, 0, 0, 0, 3086, 193, 0, 0, 0, 0, 0, 0, 2300, 21, 3496, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2096, 0, 0, 0, 671, 42, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 768, 0, 0, 589, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 1961, 90, 0, 2634, 1013, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3834, 4820, 4957, 3305, 5829, 2869, 1447, 5707] avg=4096.0000 max=5829 max_rank=4 min=1447 min_rank=6 imbalance_ratio=1.423096 max_load_reduction=26939 max_load_reduction_pct=0.822113 moved_to_echo=15852 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 36, 113, 3863, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 502, 1084, 0, 0, 0, 0, 0, 0, 0, 1127, 0, 0, 5, 12, 0, 3989, 408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 51, 0, 0, 38, 3911, 212, 0, 0, 0, 0, 0, 0, 3953, 0, 4, 142, 1, 4, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3979, 0, 0, 140, 740, 0, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1161, 0, 0, 68, 3639, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 34, 3, 2902, 187, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4015, 1645, 5541, 4216, 4336, 5001, 4868, 3146] avg=4096.0000 max=5541 max_rank=2 min=1645 min_rank=1 imbalance_ratio=1.352783 max_load_reduction=27227 max_load_reduction_pct=0.830902 moved_to_echo=17351 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 115, 4091, 0, 179, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 0, 0, 0, 1374, 0, 0, 2966, 1518, 0, 0, 0, 0, 0, 0, 0, 114, 0, 0, 0, 0, 0, 0, 570, 342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 3954, 224, 0, 0, 0, 0, 0, 0, 0, 820, 2011, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3819, 0, 0, 1, 780, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1640, 0, 0, 0, 3907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 4, 3064, 1107, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4386, 5865, 1026, 4205, 2876, 4602, 5548, 4260] avg=4096.0000 max=5865 max_rank=1 min=1026 min_rank=2 imbalance_ratio=1.431885 max_load_reduction=26903 max_load_reduction_pct=0.821014 moved_to_echo=17286 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[38, 0, 0, 0, 114, 4096, 0, 313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 1220, 0, 0, 62, 160, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 390, 1442, 60, 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3924, 216, 0, 0, 0, 0, 0, 0, 0, 0, 3423, 3, 164, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173, 0, 0, 26, 3719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 625, 0, 19, 2952, 22, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4561, 1533, 6570, 4141, 3606, 4816, 3918, 3623] avg=4096.0000 max=6570 max_rank=2 min=1533 min_rank=1 imbalance_ratio=1.604004 max_load_reduction=26198 max_load_reduction_pct=0.799500 moved_to_echo=15963 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 111, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2260, 0, 0, 890, 0, 0, 0, 0, 0, 0, 0, 0, 2074, 0, 0, 0, 3, 0, 3902, 0, 307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 3984, 228, 0, 0, 0, 0, 0, 0, 191, 0, 1780, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 67, 0, 0, 3084, 331, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4207, 3150, 6286, 4231, 6067, 4866, 477, 3484] avg=4096.0000 max=6286 max_rank=2 min=477 min_rank=6 imbalance_ratio=1.534668 max_load_reduction=26482 max_load_reduction_pct=0.808167 moved_to_echo=14894 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 11, 115, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 265, 0, 0, 2854, 1580, 0, 0, 0, 0, 0, 0, 2, 5, 0, 0, 0, 0, 0, 170, 593, 0, 0, 0, 0, 0, 0, 0, 2, 0, 22, 0, 0, 0, 1, 0, 3977, 224, 0, 0, 0, 0, 0, 0, 0, 49, 2, 2, 81, 8, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1853, 0, 0, 2268, 779, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1708, 29, 0, 257, 3641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3058, 1066, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4177, 4699, 770, 4226, 4236, 4901, 5635, 4124] avg=4096.0000 max=5635 max_rank=6 min=770 min_rank=2 imbalance_ratio=1.375732 max_load_reduction=27133 max_load_reduction_pct=0.828033 moved_to_echo=18896 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 512, 256, 128, 128, 8192, 0, 0, 16000, 8192, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 128, 8960, 3712, 0, 1024, 19456, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 11520, 0, 0, 128, 128, 512, 15744, 3200, 2176, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[24064, 0, 128, 2432, 6144, 128, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[6656, 1152, 12160, 256, 384, 384, 12288, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 128, 2816, 128, 128, 23936, 6016, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 256, 0, 128, 128, 30720, 1792, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 128, 896, 31360, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 77, 4005, 0, 4080, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 60, 4096, 3987, 0, 0, 3, 0, 0, 0, 0, 0, 10, 0, 2821, 0, 61, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1271, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[202, 16, 0, 4081, 0, 4073, 0, 31, 0, 0, 0, 4061, 0, 0, 32, 32, 0, 0, 3477, 0, 0, 0, 0, 1, 0, 2, 46, 0, 3362, 0, 3805, 0, 0, 0, 0, 5, 0, 4, 912, 0, 0, 0, 0, 0, 0, 20, 21, 4055, 108, 1, 0, 0, 4089, 0, 0, 27, 0, 0, 0, 0, 0, 7, 298, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 3940, 0, 4096, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 5, 1, 0, 4056, 128, 0, 0, 0, 0, 0, 44, 0, 0, 2, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 111, 0, 1, 0, 0, 3, 0, 0, 3147, 0, 0, 2, 0, 4035, 11, 2245, 0, 0, 1341, 0, 47, 0, 384, 1026, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 34, 0, 4057, 0, 4003, 0, 5, 0, 0, 0, 2888, 0, 0, 0, 0, 0, 0, 4090, 0, 2, 0, 0, 0, 310, 0, 0, 0, 0, 0, 3919, 0, 143, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 936, 0, 0, 0, 0, 3779, 0, 1740, 79, 0, 4091, 0, 0, 0, 0, 2669, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[107, 0, 1671, 3911, 0, 3983, 0, 0, 0, 0, 0, 3343, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 3341, 0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4028, 0, 0, 0, 11, 4092, 0, 0, 0, 0, 9, 0, 0, 0, 0, 87, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[2111, 67, 0, 3996, 0, 4094, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 105, 0, 4034, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 1813, 0, 0, 0, 0, 0, 0, 6, 1, 0, 0, 93, 0, 0, 0, 0, 0, 3, 0, 3880, 0, 15, 0, 0, 3972, 0, 0, 1, 0, 367, 0, 32, 0, 8, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 2, 4023, 81, 4071, 0, 0, 2, 0, 0, 4061, 0, 0, 15, 0, 0, 0, 4030, 0, 50, 0, 136, 0, 0, 0, 0, 0, 1608, 0, 3850, 0, 1060, 0, 0, 0, 0, 2, 2607, 0, 0, 0, 0, 4, 0, 0, 0, 773, 0, 0, 4, 0, 3450, 29, 15, 35, 0, 958, 0, 0, 0, 1, 1901, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 5, 7, 4092, 0, 4087, 3, 0, 0, 0, 0, 4010, 0, 0, 0, 0, 217, 0, 3467, 859, 0, 1, 0, 0, 0, 0, 0, 0, 304, 0, 4092, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 1, 37, 0, 3981, 0, 0, 0, 0, 3837, 0, 69, 0, 0, 3691, 0, 0, 4, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 77, 3537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 268, 0, 0, 0, 0, 0, 0, 0, 1, 60, 3025, 3987, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 2308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 1, 0, 3485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4096, 468, 335, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3709, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1271, 0, 0, 0, 0, 0, 0, 1071, 595, 178, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3614, 4364, 7076, 2318, 3547, 5025, 3709, 3115] avg=4096.0000 max=7076 max_rank=2 min=2318 min_rank=3 imbalance_ratio=1.727539 max_load_reduction=25692 max_load_reduction_pct=0.784058 moved_to_echo=15396 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[202, 16, 0, 3604, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 32, 32, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2568, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 46, 0, 3362, 0, 3113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4, 912, 0, 3479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 21, 4055, 477, 452, 119, 0, 0, 0, 0, 0, 108, 1, 0, 0, 3702, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 298, 0, 909, 594, 240, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3853, 4393, 2569, 6523, 4400, 5144, 3838, 2048] avg=4096.0000 max=6523 max_rank=3 min=2048 min_rank=7 imbalance_ratio=1.592529 max_load_reduction=26245 max_load_reduction_pct=0.800934 moved_to_echo=15430 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 5, 264, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2995, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 2, 0, 3333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 3499, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 3147, 460, 484, 117, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3654, 11, 2245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1341, 0, 47, 0, 384, 1026, 0, 1061, 597, 258, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3480, 4337, 3124, 3379, 3610, 4212, 5912, 4714] avg=4096.0000 max=5912 max_rank=6 min=3124 min_rank=2 imbalance_ratio=1.443359 max_load_reduction=26856 max_load_reduction_pct=0.819580 moved_to_echo=18448 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 34, 0, 3583, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2888, 0, 0, 0, 0, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3020, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 310, 0, 0, 0, 0, 0, 3205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 1, 0, 0, 0, 0, 0, 3420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 936, 474, 466, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3421, 0, 1740, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 2669, 0, 1070, 583, 248, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3622, 3136, 3022, 3515, 3564, 2008, 5240, 8661] avg=4096.0000 max=8661 max_rank=7 min=2008 min_rank=5 imbalance_ratio=2.114502 max_load_reduction=24107 max_load_reduction_pct=0.735687 moved_to_echo=19473 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2111, 67, 0, 3529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 260, 0, 0, 0, 0, 0, 0, 0, 105, 0, 2979, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 0, 0, 93, 3497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3880, 467, 0, 116, 0, 0, 0, 0, 0, 0, 15, 0, 0, 3596, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 367, 0, 32, 0, 8, 1, 0, 1055, 597, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5707, 4357, 7156, 1813, 3597, 4466, 3612, 2060] avg=4096.0000 max=7156 max_rank=2 min=1813 min_rank=3 imbalance_ratio=1.747070 max_load_reduction=25612 max_load_reduction_pct=0.781616 moved_to_echo=13735 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 3553, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4061, 0, 0, 15, 0, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2976, 0, 50, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1608, 0, 3150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 0, 0, 0, 0, 2, 2607, 0, 3478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 773, 470, 457, 100, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3124, 29, 15, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 958, 0, 0, 0, 1, 1901, 0, 1054, 593, 243, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3636, 4304, 3162, 4758, 7147, 1804, 3207, 4750] avg=4096.0000 max=7147 max_rank=4 min=1804 min_rank=5 imbalance_ratio=1.744873 max_load_reduction=25621 max_load_reduction_pct=0.781891 moved_to_echo=16908 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[107, 0, 1671, 3449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3343, 0, 0, 0, 0, 273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 2726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 3403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4028, 462, 401, 123, 0, 0, 0, 0, 0, 0, 0, 0, 11, 3696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 87, 0, 1074, 580, 214, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5227, 3616, 3022, 6811, 3407, 5014, 3707, 1964] avg=4096.0000 max=6811 max_rank=3 min=1964 min_rank=7 imbalance_ratio=1.662842 max_load_reduction=25957 max_load_reduction_pct=0.792145 moved_to_echo=14092 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 7, 3614, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4010, 0, 0, 0, 0, 251, 0, 0, 0, 0, 0, 0, 0, 217, 0, 2560, 859, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 304, 0, 3347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 3492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 37, 0, 3981, 478, 486, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3474, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3691, 0, 0, 4, 0, 0, 0, 907, 595, 259, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3629, 4261, 3637, 3651, 3496, 5095, 3543, 5456] avg=4096.0000 max=5456 max_rank=7 min=3496 min_rank=4 imbalance_ratio=1.332031 max_load_reduction=27312 max_load_reduction_pct=0.833496 moved_to_echo=17590 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 11776, 0, 128, 128, 512, 6016, 0, 8320, 4736, 1664, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[384, 128, 23168, 9088, 128, 128, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[384, 128, 128, 0, 11264, 0, 21248, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[1280, 128, 128, 128, 128, 128, 3584, 256, 27776, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 128, 128, 128, 128, 24960, 3840, 3200, 1024, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2432, 128, 1792, 28416, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 28416, 128, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 0, 30720, 128, 0, 128, 128, 2176, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33792 slack_slots=31744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True + [2026-09-21 08:24:24] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 268.2 | throughput per GPU (TFLOP/s/GPU): 120.1 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.023791E+01 | loss scale: 1.0 | grad norm: 29.990 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[2486, 776, 232, 26, 27, 813, 928, 59, 78, 1284, 39, 1, 56, 175, 128, 716, 46, 467, 591, 1468, 37, 332, 325, 63, 533, 42, 46, 831, 781, 875, 371, 658, 187, 281, 693, 141, 161, 1290, 290, 837, 623, 1433, 874, 12, 2358, 977, 20, 400, 928, 518, 940, 118, 1161, 774, 387, 53, 718, 14, 582, 99, 6, 328, 235, 40] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[2002, 783, 108, 39, 19, 860, 935, 80, 8, 1454, 36, 1, 47, 65, 52, 675, 65, 530, 343, 1597, 38, 437, 400, 118, 705, 55, 47, 889, 836, 1099, 176, 522, 242, 347, 502, 365, 172, 1482, 284, 637, 505, 1057, 877, 5, 1803, 1243, 22, 462, 1035, 471, 735, 107, 1289, 1073, 691, 246, 772, 2, 553, 42, 42, 259, 358, 67] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1855, 751, 132, 81, 44, 443, 1028, 115, 9, 1656, 103, 0, 43, 201, 99, 765, 55, 348, 523, 1550, 19, 634, 366, 114, 812, 43, 104, 789, 456, 738, 364, 863, 222, 306, 666, 448, 296, 1759, 236, 787, 468, 1393, 583, 19, 2154, 1023, 26, 504, 1099, 585, 411, 27, 1100, 621, 351, 91, 1005, 39, 582, 53, 15, 111, 624, 61] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1937, 741, 118, 81, 29, 689, 967, 80, 157, 966, 64, 1, 29, 163, 101, 919, 108, 486, 558, 1407, 41, 436, 286, 35, 859, 65, 126, 885, 466, 1132, 241, 808, 107, 245, 654, 262, 173, 1417, 268, 793, 471, 1637, 541, 24, 2335, 1071, 43, 389, 1026, 530, 825, 43, 937, 825, 466, 116, 1316, 10, 618, 18, 5, 69, 469, 84] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1948, 621, 191, 33, 42, 1022, 894, 128, 149, 1336, 21, 1, 31, 229, 177, 296, 87, 630, 861, 1866, 55, 378, 293, 133, 634, 112, 45, 465, 598, 837, 316, 398, 339, 261, 795, 453, 188, 1703, 197, 871, 315, 835, 665, 9, 1416, 891, 23, 546, 1072, 382, 1192, 116, 979, 669, 1224, 166, 1177, 41, 593, 11, 82, 227, 418, 85] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1840, 737, 193, 54, 18, 752, 1438, 79, 13, 1924, 23, 1, 36, 177, 278, 524, 68, 333, 515, 1172, 69, 426, 490, 40, 715, 177, 20, 732, 513, 760, 208, 639, 310, 366, 583, 318, 300, 1196, 222, 816, 514, 806, 993, 19, 2137, 1100, 30, 539, 644, 592, 849, 153, 1125, 837, 726, 48, 1067, 6, 709, 30, 20, 159, 564, 26] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1787, 708, 181, 26, 25, 722, 1229, 75, 20, 1374, 94, 3, 16, 145, 138, 614, 135, 268, 755, 1237, 59, 442, 404, 84, 577, 125, 25, 676, 551, 990, 281, 728, 306, 355, 503, 323, 215, 1841, 510, 629, 364, 1501, 627, 17, 1816, 1297, 41, 448, 802, 861, 905, 85, 931, 732, 613, 131, 880, 2, 511, 88, 64, 119, 704, 53] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1503, 709, 86, 60, 34, 952, 1025, 207, 54, 881, 63, 12, 35, 109, 174, 527, 40, 428, 555, 1817, 23, 703, 276, 38, 982, 48, 67, 324, 526, 1648, 409, 623, 207, 243, 655, 163, 255, 1729, 238, 438, 476, 1183, 800, 26, 2072, 1270, 37, 702, 904, 494, 954, 116, 1102, 736, 618, 148, 966, 28, 756, 111, 18, 123, 166, 96] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1546, 776, 232, 26, 27, 813, 928, 59, 0, 0, 0, 0, 0, 0, 0, 0, 78, 1284, 39, 1, 56, 175, 128, 716, 1038, 496, 354, 0, 0, 0, 0, 0, 46, 467, 591, 1468, 37, 332, 325, 63, 444, 136, 21, 1, 0, 0, 0, 0, 533, 42, 46, 831, 781, 854, 371, 658, 0, 0, 0, 0, 0, 0, 0, 0, 187, 281, 693, 141, 161, 1154, 290, 837, 0, 0, 0, 0, 0, 0, 0, 0, 623, 1432, 874, 12, 0, 977, 20, 400, 0, 0, 0, 0, 0, 0, 0, 0, 928, 518, 940, 118, 123, 774, 387, 53, 0, 0, 0, 0, 0, 0, 0, 0, 718, 14, 582, 99, 6, 328, 235, 40, 2004, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4407, 4365, 3931, 4116, 3744, 4338, 3841, 4026] avg=4096.0000 max=4407 max_rank=0 min=3744 min_rank=4 imbalance_ratio=1.075928 max_load_reduction=28361 max_load_reduction_pct=0.865509 moved_to_echo=15949 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1153, 751, 132, 81, 44, 443, 1028, 115, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1656, 103, 0, 43, 201, 99, 765, 984, 370, 323, 0, 0, 0, 0, 0, 55, 348, 523, 1550, 19, 634, 366, 114, 332, 185, 18, 1, 0, 0, 0, 0, 812, 43, 104, 789, 456, 720, 364, 863, 0, 0, 0, 0, 0, 0, 0, 0, 222, 306, 666, 448, 296, 1574, 236, 787, 0, 0, 0, 0, 0, 0, 0, 0, 468, 1392, 583, 19, 0, 1023, 26, 504, 0, 0, 0, 0, 0, 0, 0, 0, 1099, 585, 411, 27, 116, 621, 351, 91, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 39, 582, 53, 15, 111, 624, 61, 1831, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3747, 4553, 4145, 4151, 4535, 4015, 3301, 4321] avg=4096.0000 max=4553 max_rank=1 min=3301 min_rank=6 imbalance_ratio=1.111572 max_load_reduction=28215 max_load_reduction_pct=0.861053 moved_to_echo=16172 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1245, 783, 108, 39, 19, 860, 935, 80, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1454, 36, 1, 47, 65, 52, 675, 1153, 399, 271, 0, 0, 0, 0, 0, 65, 530, 343, 1597, 38, 437, 400, 118, 358, 156, 26, 0, 0, 0, 0, 0, 705, 55, 47, 889, 836, 1073, 176, 522, 0, 0, 0, 0, 0, 0, 0, 0, 242, 347, 502, 365, 172, 1326, 284, 637, 0, 0, 0, 0, 0, 0, 0, 0, 505, 1057, 877, 5, 0, 1243, 22, 462, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 471, 735, 107, 136, 1073, 691, 246, 0, 0, 0, 0, 0, 0, 0, 0, 772, 2, 553, 42, 42, 259, 358, 67, 1532, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4161, 4068, 4303, 3875, 4171, 4494, 3627] avg=4096.0000 max=4494 max_rank=6 min=3627 min_rank=7 imbalance_ratio=1.097168 max_load_reduction=28274 max_load_reduction_pct=0.862854 moved_to_echo=16167 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1205, 741, 118, 81, 29, 689, 967, 80, 0, 0, 0, 0, 0, 0, 0, 0, 157, 966, 64, 1, 29, 163, 101, 919, 838, 386, 351, 0, 0, 0, 0, 0, 108, 486, 558, 1407, 41, 436, 286, 35, 346, 149, 27, 1, 0, 0, 0, 0, 859, 65, 126, 885, 466, 1105, 241, 808, 0, 0, 0, 0, 0, 0, 0, 0, 107, 245, 654, 262, 173, 1268, 268, 793, 0, 0, 0, 0, 0, 0, 0, 0, 471, 1636, 541, 24, 0, 1071, 43, 389, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 530, 825, 43, 99, 825, 466, 116, 0, 0, 0, 0, 0, 0, 0, 0, 1316, 10, 618, 18, 5, 69, 469, 84, 1984, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3910, 3975, 3880, 4555, 3770, 4175, 3930, 4573] avg=4096.0000 max=4573 max_rank=7 min=3770 min_rank=4 imbalance_ratio=1.116455 max_load_reduction=28195 max_load_reduction_pct=0.860443 moved_to_echo=16448 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1212, 621, 191, 33, 42, 1022, 894, 128, 0, 0, 0, 0, 0, 0, 0, 0, 149, 1336, 21, 1, 31, 229, 177, 296, 876, 388, 213, 0, 0, 0, 0, 0, 87, 630, 861, 1866, 55, 378, 293, 133, 348, 179, 20, 0, 0, 0, 0, 0, 634, 112, 45, 465, 598, 817, 316, 398, 0, 0, 0, 0, 0, 0, 0, 0, 339, 261, 795, 453, 188, 1524, 197, 871, 0, 0, 0, 0, 0, 0, 0, 0, 315, 835, 665, 9, 0, 891, 23, 546, 0, 0, 0, 0, 0, 0, 0, 0, 1072, 382, 1192, 116, 103, 669, 1224, 166, 0, 0, 0, 0, 0, 0, 0, 0, 1177, 41, 593, 11, 82, 227, 418, 85, 1203, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4143, 3717, 4850, 3385, 4628, 3284, 4924, 3837] avg=4096.0000 max=4924 max_rank=6 min=3284 min_rank=5 imbalance_ratio=1.202148 max_load_reduction=27844 max_load_reduction_pct=0.849731 moved_to_echo=16673 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1144, 737, 193, 54, 18, 752, 1438, 79, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1924, 23, 1, 36, 177, 278, 524, 1006, 367, 322, 0, 0, 0, 0, 0, 68, 333, 515, 1172, 69, 426, 490, 40, 329, 126, 18, 0, 0, 0, 0, 0, 715, 177, 20, 732, 513, 742, 208, 639, 0, 0, 0, 0, 0, 0, 0, 0, 310, 366, 583, 318, 300, 1070, 222, 816, 0, 0, 0, 0, 0, 0, 0, 0, 514, 806, 993, 19, 0, 1100, 30, 539, 0, 0, 0, 0, 0, 0, 0, 0, 644, 592, 849, 153, 119, 837, 726, 48, 0, 0, 0, 0, 0, 0, 0, 0, 1067, 6, 709, 30, 20, 159, 564, 26, 1815, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4415, 4671, 3586, 3746, 3985, 4001, 3968, 4396] avg=4096.0000 max=4671 max_rank=1 min=3586 min_rank=2 imbalance_ratio=1.140381 max_load_reduction=28097 max_load_reduction_pct=0.857452 moved_to_echo=16350 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1104, 708, 181, 26, 25, 722, 1229, 75, 0, 0, 0, 0, 0, 0, 0, 0, 20, 1374, 94, 3, 16, 145, 138, 614, 836, 360, 274, 0, 0, 0, 0, 0, 135, 268, 755, 1237, 59, 442, 404, 84, 323, 197, 28, 5, 0, 0, 0, 0, 577, 125, 25, 676, 551, 962, 281, 728, 0, 0, 0, 0, 0, 0, 0, 0, 306, 355, 503, 323, 215, 1644, 510, 629, 0, 0, 0, 0, 0, 0, 0, 0, 364, 1496, 627, 17, 0, 1297, 41, 448, 0, 0, 0, 0, 0, 0, 0, 0, 802, 861, 905, 85, 95, 732, 613, 131, 0, 0, 0, 0, 0, 0, 0, 0, 880, 2, 511, 88, 64, 119, 704, 53, 1542, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4070, 3874, 3937, 3925, 4485, 4290, 4224, 3963] avg=4096.0000 max=4485 max_rank=4 min=3874 min_rank=1 imbalance_ratio=1.094971 max_load_reduction=28283 max_load_reduction_pct=0.863129 moved_to_echo=16962 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[934, 709, 86, 60, 34, 952, 1025, 207, 0, 0, 0, 0, 0, 0, 0, 0, 54, 881, 63, 12, 35, 109, 174, 527, 986, 300, 311, 0, 0, 0, 0, 0, 40, 428, 555, 1817, 23, 703, 276, 38, 269, 182, 40, 0, 0, 0, 0, 0, 982, 48, 67, 324, 526, 1608, 409, 623, 0, 0, 0, 0, 0, 0, 0, 0, 207, 243, 655, 163, 255, 1547, 238, 438, 0, 0, 0, 0, 0, 0, 0, 0, 476, 1183, 800, 26, 0, 1270, 37, 702, 0, 0, 0, 0, 0, 0, 0, 0, 904, 494, 954, 116, 116, 736, 618, 148, 0, 0, 0, 0, 0, 0, 0, 0, 966, 28, 756, 111, 18, 123, 166, 96, 1761, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4007, 3452, 4371, 4587, 3746, 4494, 4086, 4025] avg=4096.0000 max=4587 max_rank=3 min=3452 min_rank=1 imbalance_ratio=1.119873 max_load_reduction=28181 max_load_reduction_pct=0.860016 moved_to_echo=16351 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[9600, 5888, 1280, 512, 256, 6272, 8448, 896, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[5888, 768, 512, 5632, 4736, 7936, 2432, 5248, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1920, 2432, 5120, 2560, 1792, 11136, 2304, 5888, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[7552, 4480, 6912, 768, 1024, 6272, 5120, 1024, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[640, 3584, 4736, 12160, 384, 3840, 2944, 640, 2816, 1408, 256, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[512, 10880, 512, 128, 384, 1280, 1152, 5120, 7808, 3072, 2432, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3840, 9856, 6016, 256, 0, 8960, 256, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[7936, 256, 4992, 512, 256, 1408, 3584, 512, 13696, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 759, 376, 0, 30, 6, 287, 5, 0, 60, 0, 2, 4014, 1, 3, 0, 3891, 1, 48, 0, 0, 1434, 0, 3488, 0, 0, 0, 4, 1000, 0, 0, 1, 3653, 0, 54, 0, 0, 0, 0, 580, 32, 1096, 1331, 26, 4079, 0, 1, 8, 2723, 263, 3, 0, 0, 37, 0, 3363, 52, 0, 26, 30, 0, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[8, 375, 36, 0, 0, 0, 3, 0, 8, 0, 0, 24, 4056, 0, 0, 0, 3081, 0, 0, 0, 3, 1853, 1, 3957, 0, 0, 0, 0, 1911, 0, 0, 11, 4045, 0, 1, 0, 0, 0, 4, 1023, 39, 1041, 564, 0, 4026, 0, 13, 0, 2519, 40, 0, 25, 0, 0, 0, 4081, 0, 0, 3, 0, 2, 0, 15, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1, 213, 247, 0, 0, 63, 952, 0, 0, 6, 0, 0, 4067, 1, 1, 0, 1117, 5, 0, 0, 0, 3306, 0, 3328, 4, 162, 0, 1, 1962, 0, 8, 7, 2511, 0, 23, 1, 0, 0, 124, 1759, 173, 2906, 18, 0, 3965, 0, 215, 5, 312, 1185, 34, 3, 0, 0, 0, 3897, 0, 0, 14, 0, 172, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[6, 1797, 101, 0, 0, 0, 0, 0, 28, 0, 2, 40, 3906, 0, 0, 7, 3786, 5, 5, 0, 0, 520, 9, 1994, 1, 8, 0, 17, 3805, 0, 1, 4, 2687, 6, 13, 0, 3, 0, 320, 870, 12, 3043, 61, 4, 4089, 0, 0, 1, 1147, 633, 1, 17, 0, 0, 0, 3758, 32, 0, 27, 0, 2, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[35, 2080, 2294, 0, 0, 0, 123, 1, 191, 1, 32, 0, 3977, 7, 1, 1, 2629, 0, 117, 5, 7, 857, 12, 4019, 3, 12, 1, 0, 1083, 25, 0, 4, 3613, 0, 1, 0, 0, 0, 146, 1013, 16, 734, 112, 0, 4080, 1, 0, 1, 857, 520, 0, 1, 0, 2, 0, 4086, 1, 0, 50, 3, 2, 0, 12, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 3174, 134, 0, 0, 0, 15, 0, 0, 0, 57, 0, 3993, 0, 31, 0, 2405, 0, 5, 0, 0, 2924, 7, 822, 0, 0, 0, 0, 3379, 0, 0, 2, 1006, 0, 0, 0, 0, 0, 399, 1329, 15, 3195, 1, 0, 4091, 0, 13, 0, 2137, 1562, 0, 4, 0, 2, 0, 2055, 8, 0, 0, 0, 3, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 246, 321, 0, 0, 1, 469, 0, 0, 0, 107, 2, 3607, 0, 97, 0, 3076, 11, 22, 35, 0, 1372, 0, 1015, 0, 131, 1, 2, 2970, 0, 5, 80, 4095, 0, 2, 0, 0, 1, 126, 977, 3, 2499, 193, 2, 3927, 0, 1, 0, 2050, 1197, 44, 3, 0, 0, 1, 4054, 0, 0, 14, 0, 0, 0, 9, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[24, 2205, 11, 6, 22, 0, 106, 0, 2, 20, 2, 0, 3745, 1, 3, 0, 2004, 0, 104, 0, 5, 3201, 0, 2418, 0, 361, 31, 35, 2151, 0, 3, 145, 3583, 0, 7, 0, 0, 0, 183, 864, 0, 722, 426, 0, 3972, 0, 0, 0, 1452, 976, 0, 1187, 2, 0, 0, 2646, 0, 0, 91, 0, 50, 0, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 759, 376, 0, 30, 6, 287, 5, 1012, 990, 0, 0, 0, 0, 0, 0, 0, 60, 0, 2, 4014, 1, 3, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 1, 48, 0, 0, 1434, 0, 2801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1000, 0, 0, 1, 926, 687, 230, 0, 0, 0, 0, 0, 3327, 0, 54, 0, 0, 0, 0, 580, 0, 0, 0, 0, 0, 0, 0, 0, 32, 1096, 1331, 26, 1804, 0, 1, 8, 0, 0, 0, 0, 0, 0, 0, 0, 2723, 263, 3, 0, 0, 37, 0, 1425, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 26, 30, 0, 0, 1, 0, 3891, 1285, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3465, 4176, 4284, 2848, 3961, 4298, 4451, 5285] avg=4096.0000 max=5285 max_rank=7 min=2848 min_rank=3 imbalance_ratio=1.290283 max_load_reduction=27483 max_load_reduction_pct=0.838715 moved_to_echo=17995 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8, 375, 36, 0, 0, 0, 3, 0, 1228, 977, 0, 0, 0, 0, 0, 0, 8, 0, 0, 24, 4056, 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1853, 1, 3178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1911, 0, 0, 11, 1124, 779, 255, 0, 0, 0, 0, 0, 3684, 0, 1, 0, 0, 0, 4, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 39, 1041, 564, 0, 1781, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2519, 40, 0, 25, 0, 0, 0, 1729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 15, 0, 3081, 1268, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2627, 4194, 5035, 4080, 4712, 3438, 4313, 4369] avg=4096.0000 max=5035 max_rank=2 min=2627 min_rank=0 imbalance_ratio=1.229248 max_load_reduction=27733 max_load_reduction_pct=0.846344 moved_to_echo=16832 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[6, 1797, 101, 0, 0, 0, 0, 0, 1131, 992, 0, 0, 0, 0, 0, 0, 28, 0, 2, 40, 3906, 0, 0, 7, 70, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 0, 0, 520, 9, 1601, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8, 0, 17, 3805, 0, 1, 4, 1035, 393, 169, 0, 0, 0, 0, 0, 2448, 6, 13, 0, 3, 0, 320, 870, 0, 0, 0, 0, 0, 0, 0, 0, 12, 3043, 61, 4, 1809, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1147, 633, 1, 17, 0, 0, 0, 1592, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 27, 0, 2, 0, 0, 0, 3786, 1288, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4027, 4053, 2140, 5433, 3660, 4930, 3390, 5135] avg=4096.0000 max=5433 max_rank=3 min=2140 min_rank=2 imbalance_ratio=1.326416 max_load_reduction=27335 max_load_reduction_pct=0.834198 moved_to_echo=17115 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 213, 247, 0, 0, 63, 952, 0, 1173, 962, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4067, 1, 1, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 3306, 0, 2673, 0, 0, 0, 0, 0, 0, 0, 0, 4, 162, 0, 1, 1962, 0, 8, 7, 1074, 655, 158, 0, 0, 0, 0, 0, 2287, 0, 23, 1, 0, 0, 124, 1759, 0, 0, 0, 0, 0, 0, 0, 0, 173, 2906, 18, 0, 1754, 0, 215, 5, 0, 0, 0, 0, 0, 0, 0, 0, 312, 1185, 34, 3, 0, 0, 0, 1650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 172, 0, 0, 0, 1117, 1249, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3611, 4141, 5984, 4031, 4194, 5071, 3184, 2552] avg=4096.0000 max=5984 max_rank=2 min=2552 min_rank=7 imbalance_ratio=1.460938 max_load_reduction=26784 max_load_reduction_pct=0.817383 moved_to_echo=15001 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 3174, 134, 0, 0, 0, 15, 0, 618, 993, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 3993, 0, 31, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 2924, 7, 660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3379, 0, 0, 2, 566, 162, 63, 0, 0, 0, 0, 0, 917, 0, 0, 0, 0, 0, 399, 1329, 0, 0, 0, 0, 0, 0, 0, 0, 15, 3195, 1, 0, 1809, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2137, 1562, 0, 4, 0, 2, 0, 871, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 3, 0, 0, 0, 2405, 1289, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4934, 4107, 3596, 4172, 2645, 5033, 4576, 3705] avg=4096.0000 max=5033 max_rank=5 min=2645 min_rank=4 imbalance_ratio=1.228760 max_load_reduction=27735 max_load_reduction_pct=0.846405 moved_to_echo=15959 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 246, 321, 0, 0, 1, 469, 0, 1220, 953, 0, 0, 0, 0, 0, 0, 0, 0, 107, 2, 3607, 0, 97, 0, 108, 0, 0, 0, 0, 0, 0, 0, 0, 11, 22, 35, 0, 1372, 0, 815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 1, 2, 2970, 0, 5, 80, 1117, 200, 258, 0, 0, 0, 0, 0, 3729, 0, 2, 0, 0, 1, 126, 977, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2499, 193, 2, 1737, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2050, 1197, 44, 3, 0, 0, 1, 1717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 9, 0, 3076, 1237, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3210, 3921, 2255, 4764, 4835, 4435, 5012, 4336] avg=4096.0000 max=5012 max_rank=6 min=2255 min_rank=2 imbalance_ratio=1.223633 max_load_reduction=27756 max_load_reduction_pct=0.847046 moved_to_echo=18618 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[35, 2080, 2294, 0, 0, 0, 123, 1, 1232, 995, 0, 0, 0, 0, 0, 0, 191, 1, 32, 0, 3977, 7, 1, 1, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 5, 7, 857, 12, 3224, 0, 0, 0, 0, 0, 0, 0, 0, 3, 12, 1, 0, 1083, 25, 0, 4, 1130, 795, 231, 0, 0, 0, 0, 0, 3283, 0, 1, 0, 0, 0, 146, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 16, 734, 112, 0, 1794, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 857, 520, 0, 1, 0, 2, 0, 1724, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 50, 3, 2, 0, 12, 0, 2629, 1291, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6760, 4309, 4222, 3284, 4443, 2658, 3104, 3988] avg=4096.0000 max=6760 max_rank=0 min=2658 min_rank=5 imbalance_ratio=1.650391 max_load_reduction=26008 max_load_reduction_pct=0.793701 moved_to_echo=14193 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[24, 2205, 11, 6, 22, 0, 106, 0, 796, 964, 0, 0, 0, 0, 0, 0, 2, 20, 2, 0, 3745, 1, 3, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 5, 3201, 0, 1942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 361, 31, 35, 2151, 0, 3, 145, 729, 476, 225, 0, 0, 0, 0, 0, 3264, 0, 7, 0, 0, 0, 183, 864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 722, 426, 0, 1757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1452, 976, 0, 1187, 2, 0, 0, 1121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 50, 0, 2, 0, 2004, 1251, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4134, 3867, 5252, 4156, 4318, 2905, 4738, 3398] avg=4096.0000 max=5252 max_rank=2 min=2905 min_rank=5 imbalance_ratio=1.282227 max_load_reduction=27516 max_load_reduction_pct=0.839722 moved_to_echo=15359 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 10880, 3584, 128, 128, 128, 2048, 128, 8448, 7936, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[13312, 6400, 128, 1280, 128, 128, 128, 11904, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[384, 15360, 2816, 128, 14336, 128, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 256, 128, 256, 0, 128, 0, 22016, 10240, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 384, 128, 128, 15488, 128, 16896, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 768, 128, 128, 18304, 128, 128, 256, 7808, 4224, 1664, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[23040, 128, 128, 128, 128, 128, 1408, 8448, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[256, 128, 256, 128, 31488, 128, 256, 128, 768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1919, 80, 5, 0, 0, 0, 0, 106, 3108, 0, 3567, 1, 0, 0, 0, 0, 0, 111, 0, 4096, 131, 0, 1, 0, 0, 6, 0, 3706, 0, 0, 0, 0, 0, 0, 225, 0, 3956, 6, 0, 2, 3590, 0, 3902, 0, 0, 96, 1, 0, 423, 0, 0, 0, 0, 3730, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 1, 2, 4, 15, 0, 71, 0, 901, 0, 9, 0, 2, 610, 3816, 74, 2974, 0, 0, 0, 0, 0, 0, 699, 0, 4078, 3216, 0, 0, 0, 0, 33, 121, 3838, 35, 44, 0, 0, 0, 0, 44, 112, 4096, 2, 0, 59, 520, 0, 3967, 1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 3417, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[21, 1, 0, 0, 97, 0, 0, 0, 38, 0, 0, 236, 5, 0, 0, 0, 8, 3862, 7, 4005, 0, 0, 0, 0, 0, 0, 32, 1, 3706, 144, 0, 95, 0, 2, 0, 0, 4086, 2, 0, 0, 0, 0, 0, 144, 20, 3586, 0, 1, 0, 3922, 4, 4031, 45, 0, 27, 15, 34, 486, 0, 6, 0, 15, 4084, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[197, 1, 0, 0, 0, 0, 0, 5, 0, 917, 0, 1403, 4, 0, 0, 0, 2, 3394, 0, 4065, 1028, 0, 1, 0, 0, 0, 122, 0, 1963, 224, 0, 6, 0, 0, 371, 0, 4083, 0, 3, 0, 0, 0, 0, 5, 4, 3951, 0, 0, 4, 2690, 0, 4053, 0, 0, 1, 178, 0, 3, 0, 0, 0, 0, 4090, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[14, 0, 6, 1, 0, 0, 0, 2, 1, 926, 0, 22, 0, 0, 0, 0, 313, 2719, 4, 1548, 73, 0, 0, 0, 0, 1, 40, 0, 4080, 2295, 0, 0, 0, 0, 823, 4, 3812, 0, 13, 1, 0, 0, 0, 4, 0, 2721, 0, 0, 2, 3185, 0, 4083, 0, 1, 25, 1805, 0, 149, 0, 0, 0, 0, 4095, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[234, 0, 0, 0, 27, 0, 0, 0, 0, 734, 0, 57, 0, 0, 0, 0, 14, 3737, 25, 81, 149, 0, 0, 0, 0, 0, 24, 0, 3903, 2314, 0, 0, 6, 0, 604, 0, 4014, 0, 36, 0, 0, 0, 0, 1528, 1, 3323, 2, 0, 0, 3799, 0, 4057, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 4095, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1889, 0, 68, 0, 3, 0, 0, 0, 52, 2400, 0, 0, 27, 0, 11, 14, 78, 2484, 16, 1743, 1120, 1, 0, 0, 0, 0, 0, 0, 1473, 222, 1, 79, 1, 0, 818, 2, 4077, 1, 105, 3, 0, 0, 0, 0, 8, 3974, 8, 0, 0, 4056, 0, 3976, 0, 28, 5, 14, 0, 0, 0, 0, 0, 0, 4011, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[17, 0, 112, 0, 0, 0, 3, 4, 238, 830, 0, 695, 0, 0, 0, 0, 0, 1428, 18, 804, 0, 0, 0, 1, 0, 0, 0, 0, 3503, 1402, 0, 745, 2, 15, 0, 4, 4076, 0, 139, 11, 0, 0, 0, 407, 0, 3926, 2146, 0, 32, 2374, 14, 4076, 0, 0, 0, 1964, 4, 0, 25, 0, 0, 0, 3750, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 1, 2, 4, 15, 3435, 342, 0, 0, 0, 0, 0, 0, 0, 71, 0, 901, 0, 9, 0, 2, 1867, 903, 257, 150, 0, 0, 0, 0, 610, 1607, 74, 2974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 699, 0, 3175, 3216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 121, 3581, 35, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 112, 3899, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 520, 0, 532, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3417, 0, 47, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3804, 4160, 5265, 7090, 3814, 4057, 1113, 3465] avg=4096.0000 max=7090 max_rank=3 min=1113 min_rank=6 imbalance_ratio=1.730957 max_load_reduction=25678 max_load_reduction_pct=0.783630 moved_to_echo=12449 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 3379, 278, 0, 0, 0, 0, 0, 0, 0, 1919, 80, 5, 0, 0, 0, 0, 1521, 907, 248, 145, 0, 0, 0, 0, 106, 1309, 0, 3567, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 0, 3189, 131, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 3458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 225, 0, 3765, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3590, 0, 523, 0, 0, 96, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 423, 0, 0, 0, 0, 3730, 0, 46, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3657, 4825, 4983, 3432, 3464, 3996, 4212, 4199] avg=4096.0000 max=4983 max_rank=2 min=3432 min_rank=3 imbalance_ratio=1.216553 max_load_reduction=27785 max_load_reduction_pct=0.847931 moved_to_echo=15871 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[21, 1, 0, 0, 97, 0, 0, 0, 3490, 346, 0, 0, 0, 0, 0, 0, 38, 0, 0, 236, 5, 0, 0, 0, 1890, 821, 274, 132, 0, 0, 0, 0, 8, 1626, 7, 4005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 1, 2885, 144, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3812, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 20, 3413, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3922, 4, 541, 45, 0, 27, 15, 0, 0, 0, 0, 0, 0, 0, 0, 34, 486, 0, 6, 0, 15, 4084, 0, 41, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3955, 3396, 5646, 3157, 3816, 3578, 4554, 4666] avg=4096.0000 max=5646 max_rank=2 min=3157 min_rank=3 imbalance_ratio=1.378418 max_load_reduction=27122 max_load_reduction_pct=0.827698 moved_to_echo=16614 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[197, 1, 0, 0, 0, 0, 0, 5, 3509, 304, 0, 0, 0, 0, 0, 0, 0, 917, 0, 1403, 4, 0, 0, 0, 1661, 434, 274, 145, 0, 0, 0, 0, 2, 1429, 0, 4065, 1028, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 0, 1529, 224, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 371, 0, 3809, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4, 3760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2690, 0, 544, 0, 0, 1, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4090, 0, 46, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4016, 4838, 6525, 1881, 4183, 3769, 3417, 4139] avg=4096.0000 max=6525 max_rank=2 min=1881 min_rank=3 imbalance_ratio=1.593018 max_load_reduction=26243 max_load_reduction_pct=0.800873 moved_to_echo=15508 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1889, 0, 68, 0, 3, 0, 0, 0, 3443, 222, 0, 0, 0, 0, 0, 0, 52, 2400, 0, 0, 27, 0, 11, 14, 1215, 326, 273, 146, 0, 0, 0, 0, 78, 1047, 16, 1743, 1120, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1147, 222, 1, 79, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 818, 2, 3804, 1, 105, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3782, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 0, 533, 0, 28, 5, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4011, 0, 46, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5625, 4464, 4005, 1449, 4734, 3798, 4636, 4057] avg=4096.0000 max=5625 max_rank=0 min=1449 min_rank=3 imbalance_ratio=1.373291 max_load_reduction=27143 max_load_reduction_pct=0.828339 moved_to_echo=17225 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[234, 0, 0, 0, 27, 0, 0, 0, 3513, 335, 0, 0, 0, 0, 0, 0, 0, 734, 0, 57, 0, 0, 0, 0, 1829, 864, 269, 122, 0, 0, 0, 0, 14, 1573, 25, 81, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 3039, 2314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 604, 0, 3745, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1528, 1, 3163, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3799, 0, 544, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4095, 0, 38, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 3875, 1842, 5377, 4391, 4694, 4344, 4136] avg=4096.0000 max=5377 max_rank=3 min=1842 min_rank=2 imbalance_ratio=1.312744 max_load_reduction=27391 max_load_reduction_pct=0.835907 moved_to_echo=17565 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[14, 0, 6, 1, 0, 0, 0, 2, 3540, 246, 0, 0, 0, 0, 0, 0, 1, 926, 0, 22, 0, 0, 0, 0, 1334, 908, 261, 104, 0, 0, 0, 0, 313, 1139, 4, 1548, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 40, 0, 3172, 2295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 823, 4, 3551, 0, 13, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3185, 0, 543, 0, 1, 25, 1805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 0, 0, 0, 0, 4095, 0, 35, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3809, 3556, 3077, 5508, 4392, 2586, 5561, 4279] avg=4096.0000 max=5561 max_rank=6 min=2586 min_rank=5 imbalance_ratio=1.357666 max_load_reduction=27207 max_load_reduction_pct=0.830292 moved_to_echo=16818 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[17, 0, 112, 0, 0, 0, 3, 4, 3529, 128, 0, 0, 0, 0, 0, 0, 238, 830, 0, 695, 0, 0, 0, 0, 698, 776, 273, 144, 0, 0, 0, 0, 0, 602, 18, 804, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2727, 1402, 0, 745, 0, 0, 0, 0, 0, 0, 0, 0, 2, 15, 0, 4, 3803, 0, 139, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 407, 0, 3737, 2146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 2374, 14, 547, 0, 0, 0, 1964, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 25, 0, 0, 0, 3750, 3, 45, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3793, 3654, 1425, 4874, 3974, 6290, 4931, 3827] avg=4096.0000 max=6290 max_rank=5 min=1425 min_rank=2 imbalance_ratio=1.535645 max_load_reduction=26478 max_load_reduction_pct=0.808044 moved_to_echo=19022 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1152, 10368, 256, 18816, 2432, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[384, 7808, 128, 3328, 128, 128, 128, 128, 12032, 6016, 2176, 1152, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 1152, 128, 20864, 9984, 128, 1024, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 24192, 128, 4352, 128, 128, 256, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 0, 2432, 256, 28160, 2176, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[2432, 128, 256, 128, 128, 128, 128, 128, 27904, 2304, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 2688, 256, 29568, 128, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 1152, 128, 128, 0, 128, 31360, 128, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[1, 0, 52, 0, 3930, 2, 4093, 3999, 0, 20, 161, 0, 0, 0, 0, 0, 4064, 0, 911, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 66, 6, 4, 186, 1, 0, 2940, 0, 0, 2, 4032, 57, 1, 0, 8, 0, 0, 2, 0, 0, 39, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4086, 5, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 618, 2, 4094, 977, 4086, 4052, 0, 60, 0, 0, 0, 1, 0, 7, 4083, 0, 1, 4, 0, 0, 1, 0, 6, 0, 0, 17, 0, 704, 0, 16, 3708, 0, 0, 1467, 0, 1, 0, 4013, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 711, 4064, 28, 0, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[18, 1, 0, 0, 4095, 0, 4091, 3810, 0, 0, 41, 0, 1, 0, 0, 2, 4096, 0, 0, 0, 51, 0, 0, 66, 0, 2, 0, 0, 0, 7, 1, 0, 3361, 0, 0, 3537, 0, 1, 0, 4054, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1432, 4069, 2, 0, 24] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 434, 0, 4096, 1, 4064, 4071, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 8, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1270, 0, 0, 3329, 9, 0, 816, 0, 0, 0, 1479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2073, 3986, 3031, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[2, 0, 4, 0, 4095, 168, 4092, 3910, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 9, 0, 0, 0, 0, 2, 1407, 0, 10, 0, 1, 250, 0, 0, 119, 1, 0, 3434, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 807, 4094, 2163, 0, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4081, 25, 4095, 2373, 0, 0, 0, 0, 18, 4, 0, 18, 4086, 0, 0, 0, 37, 0, 5, 4, 33, 0, 0, 0, 0, 0, 0, 0, 3687, 3, 0, 2060, 0, 0, 0, 4067, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 3541, 3846, 401, 0, 376] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 2, 0, 1, 4053, 462, 3932, 3858, 0, 35, 2, 0, 1, 0, 0, 0, 4094, 0, 0, 0, 5, 14, 0, 6, 54, 0, 9, 0, 38, 430, 0, 2, 4060, 0, 0, 2870, 0, 0, 0, 3487, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2, 0, 0, 0, 189, 4095, 63, 1, 997] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[5, 0, 124, 0, 4090, 56, 4096, 3998, 2, 0, 0, 0, 0, 5, 0, 0, 3923, 0, 0, 0, 130, 0, 0, 0, 2390, 0, 0, 0, 0, 2891, 0, 7, 2076, 7, 0, 621, 0, 0, 0, 2772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4, 0, 1, 0, 0, 0, 0, 2192, 3209, 143, 0, 21] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 52, 0, 0, 2, 0, 3965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 161, 0, 0, 0, 0, 0, 3993, 605, 0, 0, 0, 0, 0, 0, 3937, 0, 911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 66, 6, 4, 2121, 50, 127, 34, 0, 0, 0, 0, 136, 1, 0, 2940, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 1, 0, 8, 0, 0, 2, 0, 4093, 18, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 3912, 39, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 1360, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4020, 4779, 4848, 2411, 3079, 4179, 3990, 5462] avg=4096.0000 max=5462 max_rank=7 min=2411 min_rank=3 imbalance_ratio=1.333496 max_load_reduction=27306 max_load_reduction_pct=0.833313 moved_to_echo=16710 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 434, 0, 0, 1, 0, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1464, 590, 0, 0, 0, 0, 0, 0, 3968, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1270, 0, 0, 2069, 896, 128, 35, 0, 0, 0, 0, 2433, 9, 0, 816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2073, 1327, 3031, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4471, 2055, 3976, 4401, 3258, 4081, 4094, 6432] avg=4096.0000 max=6432 max_rank=7 min=2055 min_rank=1 imbalance_ratio=1.570312 max_load_reduction=26336 max_load_reduction_pct=0.803711 moved_to_echo=17865 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 618, 2, 0, 977, 0, 4018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 1, 0, 7, 3974, 601, 0, 0, 0, 0, 0, 0, 3955, 0, 1, 4, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 17, 0, 704, 0, 16, 2110, 998, 128, 34, 0, 0, 0, 0, 2710, 0, 0, 1467, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4086, 17, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 4077, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 711, 1353, 28, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5615, 4643, 3961, 4013, 4178, 4104, 4159, 2095] avg=4096.0000 max=5615 max_rank=0 min=2095 min_rank=7 imbalance_ratio=1.370850 max_load_reduction=27153 max_load_reduction_pct=0.828644 moved_to_echo=14536 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[18, 1, 0, 0, 0, 0, 0, 3778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 1, 0, 0, 2, 4014, 602, 0, 0, 0, 0, 0, 0, 3968, 0, 0, 0, 51, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 7, 1, 0, 2112, 905, 128, 32, 0, 0, 0, 0, 2456, 0, 0, 3537, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4091, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4078, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1432, 1355, 2, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3797, 4660, 4085, 3187, 5994, 4109, 4123, 2813] avg=4096.0000 max=5994 max_rank=4 min=2813 min_rank=7 imbalance_ratio=1.463379 max_load_reduction=26774 max_load_reduction_pct=0.817078 moved_to_echo=17039 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 1, 0, 462, 0, 3825, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 2, 0, 1, 0, 0, 0, 3454, 606, 0, 0, 0, 0, 0, 0, 3966, 0, 0, 0, 5, 14, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 9, 0, 38, 430, 0, 2, 2126, 1093, 128, 33, 0, 0, 0, 0, 2967, 0, 0, 2870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3932, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2, 4035, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 1363, 63, 1, 997, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4290, 4098, 3991, 3913, 5837, 3951, 4075, 2613] avg=4096.0000 max=5837 max_rank=4 min=2613 min_rank=7 imbalance_ratio=1.425049 max_load_reduction=26931 max_load_reduction_pct=0.821869 moved_to_echo=16476 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 4, 0, 0, 168, 0, 3877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4057, 606, 0, 0, 0, 0, 0, 0, 3968, 0, 9, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1407, 0, 10, 0, 1, 250, 0, 0, 2125, 32, 128, 33, 0, 0, 0, 0, 87, 1, 0, 3434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 4092, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 807, 1363, 2163, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4051, 4664, 3979, 3986, 3522, 4112, 4116, 4338] avg=4096.0000 max=4664 max_rank=1 min=3522 min_rank=4 imbalance_ratio=1.138672 max_load_reduction=28104 max_load_reduction_pct=0.857666 moved_to_echo=16088 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 25, 0, 2349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 4, 0, 18, 4028, 573, 0, 0, 0, 0, 0, 0, 3952, 0, 0, 0, 37, 0, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 2001, 995, 134, 24, 0, 0, 0, 0, 2692, 3, 0, 2060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4095, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4063, 39, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3541, 1272, 401, 0, 376, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2375, 4641, 3998, 3187, 4755, 4118, 4103, 5591] avg=4096.0000 max=5591 max_rank=7 min=2375 min_rank=0 imbalance_ratio=1.364990 max_load_reduction=27177 max_load_reduction_pct=0.829376 moved_to_echo=18567 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 124, 0, 0, 56, 0, 3964, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 5, 0, 0, 2746, 475, 0, 0, 0, 0, 0, 0, 3800, 0, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2390, 0, 0, 0, 0, 2891, 0, 7, 1666, 559, 123, 34, 0, 0, 0, 0, 1517, 7, 0, 621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 18, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4, 0, 1, 0, 4072, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2192, 1068, 143, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4149, 3228, 3930, 7670, 2145, 4114, 4108, 3424] avg=4096.0000 max=7670 max_rank=3 min=2145 min_rank=4 imbalance_ratio=1.872559 max_load_reduction=25098 max_load_reduction_pct=0.765930 moved_to_echo=13791 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 0, 15104, 10496, 5888, 128, 1536, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[15104, 128, 0, 17792, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[3968, 128, 128, 128, 128, 5632, 128, 128, 16384, 5632, 1024, 384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 1280, 128, 0, 1792, 0, 29824, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 256, 0, 128, 128, 0, 128, 27776, 4736, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[0, 128, 128, 0, 128, 128, 128, 128, 32512, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[31616, 0, 1024, 128, 256, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 128, 0, 128, 128, 32640, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33792 slack_slots=31744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33792 slack_slots=31744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 36, 0, 0, 0, 4066, 8, 80, 0, 0, 29, 4042, 0, 0, 801, 0, 0, 3834, 5, 4057, 3, 1, 4, 0, 0, 3613, 4019, 237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 5, 0, 0, 15, 1, 2, 3192, 0, 0, 1, 3, 1, 0, 2, 0, 0, 570, 0, 0, 0, 0, 51] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 4073, 0, 5, 0, 0, 0, 3078, 0, 0, 1, 22, 0, 4090, 0, 3928, 3186, 0, 0, 0, 0, 4094, 3792, 2, 0, 0, 21, 0, 0, 2, 0, 55, 0, 4092, 0, 0, 3, 0, 0, 161, 0, 3, 3, 2147, 1, 1, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 1, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 671, 42, 37, 4013, 1, 0, 2, 0, 0, 4024, 0, 0, 0, 4, 0, 4044, 690, 3946, 1286, 164, 183, 0, 1, 4082, 931, 19, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4080, 0, 69, 0, 0, 0, 0, 0, 1, 464, 3779, 81, 8, 57, 42, 2, 0, 2, 2, 0, 0, 35, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 3648, 2, 5, 0, 0, 0, 3859, 3, 4086, 346, 0, 18, 0, 0, 4084, 4070, 0, 0, 0, 0, 0, 0, 393, 0, 21, 0, 4044, 0, 0, 0, 0, 0, 2, 1, 0, 0, 3874, 177, 1, 0, 14, 0, 2, 0, 21, 0, 2, 0, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 4046, 18, 3985, 2585, 1, 2, 0, 0, 4081, 3516, 0, 0, 0, 0, 0, 0, 4, 0, 31, 0, 4077, 1, 0, 0, 0, 0, 0, 0, 0, 3787, 2519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 10, 4084, 0, 0, 55, 13, 0, 2630, 2, 0, 10, 0, 0, 4008, 0, 3090, 903, 0, 38, 3, 0, 3918, 4090, 3, 0, 0, 0, 0, 0, 117, 0, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 3215, 1894, 2, 0, 29, 0, 2, 115, 4, 0, 4, 0, 0, 0, 0, 0, 434] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 3553, 0, 0, 76, 0, 6, 3502, 4, 3379, 538, 0, 4, 0, 0, 4096, 4080, 344, 0, 0, 0, 0, 0, 97, 0, 7, 0, 4093, 0, 21, 0, 0, 0, 0, 0, 0, 764, 4095, 1, 1, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 109, 0, 1, 4096, 10, 0, 0, 0, 0, 3217, 0, 0, 103, 0, 6, 4094, 0, 2874, 6, 0, 83, 0, 0, 3997, 3733, 0, 0, 0, 0, 0, 1, 1572, 0, 662, 0, 3977, 0, 1, 0, 0, 0, 72, 3, 0, 64, 4041, 0, 0, 0, 16, 0, 0, 0, 25, 0, 0, 0, 0, 5, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 36, 0, 0, 0, 3970, 8, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 29, 4042, 0, 0, 801, 433, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3134, 3, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 442, 4019, 237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 15, 1, 2, 2725, 0, 0, 0, 0, 0, 0, 0, 3192, 0, 0, 1, 3, 1, 0, 2, 434, 344, 96, 0, 0, 0, 0, 0, 0, 0, 570, 0, 0, 0, 0, 51, 3834, 13, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4014, 5874, 3147, 4698, 3746, 2748, 4073, 4468] avg=4096.0000 max=5874 max_rank=1 min=2748 min_rank=5 imbalance_ratio=1.434082 max_load_reduction=26894 max_load_reduction_pct=0.820740 moved_to_echo=15035 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 489, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 3078, 2585, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 3516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 31, 0, 3734, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3787, 3078, 0, 0, 0, 0, 0, 0, 0, 2519, 0, 0, 0, 0, 0, 0, 0, 426, 343, 97, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 1, 0, 0, 4046, 15, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3999, 981, 5684, 4015, 3770, 6865, 3385, 4069] avg=4096.0000 max=6865 max_rank=5 min=981 min_rank=1 imbalance_ratio=1.676025 max_load_reduction=25903 max_load_reduction_pct=0.790497 moved_to_echo=18089 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 671, 42, 37, 3918, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4024, 0, 0, 0, 489, 476, 0, 0, 0, 0, 0, 0, 4, 0, 0, 690, 3048, 1286, 164, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 499, 931, 19, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3737, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 1, 464, 3079, 0, 0, 0, 0, 0, 0, 0, 3779, 81, 8, 57, 42, 2, 0, 2, 422, 343, 95, 0, 0, 0, 0, 0, 2, 0, 0, 35, 0, 0, 0, 0, 4044, 15, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4669, 4991, 5375, 1456, 3737, 3613, 4831, 4096] avg=4096.0000 max=5375 max_rank=2 min=1456 min_rank=3 imbalance_ratio=1.312256 max_load_reduction=27393 max_load_reduction_pct=0.835968 moved_to_echo=16277 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 0, 3993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3648, 2, 5, 0, 489, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3156, 346, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 500, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 393, 0, 21, 0, 3704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 3080, 0, 0, 0, 0, 0, 0, 0, 3874, 177, 1, 0, 14, 0, 2, 0, 437, 340, 97, 0, 0, 0, 0, 0, 21, 0, 2, 0, 0, 0, 0, 1, 3859, 15, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3997, 4637, 3523, 4570, 4118, 3083, 4942, 3898] avg=4096.0000 max=4942 max_rank=6 min=3083 min_rank=5 imbalance_ratio=1.206543 max_load_reduction=27826 max_load_reduction_pct=0.849182 moved_to_echo=16041 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 10, 3987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 13, 0, 2630, 2, 0, 10, 469, 373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2387, 903, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 480, 4090, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 0, 1, 0, 3749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3215, 2955, 0, 0, 0, 0, 0, 0, 0, 1894, 2, 0, 29, 0, 2, 115, 4, 330, 345, 97, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 434, 4008, 14, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3997, 3552, 3328, 4576, 3867, 6170, 2818, 4460] avg=4096.0000 max=6170 max_rank=5 min=2818 min_rank=6 imbalance_ratio=1.506348 max_load_reduction=26598 max_load_reduction_pct=0.811707 moved_to_echo=17315 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 3553, 0, 0, 76, 491, 407, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4, 2611, 538, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 4080, 344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 7, 0, 3749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 764, 3089, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 0, 0, 0, 0, 361, 344, 97, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 3502, 15, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3999, 4533, 3163, 4925, 3853, 3874, 4899, 3522] avg=4096.0000 max=4925 max_rank=3 min=3163 min_rank=2 imbalance_ratio=1.202393 max_load_reduction=27843 max_load_reduction_pct=0.849701 moved_to_echo=16148 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 3973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 3078, 0, 0, 1, 493, 478, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 3024, 3186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 3792, 2, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 55, 0, 3743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 161, 0, 3, 3, 3091, 0, 0, 0, 0, 0, 0, 0, 2147, 1, 1, 0, 0, 0, 0, 0, 426, 349, 100, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 1, 1, 4090, 19, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3974, 4055, 6232, 4306, 3800, 3261, 3024, 4116] avg=4096.0000 max=6232 max_rank=2 min=3024 min_rank=6 imbalance_ratio=1.521484 max_load_reduction=26536 max_load_reduction_pct=0.809814 moved_to_echo=14201 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 109, 0, 1, 3999, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3217, 0, 0, 103, 479, 346, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 2221, 6, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 3733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1572, 0, 662, 0, 3642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 72, 3, 0, 64, 3014, 0, 0, 0, 0, 0, 0, 0, 4041, 0, 0, 0, 16, 0, 0, 0, 307, 335, 97, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 5, 0, 0, 4094, 15, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4145, 2316, 4222, 5877, 3154, 4796, 4139] avg=4096.0000 max=5877 max_rank=4 min=2316 min_rank=2 imbalance_ratio=1.434814 max_load_reduction=26891 max_load_reduction_pct=0.820648 moved_to_echo=17966 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 24320, 128, 128, 1024, 3840, 3584, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 0, 768, 22784, 8960, 256, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[25600, 384, 128, 128, 128, 128, 128, 128, 3200, 2816, 896, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 256, 128, 128, 8320, 24192, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 3968, 28288, 640, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 2304, 0, 896, 0, 29824, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 128, 896, 128, 128, 31872, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 640, 128, 0, 128, 128, 512, 31488, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 3900, 1427, 10, 4029, 4035, 0, 0, 4, 1, 0, 0, 947, 0, 0, 4092, 0, 15, 0, 0, 0, 204, 0, 0, 7, 1191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 783, 0, 131, 0, 0, 0, 0, 18, 7, 0, 0, 3, 5, 0, 0, 3658, 4088, 0, 20, 0, 0, 4082, 0, 5, 0, 0, 106, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 2, 3872, 562, 0, 4022, 4092, 0, 5, 0, 0, 0, 0, 1769, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 716, 0, 0, 1, 0, 122, 0, 0, 0, 0, 571, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4028, 0, 683, 0, 0, 3976, 0, 105, 0, 0, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 3886, 889, 0, 46, 3945, 0, 0, 0, 0, 0, 0, 1979, 0, 0, 4094, 3, 1, 0, 0, 0, 1526, 264, 206, 274, 3240, 0, 0, 0, 1, 0, 0, 0, 36, 0, 1289, 0, 0, 370, 0, 12, 0, 0, 0, 0, 0, 42, 0, 0, 0, 3006, 3964, 0, 4, 0, 0, 3691, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4057, 0, 0, 400, 4076, 0, 50, 1, 4, 0, 0, 4076, 0, 1, 4045, 0, 99, 99, 0, 0, 0, 9, 1, 0, 1326, 0, 1, 0, 0, 163, 0, 0, 0, 0, 207, 0, 0, 5, 0, 0, 0, 0, 3, 1, 1, 29, 3, 0, 0, 3725, 4084, 0, 1942, 0, 1, 4092, 195, 47, 9, 0, 16, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 3742, 1552, 2, 3, 4094, 7, 0, 0, 0, 0, 0, 3858, 0, 0, 4072, 0, 0, 2, 0, 0, 394, 0, 7, 0, 2327, 0, 0, 0, 0, 0, 21, 0, 0, 0, 1167, 1361, 64, 0, 0, 8, 0, 23, 0, 137, 12, 0, 0, 2, 0, 3195, 2282, 0, 0, 0, 514, 3918, 0, 0, 0, 0, 0, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 8, 4034, 26, 0, 3888, 4095, 0, 3, 0, 0, 0, 0, 1784, 0, 1, 4067, 0, 9, 0, 2, 0, 51, 2, 0, 18, 0, 0, 0, 7, 0, 689, 0, 0, 0, 0, 3451, 1, 14, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 3907, 2871, 0, 3, 0, 2, 3787, 29, 14, 0, 0, 2, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4095, 1562, 0, 12, 4006, 0, 7, 0, 0, 1, 0, 2911, 0, 0, 4093, 0, 1, 4, 0, 0, 644, 96, 0, 0, 2426, 0, 0, 0, 0, 674, 0, 0, 0, 0, 28, 20, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 0, 3965, 4094, 0, 0, 0, 22, 3956, 2, 0, 0, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4004, 0, 0, 3, 4093, 0, 0, 0, 0, 68, 30, 3527, 0, 0, 3980, 444, 0, 184, 0, 0, 399, 700, 0, 1, 1136, 0, 0, 0, 0, 1193, 0, 69, 0, 0, 158, 4, 0, 2, 0, 0, 0, 0, 0, 2, 0, 5, 0, 0, 0, 3910, 4095, 0, 0, 0, 0, 4096, 2, 661, 0, 0, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1839, 0, 0, 400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 1, 4, 0, 0, 4076, 0, 1, 802, 611, 74, 0, 0, 0, 0, 0, 3434, 0, 99, 99, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1326, 0, 1, 0, 0, 163, 1416, 832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 0, 0, 5, 3178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 1, 29, 4076, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3725, 71, 0, 1942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4018, 195, 47, 9, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2239, 5619, 3641, 3739, 3390, 4113, 5741, 4286] avg=4096.0000 max=5741 max_rank=6 min=2239 min_rank=0 imbalance_ratio=1.401611 max_load_reduction=27027 max_load_reduction_pct=0.824799 moved_to_echo=17530 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 1756, 562, 0, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 1769, 0, 0, 765, 619, 71, 0, 0, 0, 0, 0, 3477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 716, 0, 0, 1, 0, 122, 1351, 821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 571, 47, 0, 0, 3135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 69, 0, 683, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3905, 0, 105, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6342, 3229, 3477, 3011, 3753, 4095, 4849, 4012] avg=4096.0000 max=6342 max_rank=0 min=3011 min_rank=3 imbalance_ratio=1.548340 max_load_reduction=26426 max_load_reduction_pct=0.806458 moved_to_echo=16709 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1697, 1552, 2, 3, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3858, 0, 0, 739, 616, 70, 0, 0, 0, 0, 0, 3456, 0, 0, 2, 0, 0, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 2327, 0, 0, 0, 0, 0, 1306, 465, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 1167, 1361, 64, 0, 1776, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 23, 0, 137, 12, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3195, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 3848, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3261, 5283, 3852, 4105, 4389, 4276, 3236, 4366] avg=4096.0000 max=5283 max_rank=1 min=3236 min_rank=6 imbalance_ratio=1.289795 max_load_reduction=27485 max_load_reduction_pct=0.838776 moved_to_echo=16267 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1768, 1427, 10, 4029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 947, 0, 0, 771, 619, 73, 0, 0, 0, 0, 0, 3473, 0, 15, 0, 0, 0, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1191, 0, 0, 0, 0, 0, 1361, 833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 783, 0, 131, 0, 3181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 7, 0, 0, 3, 4035, 3, 0, 0, 0, 0, 0, 0, 5, 0, 0, 3658, 71, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 0, 5, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7234, 2415, 3692, 3392, 4095, 4066, 3754, 4120] avg=4096.0000 max=7234 max_rank=0 min=2415 min_rank=1 imbalance_ratio=1.766113 max_load_reduction=25534 max_load_reduction_pct=0.779236 moved_to_echo=16035 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1762, 889, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1979, 0, 0, 768, 619, 66, 0, 0, 0, 0, 0, 3475, 3, 1, 0, 0, 0, 1526, 264, 0, 0, 0, 0, 0, 0, 0, 0, 206, 274, 3240, 0, 0, 0, 1, 0, 1356, 808, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 1289, 0, 0, 370, 3085, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 42, 3945, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3006, 68, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2697, 3432, 5269, 5885, 4780, 4002, 3078, 3625] avg=4096.0000 max=5885 max_rank=3 min=2697 min_rank=0 imbalance_ratio=1.436768 max_load_reduction=26883 max_load_reduction_pct=0.820404 moved_to_echo=15485 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1857, 1562, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 1, 0, 2911, 0, 0, 809, 619, 71, 0, 0, 0, 0, 0, 3474, 0, 1, 4, 0, 0, 644, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2426, 0, 0, 0, 0, 674, 1429, 834, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 20, 2, 1, 3186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 3, 0, 0, 0, 0, 0, 0, 145, 0, 0, 3965, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 3885, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3431, 4418, 4219, 5363, 3237, 4009, 4181, 3910] avg=4096.0000 max=5363 max_rank=3 min=3237 min_rank=4 imbalance_ratio=1.309326 max_load_reduction=27405 max_load_reduction_pct=0.836334 moved_to_echo=15337 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 1823, 26, 0, 3888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1784, 0, 1, 801, 618, 73, 0, 0, 0, 0, 0, 3449, 0, 9, 0, 2, 0, 51, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 7, 0, 689, 1410, 588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3451, 1, 14, 0, 2238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 4095, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3907, 38, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3714, 29, 14, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5745, 3280, 3513, 2712, 5704, 4104, 3948, 3762] avg=4096.0000 max=5745 max_rank=0 min=2712 min_rank=3 imbalance_ratio=1.402588 max_load_reduction=27023 max_load_reduction_pct=0.824677 moved_to_echo=17518 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1816, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 30, 3527, 0, 0, 791, 602, 74, 0, 0, 0, 0, 0, 3378, 444, 0, 184, 0, 0, 399, 700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1136, 0, 0, 0, 0, 1193, 1397, 834, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 158, 4, 0, 2, 3187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5, 4093, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3910, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 2, 661, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1819, 5092, 5105, 4561, 3420, 4103, 3981, 4687] avg=4096.0000 max=5105 max_rank=2 min=1819 min_rank=0 imbalance_ratio=1.246338 max_load_reduction=27663 max_load_reduction_pct=0.844208 moved_to_echo=16191 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 14336, 6144, 128, 12416, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 20864, 0, 128, 6272, 4992, 640, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[256, 384, 12416, 0, 128, 128, 128, 2944, 11136, 6016, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 0, 7680, 1536, 256, 384, 23040, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[27648, 512, 128, 384, 128, 0, 3328, 1152, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 128, 0, 29568, 512, 0, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[640, 31104, 256, 896, 128, 0, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 0, 128, 128, 256, 128, 128, 32512, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1, 12, 0, 0, 12, 1332, 0, 0, 0, 3, 0, 0, 0, 9, 10, 0, 0, 0, 0, 0, 0, 0, 4077, 1, 6, 3991, 3, 3, 0, 0, 263, 13, 2, 0, 0, 4095, 0, 0, 0, 0, 4052, 0, 0, 0, 3209, 3988, 2243, 0, 176, 0, 0, 1, 0, 0, 3173, 0, 23, 0, 0, 0, 1, 0, 51, 2018] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 13, 0, 771, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 147, 0, 0, 2, 0, 535, 1520, 0, 721, 0, 3403, 0, 0, 0, 0, 3369, 0, 0, 0, 3724, 4096, 1489, 0, 658, 0, 36, 0, 0, 0, 3080, 0, 0, 0, 0, 0, 0, 0, 1001, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1, 21, 110, 0, 7, 10, 2, 0, 0, 0, 5, 0, 0, 3, 0, 0, 13, 0, 0, 0, 27, 0, 3965, 1, 0, 1, 0, 5, 40, 0, 13, 492, 0, 4014, 0, 3472, 52, 0, 0, 947, 3086, 0, 62, 0, 4047, 4096, 49, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 4, 0, 0, 4068, 4063] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 5, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 3, 0, 0, 625, 419, 0, 4090, 2, 4093, 0, 0, 0, 47, 597, 0, 26, 0, 4036, 4096, 2824, 0, 3, 0, 73, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3705, 3990] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 6, 4, 0, 0, 365, 0, 0, 0, 0, 0, 0, 0, 8, 0, 2, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 26, 0, 0, 3, 0, 785, 3817, 0, 3454, 12, 3662, 0, 0, 0, 1, 4088, 0, 34, 0, 3116, 4095, 58, 0, 11, 0, 0, 0, 0, 0, 249, 0, 0, 2, 0, 0, 0, 0, 838, 4039] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 158, 1, 0, 0, 87, 0, 0, 0, 0, 0, 194, 0, 4, 0, 0, 0, 2, 0, 0, 0, 3946, 0, 0, 76, 0, 2, 7, 5, 197, 1258, 0, 492, 199, 1375, 0, 0, 0, 0, 3767, 0, 0, 0, 3295, 3937, 3659, 0, 28, 0, 0, 0, 0, 0, 1887, 1, 0, 0, 0, 0, 2, 14, 4079, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 9, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 0, 0, 30, 0, 0, 0, 0, 549, 128, 0, 2153, 23, 4096, 0, 0, 0, 0, 3399, 0, 44, 0, 4094, 3829, 1822, 0, 558, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 3896, 4084] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 461, 0, 0, 0, 5, 0, 0, 11, 38, 14, 2, 0, 0, 0, 0, 50, 0, 3786, 44, 0, 4013, 0, 0, 0, 0, 309, 0, 0, 0, 14, 3932, 0, 0, 0, 1, 4055, 0, 0, 0, 4073, 4071, 2860, 0, 160, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 0, 0, 1, 848, 3926] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 13, 0, 771, 10, 0, 0, 3692, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 70, 0, 0, 0, 0, 0, 0, 0, 0, 147, 0, 0, 2, 0, 535, 1520, 1053, 631, 0, 0, 0, 0, 0, 0, 0, 721, 0, 2003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2246, 0, 0, 0, 0, 0, 1489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 658, 0, 36, 0, 0, 0, 3080, 0, 2183, 769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1912, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4487, 4128, 4167, 3888, 2724, 3735, 6726, 2913] avg=4096.0000 max=6726 max_rank=6 min=2724 min_rank=4 imbalance_ratio=1.642090 max_load_reduction=26042 max_load_reduction_pct=0.794739 moved_to_echo=16098 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 12, 0, 0, 12, 1332, 0, 0, 3181, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 9, 10, 0, 3988, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 1, 85, 0, 0, 0, 0, 0, 0, 0, 6, 3991, 3, 3, 0, 0, 263, 13, 1267, 760, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2700, 0, 0, 0, 0, 0, 2243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 0, 1, 0, 0, 3173, 0, 1075, 926, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 1, 0, 51, 943, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4538, 4038, 4163, 6306, 2411, 4943, 5351, 1018] avg=4096.0000 max=6306 max_rank=3 min=1018 min_rank=7 imbalance_ratio=1.539551 max_load_reduction=26462 max_load_reduction_pct=0.807556 moved_to_echo=13723 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 21, 110, 0, 7, 10, 2, 0, 4013, 1, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 3, 0, 0, 4096, 34, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 27, 0, 3965, 1, 64, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 40, 0, 13, 492, 965, 644, 0, 0, 0, 0, 0, 0, 0, 4014, 0, 2043, 52, 0, 0, 947, 0, 0, 0, 0, 0, 0, 0, 0, 2057, 0, 62, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 2166, 785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4068, 1896, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4165, 4138, 4070, 2160, 7056, 2168, 3043, 5968] avg=4096.0000 max=7056 max_rank=4 min=2160 min_rank=3 imbalance_ratio=1.722656 max_load_reduction=25712 max_load_reduction_pct=0.784668 moved_to_echo=18235 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 158, 1, 0, 0, 87, 0, 3267, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 194, 0, 4, 0, 3937, 28, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3946, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 2, 7, 5, 197, 1258, 1178, 255, 0, 0, 0, 0, 0, 0, 0, 492, 199, 810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2510, 0, 0, 0, 0, 0, 3659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 1887, 1, 2183, 310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 14, 4079, 1912, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3514, 4163, 4027, 2978, 1501, 6169, 4409, 6007] avg=4096.0000 max=6169 max_rank=5 min=1501 min_rank=4 imbalance_ratio=1.506104 max_load_reduction=26599 max_load_reduction_pct=0.811737 moved_to_echo=18086 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 5, 14, 0, 0, 4001, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 625, 419, 186, 759, 0, 0, 0, 0, 0, 0, 0, 4090, 2, 2409, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 399, 0, 26, 0, 0, 0, 2824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 73, 44, 0, 0, 0, 0, 2127, 925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3705, 1862, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4025, 4131, 4084, 1992, 6548, 3249, 3172, 5567] avg=4096.0000 max=6548 max_rank=4 min=1992 min_rank=3 imbalance_ratio=1.598633 max_load_reduction=26220 max_load_reduction_pct=0.800171 moved_to_echo=18536 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 9, 0, 0, 4060, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3829, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 549, 128, 1062, 760, 0, 0, 0, 0, 0, 0, 0, 2153, 23, 2410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2266, 0, 44, 0, 0, 0, 1822, 0, 0, 0, 0, 0, 0, 0, 0, 0, 558, 0, 0, 0, 0, 0, 14, 0, 2177, 926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3896, 1906, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4070, 3864, 4110, 2529, 4586, 4132, 3675, 5802] avg=4096.0000 max=5802 max_rank=7 min=2529 min_rank=3 imbalance_ratio=1.416504 max_load_reduction=26966 max_load_reduction_pct=0.822937 moved_to_echo=18195 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 4, 0, 0, 365, 0, 0, 3090, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 2, 4095, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 3, 0, 785, 3817, 1281, 684, 0, 0, 0, 0, 0, 0, 0, 3454, 12, 2146, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2718, 0, 34, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 249, 0, 2158, 832, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 838, 1878, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3468, 4131, 4182, 6596, 5613, 2810, 3250, 2718] avg=4096.0000 max=6596 max_rank=3 min=2718 min_rank=7 imbalance_ratio=1.610352 max_load_reduction=26172 max_load_reduction_pct=0.798706 moved_to_echo=14391 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 461, 0, 0, 4039, 1, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 11, 38, 14, 2, 4071, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 3786, 44, 85, 0, 0, 0, 0, 0, 0, 0, 0, 4013, 0, 0, 0, 0, 309, 0, 1268, 729, 0, 0, 0, 0, 0, 0, 0, 0, 14, 2314, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2702, 0, 0, 0, 0, 0, 2860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 0, 0, 0, 0, 0, 0, 2093, 889, 0, 0, 0, 0, 0, 0, 94, 0, 0, 0, 0, 1, 848, 1832, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4501, 4175, 3965, 6319, 2329, 5562, 3142, 2775] avg=4096.0000 max=6319 max_rank=3 min=2329 min_rank=4 imbalance_ratio=1.542725 max_load_reduction=26449 max_load_reduction_pct=0.807159 moved_to_echo=13808 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1664, 0, 128, 128, 0, 0, 8576, 128, 16256, 6400, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[17664, 0, 256, 0, 0, 0, 15104, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 8320, 128, 128, 128, 128, 3328, 7680, 8320, 5248, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 14976, 256, 16640, 128, 0, 0, 1024, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 128, 128, 18560, 14208, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 384, 128, 896, 2304, 128, 0, 29440, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 0, 128, 0, 32128, 128, 640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 256, 128, 128, 128, 32256, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 4068, 3337, 357, 0, 1, 0, 0, 1, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 38, 0, 10, 1, 0, 0, 0, 0, 0, 0, 4037, 4071, 0, 0, 0, 139, 40, 0, 3878, 0, 0, 0, 1294, 0, 9, 0, 0, 0, 4033, 0, 25, 0, 0, 0, 3, 0, 0, 2, 0, 3009, 0, 317] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 3988, 4081, 0, 4007, 0, 0, 57, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 2, 108, 3, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4077, 1, 0, 23, 0, 0, 0, 4096, 0, 0, 0, 9, 0, 2, 11, 0, 0, 64, 0, 2, 0, 1, 1, 0, 7, 0, 0, 0, 4037, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[4, 4062, 2075, 0, 447, 0, 0, 0, 0, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4085, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1549, 2617, 0, 0, 1325, 0, 0, 0, 0, 207, 0, 1, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[33, 4085, 4095, 20, 590, 0, 0, 5, 0, 4094, 0, 116, 2, 0, 0, 0, 2, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4032, 3296, 0, 14, 0, 44, 0, 0, 4086, 0, 0, 0, 25, 0, 0, 2013, 5, 0, 2049, 0, 0, 0, 0, 0, 0, 0, 0, 2, 61, 4087, 0, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 4095, 4090, 28, 1804, 0, 0, 4, 0, 4096, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 0, 1, 0, 0, 0, 0, 0, 2, 0, 4096, 366, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 431, 933, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1, 3510, 4090, 0, 0, 9, 0, 0, 0, 4034, 8, 3198, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 35, 0, 0, 4054, 3517, 14, 217, 0, 1, 0, 0, 3724, 2, 0, 0, 0, 0, 0, 749, 29, 0, 1489, 6, 0, 0, 0, 0, 0, 0, 83, 0, 0, 3923, 47, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 4012, 4096, 132, 1607, 1, 0, 0, 0, 4095, 0, 369, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4, 0, 20, 0, 4094, 862, 0, 0, 0, 0, 0, 0, 4054, 0, 1, 1, 0, 1, 1, 3109, 0, 0, 2464, 0, 0, 0, 0, 0, 0, 0, 1, 0, 101, 3737, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 4090, 3323, 0, 9, 0, 0, 0, 0, 4096, 0, 754, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 563, 0, 1, 0, 0, 2, 0, 3888, 0, 0, 0, 0, 0, 962, 3523, 0, 0, 3338, 0, 0, 0, 0, 1, 4, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2705, 357, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 38, 0, 2524, 1418, 0, 0, 0, 0, 0, 0, 10, 1, 0, 0, 0, 0, 0, 0, 4068, 96, 0, 0, 0, 0, 0, 0, 1417, 4071, 0, 0, 0, 139, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1887, 0, 0, 0, 1294, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 25, 0, 0, 0, 632, 573, 422, 0, 0, 0, 0, 0, 3, 0, 0, 2, 0, 3009, 0, 317, 131, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3063, 3529, 3997, 4175, 5667, 3190, 5685, 3462] avg=4096.0000 max=5685 max_rank=6 min=3063 min_rank=0 imbalance_ratio=1.387939 max_load_reduction=27083 max_load_reduction_pct=0.826508 moved_to_echo=18004 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 1682, 0, 447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3541, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 2558, 1497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 97, 0, 0, 0, 0, 0, 0, 1437, 4085, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1994, 0, 0, 0, 0, 0, 1549, 2617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1325, 0, 0, 0, 0, 207, 393, 605, 423, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 132, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2133, 3550, 4061, 4159, 5523, 6160, 2953, 4229] avg=4096.0000 max=6160 max_rank=5 min=2133 min_rank=0 imbalance_ratio=1.503906 max_load_reduction=26608 max_load_reduction_pct=0.812012 moved_to_echo=18865 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3308, 0, 4007, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 108, 3, 0, 0, 0, 2561, 1497, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3988, 97, 0, 0, 0, 0, 0, 0, 1438, 4077, 1, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1994, 0, 0, 0, 9, 0, 2, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 2, 0, 1, 1, 773, 605, 423, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4037, 0, 0, 132, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7372, 3539, 4171, 4086, 5539, 2016, 1869, 4176] avg=4096.0000 max=7372 max_rank=0 min=1869 min_rank=6 imbalance_ratio=1.799805 max_load_reduction=25396 max_load_reduction_pct=0.775024 moved_to_echo=13600 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 3315, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3487, 8, 3198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 12, 0, 2535, 1361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 3510, 96, 0, 0, 0, 0, 0, 0, 1423, 3517, 14, 217, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1813, 2, 0, 0, 0, 0, 0, 749, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 1489, 6, 0, 0, 0, 0, 775, 550, 417, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 3923, 47, 0, 130, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3325, 6693, 3924, 3641, 5172, 2564, 3266, 4183] avg=4096.0000 max=6693 max_rank=1 min=2564 min_rank=5 imbalance_ratio=1.634033 max_load_reduction=26075 max_load_reduction_pct=0.795746 moved_to_echo=15185 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[33, 0, 3319, 20, 590, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3539, 0, 116, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 3, 0, 2521, 1494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4085, 96, 0, 0, 0, 0, 0, 0, 1415, 3296, 0, 14, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1988, 0, 0, 0, 25, 0, 0, 2013, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2049, 0, 0, 0, 0, 0, 776, 604, 423, 0, 0, 0, 0, 0, 0, 0, 0, 2, 61, 4087, 0, 4, 132, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3967, 3657, 4021, 4185, 4769, 4026, 3857, 4286] avg=4096.0000 max=4769 max_rank=4 min=3657 min_rank=1 imbalance_ratio=1.164307 max_load_reduction=27999 max_load_reduction_pct=0.854462 moved_to_echo=16938 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3315, 28, 1804, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3541, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 0, 2561, 1497, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 4095, 97, 0, 0, 0, 0, 0, 0, 1438, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1994, 0, 0, 0, 1, 0, 431, 933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 775, 605, 423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 132, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5151, 3597, 4539, 4195, 1804, 3359, 5896, 4227] avg=4096.0000 max=5896 max_rank=6 min=1804 min_rank=4 imbalance_ratio=1.439453 max_load_reduction=26872 max_load_reduction_pct=0.820068 moved_to_echo=15286 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3315, 132, 1607, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3535, 0, 369, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2564, 1487, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4, 0, 20, 0, 4012, 102, 0, 0, 0, 0, 0, 0, 1428, 862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1963, 0, 1, 1, 0, 1, 1, 3109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2464, 0, 0, 0, 0, 0, 781, 604, 426, 0, 0, 0, 0, 0, 0, 0, 1, 0, 101, 3737, 0, 0, 134, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5055, 3908, 4051, 4140, 2290, 5076, 4275, 3973] avg=4096.0000 max=5076 max_rank=5 min=2290 min_rank=4 imbalance_ratio=1.239258 max_load_reduction=27692 max_load_reduction_pct=0.845093 moved_to_echo=15614 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2693, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3541, 0, 754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 8, 0, 2561, 1421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 97, 0, 0, 0, 0, 0, 0, 1438, 563, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1892, 0, 0, 0, 0, 0, 962, 3523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3338, 0, 0, 0, 0, 1, 630, 575, 423, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 0, 0, 132, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2702, 4295, 4004, 4187, 2004, 6377, 4967, 4232] avg=4096.0000 max=6377 max_rank=5 min=2004 min_rank=4 imbalance_ratio=1.556885 max_load_reduction=26391 max_load_reduction_pct=0.805389 moved_to_echo=17580 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 0, 128, 256, 128, 0, 640, 0, 20480, 11776, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[11520, 20864, 128, 256, 128, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 0, 18944, 128, 128, 0, 128, 256, 5632, 4736, 3456, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[15616, 128, 128, 128, 1408, 128, 3072, 13056, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 23680, 640, 8576, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 28288, 128, 4608, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 256, 31104, 128, 384, 1152, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 0, 0, 128, 128, 128, 0, 32000, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4030, 0, 0, 0, 3096, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 339, 4094, 0, 0, 0, 102, 0, 0, 0, 4096, 4, 9, 0, 0, 0, 0, 0, 0, 0, 3627, 0, 4089, 0, 0, 557, 0, 0, 0, 0, 4093, 35, 0, 1, 0, 0, 0, 0, 0, 0, 493, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[3121, 0, 44, 0, 4089, 0, 0, 9, 0, 45, 0, 3, 196, 287, 0, 0, 0, 0, 0, 0, 1, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 3690, 0, 0, 0, 326, 0, 0, 0, 0, 0, 707, 0, 3914, 0, 0, 3809, 0, 0, 0, 0, 4067, 0, 2, 0, 1, 2, 29, 186, 0, 0, 91, 0, 0, 4059] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[2087, 0, 0, 0, 3674, 0, 0, 0, 0, 0, 0, 0, 0, 266, 0, 0, 0, 124, 0, 0, 0, 284, 4092, 0, 0, 31, 0, 0, 18, 7, 4023, 0, 3657, 4, 0, 0, 1, 0, 14, 0, 0, 0, 3965, 444, 47, 2655, 0, 0, 0, 0, 502, 0, 0, 4, 430, 1, 64, 0, 0, 9, 2574, 0, 4, 3787] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[4070, 0, 1, 0, 3091, 0, 0, 30, 0, 2, 0, 1, 9, 39, 0, 0, 595, 1, 0, 0, 0, 912, 4096, 0, 0, 0, 0, 0, 2, 0, 3917, 2, 288, 0, 0, 40, 0, 0, 0, 0, 7, 0, 3945, 0, 7, 2862, 0, 0, 0, 0, 2878, 0, 0, 0, 625, 0, 10, 1054, 3, 0, 197, 0, 0, 4084] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[3329, 4, 0, 0, 3669, 0, 0, 3, 0, 0, 4, 0, 0, 0, 12, 0, 530, 0, 0, 0, 0, 0, 3507, 0, 0, 0, 0, 0, 1, 0, 4090, 0, 246, 0, 0, 0, 0, 0, 0, 0, 11, 0, 4086, 3, 0, 4039, 0, 0, 0, 0, 2490, 0, 1, 29, 33, 0, 0, 522, 4, 0, 2055, 6, 0, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[4088, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 15, 0, 0, 0, 0, 581, 2554, 0, 0, 0, 0, 0, 0, 0, 4096, 26, 96, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4058, 0, 0, 4096, 0, 0, 0, 0, 2802, 5, 0, 5, 0, 0, 0, 0, 0, 0, 2131, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[4094, 6, 6, 0, 3985, 0, 0, 0, 1, 0, 0, 4, 4, 0, 0, 0, 0, 0, 33, 0, 0, 0, 4091, 0, 0, 5, 0, 0, 0, 1, 4071, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 4070, 0, 0, 0, 0, 4094, 0, 0, 0, 3, 0, 0, 115, 0, 2, 0, 0, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[3381, 0, 0, 0, 4016, 93, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 18, 3055, 0, 0, 0, 135, 0, 0, 0, 4089, 0, 1062, 0, 1067, 0, 0, 0, 0, 0, 20, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1135, 0, 0, 0, 0, 0, 0, 13, 0, 0, 2368, 0, 0, 4038] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4030, 0, 0, 0, 456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 3872, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 339, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 0, 0, 4096, 4, 44, 4, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 2502, 246, 0, 0, 0, 0, 0, 0, 3627, 0, 217, 0, 0, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 35, 0, 1, 0, 0, 1206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 0, 0, 2640, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4486, 4017, 4389, 4250, 2757, 4401, 5335, 3133] avg=4096.0000 max=5335 max_rank=6 min=2757 min_rank=4 imbalance_ratio=1.302490 max_load_reduction=27433 max_load_reduction_pct=0.837189 moved_to_echo=15626 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3121, 0, 44, 0, 603, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 3, 196, 287, 0, 0, 3706, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3690, 0, 44, 4, 0, 0, 0, 0, 0, 0, 0, 0, 326, 0, 0, 0, 0, 0, 3304, 243, 0, 0, 0, 0, 0, 0, 707, 0, 208, 0, 0, 3809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 2, 0, 1, 2, 1195, 0, 0, 0, 0, 0, 0, 0, 29, 186, 0, 0, 91, 0, 0, 2617, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3777, 4419, 4047, 3738, 3873, 4724, 5267, 2923] avg=4096.0000 max=5267 max_rank=6 min=2923 min_rank=7 imbalance_ratio=1.285889 max_load_reduction=27501 max_load_reduction_pct=0.839264 moved_to_echo=16787 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2087, 0, 0, 0, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, 0, 0, 3755, 163, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 0, 284, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 18, 7, 4023, 0, 44, 4, 0, 0, 0, 0, 0, 0, 3657, 4, 0, 0, 1, 0, 14, 0, 2969, 227, 0, 0, 0, 0, 0, 0, 0, 0, 210, 444, 47, 2655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 0, 0, 4, 430, 1, 1115, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 9, 2574, 0, 4, 2441, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2629, 4184, 4456, 4127, 6872, 3356, 2052, 5092] avg=4096.0000 max=6872 max_rank=4 min=2052 min_rank=6 imbalance_ratio=1.677734 max_load_reduction=25896 max_load_reduction_pct=0.790283 moved_to_echo=17372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4070, 0, 1, 0, 456, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 9, 39, 0, 0, 3736, 137, 0, 0, 0, 0, 0, 0, 595, 1, 0, 0, 0, 912, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3917, 2, 45, 4, 0, 0, 0, 0, 0, 0, 288, 0, 0, 40, 0, 0, 0, 0, 2498, 245, 0, 0, 0, 0, 0, 0, 7, 0, 209, 0, 7, 2862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2878, 0, 0, 0, 625, 0, 1203, 0, 0, 0, 0, 0, 0, 0, 10, 1054, 3, 0, 197, 0, 0, 2632, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4557, 3924, 5559, 3970, 3071, 3085, 4706, 3896] avg=4096.0000 max=5559 max_rank=2 min=3071 min_rank=4 imbalance_ratio=1.357178 max_load_reduction=27209 max_load_reduction_pct=0.830353 moved_to_echo=14758 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3329, 4, 0, 0, 541, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 12, 0, 3869, 163, 0, 0, 0, 0, 0, 0, 530, 0, 0, 0, 0, 0, 3469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4090, 0, 38, 4, 0, 0, 0, 0, 0, 0, 246, 0, 0, 0, 0, 0, 0, 0, 2965, 246, 0, 0, 0, 0, 0, 0, 11, 0, 217, 3, 0, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2490, 0, 1, 29, 33, 0, 1206, 0, 0, 0, 0, 0, 0, 0, 0, 522, 4, 0, 2055, 6, 0, 2638, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3877, 4048, 3999, 4133, 3457, 4270, 3759, 5225] avg=4096.0000 max=5225 max_rank=7 min=3457 min_rank=4 imbalance_ratio=1.275635 max_load_reduction=27543 max_load_reduction_pct=0.840546 moved_to_echo=16711 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4088, 0, 0, 0, 603, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 3843, 182, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 581, 2526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 26, 28, 4, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 3304, 246, 0, 0, 0, 0, 0, 0, 6, 0, 215, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2802, 5, 0, 5, 0, 0, 1206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2131, 0, 0, 2640, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4691, 4049, 3122, 4154, 3646, 4317, 4018, 4771] avg=4096.0000 max=4771 max_rank=7 min=3122 min_rank=2 imbalance_ratio=1.164795 max_load_reduction=27997 max_load_reduction_pct=0.854401 moved_to_echo=16752 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4094, 6, 6, 0, 579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 4, 0, 0, 0, 3873, 182, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 1, 4071, 0, 49, 9, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 3224, 249, 0, 0, 0, 0, 0, 0, 0, 0, 212, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 3, 0, 1210, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 2, 0, 0, 0, 2627, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4685, 4064, 4075, 4135, 3476, 4282, 5307, 2744] avg=4096.0000 max=5307 max_rank=6 min=2744 min_rank=7 imbalance_ratio=1.295654 max_load_reduction=27461 max_load_reduction_pct=0.838043 moved_to_echo=15809 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3381, 0, 0, 0, 592, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 3879, 179, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 18, 3022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 0, 0, 0, 4089, 0, 33, 4, 0, 0, 0, 0, 0, 0, 1062, 0, 1067, 0, 0, 0, 0, 0, 3245, 242, 0, 0, 0, 0, 0, 0, 20, 0, 217, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1135, 0, 0, 0, 0, 0, 1189, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 2368, 0, 0, 2603, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 4063, 3121, 4261, 5616, 4333, 2324, 4984] avg=4096.0000 max=5616 max_rank=4 min=2324 min_rank=6 imbalance_ratio=1.371094 max_load_reduction=27152 max_load_reduction_pct=0.828613 moved_to_echo=17257 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[5376, 128, 1408, 128, 128, 0, 128, 0, 24064, 2048, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 1920, 128, 128, 9984, 128, 128, 20864, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 0, 22144, 128, 128, 128, 1152, 128, 9600, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[28288, 128, 128, 0, 4480, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 256, 640, 128, 0, 30592, 1408, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1280, 128, 128, 0, 128, 2176, 29312, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4480, 0, 1792, 512, 128, 26240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 256, 0, 128, 128, 32128, 128, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[4096, 3936, 0, 4048, 14, 0, 0, 0, 4096, 0, 0, 98, 0, 35, 2317, 0, 0, 0, 0, 0, 0, 0, 744, 272, 0, 0, 0, 222, 0, 1867, 2991, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 34, 3, 0, 4048, 3, 3875, 0, 3, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 58] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[4091, 4011, 88, 4095, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 14, 1, 0, 0, 0, 0, 0, 0, 0, 3821, 0, 0, 0, 34, 0, 1178, 0, 3324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 6, 3912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[3976, 3752, 0, 3711, 13, 0, 0, 0, 4080, 0, 338, 1, 3, 20, 283, 1, 0, 532, 0, 0, 2, 169, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 640, 0, 16, 0, 164, 0, 0, 0, 0, 0, 3, 0, 1, 4017, 4055, 2814, 0, 4, 0, 0, 0, 0, 6, 0, 0, 0, 0, 24, 0, 0, 51, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[4033, 2944, 119, 4095, 0, 0, 0, 1, 4076, 0, 7, 52, 0, 0, 4080, 0, 0, 0, 0, 0, 4, 0, 0, 1556, 0, 0, 0, 1176, 0, 1, 1, 609, 0, 0, 6, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 2208, 3697, 1, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[3632, 3849, 0, 4020, 0, 0, 0, 0, 4094, 1, 39, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 43, 0, 0, 18, 0, 4083, 0, 0, 36, 0, 0, 0, 0, 7, 67, 0, 0, 0, 4067, 233, 4041, 0, 0, 0, 51, 377, 0, 0, 0, 0, 2, 5, 0, 0, 0, 0, 2, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[4096, 3997, 403, 4096, 0, 0, 0, 0, 4073, 0, 0, 24, 0, 0, 111, 0, 0, 17, 0, 0, 2, 0, 0, 1116, 0, 0, 0, 4, 0, 0, 0, 3501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3744, 3482, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4089, 3464, 0, 4077, 0, 0, 0, 0, 4037, 1, 117, 0, 0, 1, 21, 0, 0, 12, 0, 0, 3, 0, 420, 4062, 0, 0, 25, 36, 4, 0, 23, 2454, 0, 0, 0, 4, 0, 0, 0, 0, 1, 0, 0, 0, 4092, 2204, 3610, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[669, 4096, 4, 3440, 0, 0, 0, 0, 4096, 0, 0, 2, 15, 0, 3945, 10, 0, 0, 0, 0, 0, 0, 522, 310, 0, 0, 0, 10, 0, 1, 4091, 0, 1473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3867, 1186, 4096, 0, 671, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 264] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4091, 461, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2663, 0, 4, 0, 0, 0, 14, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3821, 871, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 1178, 0, 3324, 852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3547, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4640, 2682, 5232, 5388, 4109, 2526, 4101, 4090] avg=4096.0000 max=5388 max_rank=3 min=2526 min_rank=5 imbalance_ratio=1.315430 max_load_reduction=27380 max_load_reduction_pct=0.835571 moved_to_echo=14826 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4096, 452, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2663, 0, 0, 98, 0, 35, 2317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 744, 272, 871, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 222, 0, 1867, 2991, 0, 844, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 3481, 562, 0, 0, 0, 0, 0, 0, 0, 34, 3, 0, 0, 3, 2497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 0, 0, 0, 0, 4048, 3, 3, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 58, 4045, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4562, 5113, 2421, 5924, 4048, 2537, 4059, 4104] avg=4096.0000 max=5924 max_rank=3 min=2421 min_rank=2 imbalance_ratio=1.446289 max_load_reduction=26844 max_load_reduction_pct=0.819214 moved_to_echo=14748 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3632, 442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2662, 1, 39, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 871, 557, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 18, 0, 4083, 880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 7, 3404, 561, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 0, 233, 2604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 377, 0, 0, 0, 0, 4020, 3, 3, 0, 0, 0, 0, 0, 2, 5, 0, 0, 0, 0, 2, 6, 4064, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 2707, 5518, 5024, 4008, 2904, 4454, 4079] avg=4096.0000 max=5518 max_rank=2 min=2707 min_rank=1 imbalance_ratio=1.347168 max_load_reduction=27250 max_load_reduction_pct=0.831604 moved_to_echo=15445 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4033, 339, 119, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2650, 0, 7, 52, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1556, 867, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1176, 0, 1, 1, 609, 805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 1, 0, 2603, 559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2208, 2382, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 4095, 2, 3, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4492, 6789, 2937, 2592, 3169, 4591, 4103, 4095] avg=4096.0000 max=6789 max_rank=1 min=2592 min_rank=3 imbalance_ratio=1.657471 max_load_reduction=25979 max_load_reduction_pct=0.792816 moved_to_echo=15958 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3976, 431, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2653, 0, 338, 1, 3, 20, 283, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 0, 0, 2, 169, 0, 4092, 868, 388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 640, 612, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 164, 0, 0, 0, 0, 3318, 559, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 4055, 1814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 6, 0, 0, 3711, 3, 3, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 51, 0, 0, 4014, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4420, 3299, 6051, 1252, 4057, 5873, 3727, 4089] avg=4096.0000 max=6051 max_rank=2 min=1252 min_rank=3 imbalance_ratio=1.477295 max_load_reduction=26717 max_load_reduction_pct=0.815338 moved_to_echo=17746 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4096, 459, 403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2649, 0, 0, 24, 0, 0, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 2, 0, 0, 1116, 866, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 3501, 758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3535, 558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3744, 2244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4958, 2784, 2481, 4263, 4093, 5988, 4108, 4093] avg=4096.0000 max=5988 max_rank=5 min=2481 min_rank=2 imbalance_ratio=1.461914 max_load_reduction=26780 max_load_reduction_pct=0.817261 moved_to_echo=18282 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4089, 389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2619, 1, 117, 0, 0, 1, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 3, 0, 420, 4062, 861, 502, 0, 0, 0, 0, 0, 0, 0, 0, 25, 36, 4, 0, 23, 2454, 789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 3066, 557, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2204, 2319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 4077, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4478, 2759, 5860, 3331, 3627, 4524, 4100, 4089] avg=4096.0000 max=5860 max_rank=2 min=2759 min_rank=1 imbalance_ratio=1.430664 max_load_reduction=26908 max_load_reduction_pct=0.821167 moved_to_echo=16340 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[669, 471, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2663, 0, 0, 2, 15, 0, 3945, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 310, 871, 565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 1, 4091, 0, 892, 0, 0, 0, 0, 0, 0, 0, 1473, 0, 0, 0, 0, 0, 0, 0, 3622, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1186, 2639, 0, 0, 0, 0, 0, 0, 0, 0, 0, 671, 0, 0, 0, 0, 0, 0, 0, 3440, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 264, 3865, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1144, 6635, 2268, 4994, 5657, 3825, 4116, 4129] avg=4096.0000 max=6635 max_rank=1 min=1144 min_rank=0 imbalance_ratio=1.619873 max_load_reduction=26133 max_load_reduction_pct=0.797516 moved_to_echo=17727 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 640, 0, 0, 128, 256, 1792, 19328, 7040, 4096, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[21248, 128, 512, 256, 128, 128, 10880, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 128, 1536, 128, 3072, 7168, 14720, 6528, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1536, 128, 128, 256, 0, 0, 128, 128, 26624, 4480, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 0, 13696, 19072, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[28800, 3456, 640, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 0, 128, 128, 384, 32384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[768, 0, 128, 384, 0, 128, 0, 128, 31616, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4053, 0, 3355, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 629, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 855, 0, 0, 0, 3, 0, 0, 0, 0, 3670, 0, 3789, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[18, 0, 0, 0, 0, 0, 3318, 0, 2, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 5, 0, 4093, 0, 2, 0, 0, 0, 783, 0, 4087, 0, 3, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4046, 0, 0, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 5, 5, 0, 1, 3278, 0, 3698, 29, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 94, 0, 4083, 0, 0, 0, 0, 4, 4093, 0, 0, 1, 0, 0, 1782, 0, 4071, 3, 0, 0, 0, 0, 6, 41, 0, 0, 0, 0, 0, 3743, 0, 2, 0, 3, 0, 0, 0, 0, 3705, 0, 30, 0, 0, 4086] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 11, 4096, 0, 0, 449, 0, 0, 0, 0, 52, 0, 0, 181, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4035, 0, 0, 0, 1, 0, 0, 3, 4096, 0, 0, 0, 0, 1, 0, 0, 13, 0, 0, 0, 0, 2987, 4, 650, 0, 1, 0, 5, 0, 0, 3903, 0, 4088, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4087, 0, 1672, 0, 0, 0, 0, 0, 0, 72, 0, 108, 0, 0, 3, 176, 3959, 0, 4, 7, 0, 0, 4039, 4, 0, 10, 0, 0, 23, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3262, 1, 0, 0, 0, 0, 6, 30, 0, 4057, 0, 3127, 0, 21, 4004] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 2, 4086, 0, 893, 175, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 10, 4091, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3028, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 754, 2562, 0, 0, 0, 0, 0, 0, 0, 154, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 40, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 855, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 3935, 0, 0, 3998] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 64, 41, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 159, 0, 0, 0, 0, 4062, 118, 94, 0, 6, 0, 0, 0, 0, 3855, 0, 3886, 0, 0, 4092] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4053, 0, 141, 60, 0, 0, 0, 0, 0, 0, 3355, 31, 0, 0, 0, 0, 0, 0, 1725, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 629, 0, 3690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 855, 0, 0, 0, 3, 0, 0, 1152, 263, 0, 0, 0, 0, 0, 0, 0, 0, 793, 0, 3789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4254, 5113, 4027, 4106, 4319, 4094, 2273, 4582] avg=4096.0000 max=5113 max_rank=1 min=2273 min_rank=6 imbalance_ratio=1.248291 max_load_reduction=27655 max_load_reduction_pct=0.843964 moved_to_echo=15268 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 5, 5, 0, 1, 3278, 0, 140, 60, 0, 0, 0, 0, 0, 0, 3698, 29, 0, 0, 0, 0, 0, 5, 1741, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4093, 0, 0, 1, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1782, 0, 3670, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 41, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 3743, 0, 2, 0, 3, 0, 0, 1163, 261, 0, 0, 0, 0, 0, 0, 0, 0, 801, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3489, 5475, 4107, 4108, 5455, 4131, 5172, 831] avg=4096.0000 max=5475 max_rank=1 min=831 min_rank=7 imbalance_ratio=1.336670 max_load_reduction=27293 max_load_reduction_pct=0.832916 moved_to_echo=15589 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 11, 4096, 0, 141, 60, 0, 0, 0, 0, 0, 0, 0, 449, 0, 0, 0, 0, 52, 0, 1834, 2, 0, 0, 0, 0, 0, 0, 0, 181, 0, 0, 0, 0, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 3692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 13, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 2987, 4, 650, 0, 1, 0, 5, 1226, 263, 0, 0, 0, 0, 0, 0, 0, 0, 843, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4308, 2337, 4207, 4045, 3696, 4108, 5136, 4931] avg=4096.0000 max=5136 max_rank=6 min=2337 min_rank=1 imbalance_ratio=1.253906 max_load_reduction=27632 max_load_reduction_pct=0.843262 moved_to_echo=17871 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[18, 0, 0, 0, 0, 0, 3318, 0, 141, 60, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 3, 0, 0, 0, 0, 1922, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4093, 0, 2, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 783, 0, 3684, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 1284, 262, 0, 0, 0, 0, 0, 0, 0, 0, 884, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3537, 6024, 4025, 4110, 4470, 4094, 1578, 4930] avg=4096.0000 max=6024 max_rank=1 min=1578 min_rank=6 imbalance_ratio=1.470703 max_load_reduction=26744 max_load_reduction_pct=0.816162 moved_to_echo=15072 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4087, 0, 141, 58, 0, 0, 0, 0, 0, 0, 1672, 0, 0, 0, 0, 0, 0, 72, 1907, 2, 0, 0, 0, 0, 0, 0, 0, 108, 0, 0, 3, 176, 3891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 7, 0, 0, 4039, 4, 0, 10, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 2, 3690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4002, 0, 0, 0, 0, 0, 0, 0, 0, 3262, 1, 0, 0, 0, 0, 6, 1274, 263, 0, 0, 0, 0, 0, 0, 30, 0, 876, 0, 3127, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4286, 3653, 4178, 4074, 3715, 4002, 4806, 4054] avg=4096.0000 max=4806 max_rank=6 min=3653 min_rank=1 imbalance_ratio=1.173340 max_load_reduction=27962 max_load_reduction_pct=0.853333 moved_to_echo=16577 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 141, 60, 0, 0, 0, 0, 0, 0, 754, 2562, 0, 0, 0, 0, 0, 0, 1920, 2, 0, 0, 0, 0, 0, 0, 0, 154, 0, 0, 0, 0, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 3692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 855, 0, 0, 0, 0, 0, 0, 1283, 263, 0, 0, 0, 0, 0, 0, 0, 0, 883, 0, 3935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4297, 5238, 4180, 4106, 3732, 3996, 2401, 4818] avg=4096.0000 max=5238 max_rank=1 min=2401 min_rank=6 imbalance_ratio=1.278809 max_load_reduction=27530 max_load_reduction_pct=0.840149 moved_to_echo=14947 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 2, 4086, 0, 147, 64, 0, 0, 0, 0, 0, 0, 893, 175, 1, 0, 0, 0, 0, 0, 1929, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4095, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3679, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 1291, 265, 0, 0, 0, 0, 0, 0, 0, 0, 875, 0, 3028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4300, 3000, 4019, 4113, 3690, 4094, 5649, 3903] avg=4096.0000 max=5649 max_rank=6 min=3000 min_rank=1 imbalance_ratio=1.379150 max_load_reduction=27119 max_load_reduction_pct=0.827606 moved_to_echo=17336 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 141, 60, 0, 0, 0, 0, 0, 0, 64, 41, 9, 0, 0, 0, 0, 0, 1812, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 159, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 118, 94, 0, 6, 0, 0, 1210, 263, 0, 0, 0, 0, 0, 0, 0, 0, 833, 0, 3886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4297, 1928, 4025, 4106, 3691, 4249, 5753, 4719] avg=4096.0000 max=5753 max_rank=6 min=1928 min_rank=1 imbalance_ratio=1.404541 max_load_reduction=27015 max_load_reduction_pct=0.824432 moved_to_echo=18412 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 19968, 128, 768, 0, 128, 0, 128, 9984, 2176, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[10496, 7424, 128, 128, 0, 0, 128, 128, 14848, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 6912, 0, 25984, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 0, 3328, 128, 29568, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 256, 0, 0, 0, 32640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 512, 0, 0, 128, 256, 32128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 0, 128, 31232, 0, 1152, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33792 slack_slots=31744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 348, 2853, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 1404, 0, 0, 4096, 3735, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 3967, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 0, 0, 66, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 2765, 1178, 0, 0, 0, 58, 0, 0, 0, 4096, 0, 58, 3844, 175, 4065, 0, 0, 3614, 234, 0, 0, 0, 0, 2, 0, 0, 0, 4092, 4033, 0, 0, 0, 0, 0, 0, 0, 8, 0, 9, 3, 0, 0, 4036, 0, 418] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 11, 0, 0, 0, 10, 0, 1, 0, 138, 4038, 0, 3, 0, 0, 0, 0, 0, 4022, 0, 11, 4095, 0, 4084, 2, 15, 4088, 99, 0, 0, 0, 51, 0, 1, 0, 0, 3489, 4094, 0, 2, 0, 0, 0, 0, 1, 15, 0, 5, 0, 2, 0, 4063, 0, 424] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3981, 3, 0, 0, 5, 0, 0, 0, 4085, 1, 5, 4094, 111, 4081, 0, 9, 4073, 11, 0, 0, 0, 0, 0, 0, 0, 17, 4091, 4089, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 0, 1, 4087, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 4, 4000, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 2045, 0, 0, 4096, 2852, 0, 0, 0, 0, 0, 0, 0, 1, 3397, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4075, 0, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 0, 0, 4, 0, 0, 0, 3, 0, 7, 0, 611, 0, 0, 0, 253, 432, 10, 33, 0, 110, 0, 6, 0, 4065, 0, 5, 3562, 0, 4078, 102, 0, 4046, 3718, 1, 0, 0, 2, 0, 0, 0, 1, 4011, 3848, 0, 0, 19, 23, 0, 1, 0, 0, 57, 0, 1, 56, 37, 3664, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 0, 15, 21, 4, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2782, 1, 0, 0, 44, 0, 1, 0, 4047, 0, 0, 4070, 131, 2445, 0, 43, 4096, 3109, 1, 0, 2, 0, 0, 0, 0, 0, 4095, 4037, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 3813, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 24, 18, 0, 0, 0, 0, 2, 0, 0, 0, 23, 0, 0, 0, 0, 3485, 3565, 0, 0, 0, 0, 0, 0, 0, 3185, 0, 0, 4062, 0, 4095, 0, 0, 4096, 2114, 0, 0, 0, 0, 0, 0, 0, 0, 122, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 339, 0, 3535] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 0, 0, 3692, 248, 0, 0, 0, 0, 0, 0, 66, 0, 12, 0, 0, 0, 0, 0, 3121, 1073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2765, 1178, 0, 0, 0, 447, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 4096, 0, 58, 94, 0, 0, 0, 0, 0, 0, 0, 0, 175, 2522, 0, 0, 0, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4092, 664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3614, 58, 0, 0, 0, 0, 0, 0, 0, 9, 3, 0, 0, 4036, 0, 418, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3942, 4272, 4390, 4306, 2931, 4758, 3680, 4489] avg=4096.0000 max=4758 max_rank=5 min=2931 min_rank=4 imbalance_ratio=1.161621 max_load_reduction=28010 max_load_reduction_pct=0.854797 moved_to_echo=15858 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 3934, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3170, 370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 348, 2853, 0, 0, 0, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 872, 0, 0, 0, 3735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 62, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3967, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4186, 3540, 3355, 4194, 4607, 4749, 4159, 3978] avg=4096.0000 max=4749 max_rank=5 min=3355 min_rank=2 imbalance_ratio=1.159424 max_load_reduction=28019 max_load_reduction_pct=0.855072 moved_to_echo=17493 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 3933, 252, 0, 0, 0, 0, 0, 0, 0, 0, 3, 11, 0, 0, 0, 10, 3168, 1078, 0, 0, 0, 0, 0, 0, 0, 1, 0, 138, 4038, 0, 3, 0, 449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 0, 11, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2534, 2, 15, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 1, 0, 0, 3489, 674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 15, 4088, 62, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2, 0, 4063, 0, 424, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4186, 4270, 4629, 4133, 2650, 4215, 4168, 4517] avg=4096.0000 max=4629 max_rank=2 min=2650 min_rank=4 imbalance_ratio=1.130127 max_load_reduction=28139 max_load_reduction_pct=0.858734 moved_to_echo=15550 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 0, 0, 4, 3421, 237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 7, 0, 611, 2978, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 253, 432, 10, 33, 0, 448, 0, 0, 0, 0, 0, 0, 0, 110, 0, 6, 0, 4065, 0, 5, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2531, 102, 0, 0, 3718, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 4011, 633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 23, 0, 1, 0, 0, 4046, 54, 0, 0, 0, 0, 0, 0, 57, 0, 1, 56, 37, 3664, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3664, 4675, 1176, 4273, 6352, 4647, 4143, 3838] avg=4096.0000 max=6352 max_rank=4 min=1176 min_rank=2 imbalance_ratio=1.550781 max_load_reduction=26416 max_load_reduction_pct=0.806152 moved_to_echo=18980 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 0, 15, 21, 3909, 249, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 1, 3124, 645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2782, 1, 0, 0, 269, 0, 0, 0, 0, 0, 0, 0, 44, 0, 1, 0, 4047, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 131, 1517, 0, 43, 0, 3109, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4095, 664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3813, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4197, 3775, 3052, 4192, 4801, 4761, 4163, 3827] avg=4096.0000 max=4801 max_rank=4 min=3052 min_rank=2 imbalance_ratio=1.172119 max_load_reduction=27967 max_load_reduction_pct=0.853485 moved_to_echo=17552 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 3933, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 3170, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4000, 0, 0, 0, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1269, 0, 0, 0, 2852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3397, 674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 62, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4075, 0, 5, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4185, 3715, 4229, 4196, 4121, 4072, 4158, 4092] avg=4096.0000 max=4229 max_rank=2 min=3715 min_rank=1 imbalance_ratio=1.032471 max_load_reduction=28539 max_load_reduction_pct=0.870941 moved_to_echo=16443 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 20, 3935, 257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3169, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3981, 3, 0, 0, 453, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4085, 1, 5, 93, 0, 0, 0, 0, 0, 0, 0, 0, 111, 2520, 0, 9, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4091, 663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4073, 66, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 4087, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4213, 4250, 4437, 4189, 2651, 4771, 4140, 4117] avg=4096.0000 max=4771 max_rank=5 min=2651 min_rank=4 imbalance_ratio=1.164795 max_load_reduction=27997 max_load_reduction_pct=0.854401 moved_to_echo=15679 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 24, 18, 0, 0, 3901, 252, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 23, 0, 3165, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3485, 3565, 0, 0, 0, 450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3185, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2541, 0, 0, 0, 2114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 339, 0, 3535, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4195, 4271, 7500, 3285, 4655, 795, 4157, 3910] avg=4096.0000 max=7500 max_rank=2 min=795 min_rank=5 imbalance_ratio=1.831055 max_load_reduction=25268 max_load_reduction_pct=0.771118 moved_to_echo=13517 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[512, 16384, 128, 128, 0, 15872, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 0, 7040, 22912, 128, 128, 0, 2944, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 0, 128, 128, 640, 25088, 7040, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 0, 128, 27392, 5376, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 0, 0, 128, 128, 128, 128, 128, 30720, 2048, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 0, 128, 128, 128, 32256, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 0, 128, 0, 31744, 128, 128, 896, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 28160, 0, 4480, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 4096, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 23, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 44, 4081, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 3, 0, 0, 0, 0, 3490, 0, 899, 0, 1, 0, 0, 3619, 0, 0, 0, 19, 0, 157, 1, 0, 4050, 0, 34, 0, 0, 3, 0, 16, 0, 0, 4076, 0, 0, 1, 5, 0, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[59, 0, 4096, 13, 0, 2, 1959, 4073, 0, 0, 0, 4036, 1, 0, 0, 0, 11, 0, 0, 0, 4, 0, 0, 4096, 0, 1944, 0, 0, 0, 0, 54, 371, 0, 3762, 0, 0, 31, 0, 0, 0, 0, 13, 0, 1, 34, 0, 0, 4083, 3, 0, 0, 0, 0, 5, 0, 0, 2, 4094, 0, 0, 0, 5, 16, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 29, 4088, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3407, 0, 0, 0, 46, 0, 943, 0, 3755, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 48, 3618, 18, 0, 0, 1100, 4051, 0, 0, 0, 4075, 3, 0, 0, 0, 0, 0, 17, 0, 0, 0, 2, 4044, 0, 4012, 5, 0, 0, 3, 6, 3339, 41, 174, 0, 0, 62, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4021, 0, 0, 0, 0, 21, 0, 0, 49, 0, 4035, 1, 0, 20, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[15, 0, 4038, 0, 0, 0, 28, 4057, 0, 0, 0, 3934, 0, 0, 0, 0, 0, 30, 0, 8, 0, 0, 0, 4096, 0, 3451, 1, 0, 0, 0, 0, 814, 0, 4080, 0, 2, 0, 0, 0, 0, 0, 0, 5, 0, 53, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 22, 3, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 147, 4069, 0, 0, 0, 4096, 0, 27, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 1109, 0, 0, 0, 140, 5, 3935, 0, 2892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3990, 0, 0, 0, 0, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[52, 0, 4027, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 808, 0, 0, 0, 0, 0, 1389, 0, 4022, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1679, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 306, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 44, 3714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 17, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 899, 0, 1, 0, 0, 3619, 0, 366, 243, 91, 0, 0, 0, 0, 0, 0, 0, 19, 0, 157, 1, 0, 3807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 3, 0, 16, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 1, 5, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3758, 4125, 4072, 3389, 5219, 3984, 4150, 4071] avg=4096.0000 max=5219 max_rank=4 min=3389 min_rank=3 imbalance_ratio=1.274170 max_load_reduction=27549 max_load_reduction_pct=0.840729 moved_to_echo=17424 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3728, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 17, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 367, 246, 105, 0, 0, 0, 0, 0, 0, 24, 0, 0, 23, 0, 0, 3850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3729, 4132, 4097, 3924, 4814, 3897, 4097, 4078] avg=4096.0000 max=4814 max_rank=4 min=3729 min_rank=0 imbalance_ratio=1.175293 max_load_reduction=27954 max_load_reduction_pct=0.853088 moved_to_echo=16886 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[59, 0, 3728, 13, 0, 2, 1959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 1, 0, 0, 0, 17, 1, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 4, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1944, 0, 0, 0, 0, 54, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3762, 0, 0, 31, 0, 0, 0, 367, 245, 9, 0, 0, 0, 0, 0, 0, 13, 0, 1, 34, 0, 0, 3838, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 5, 0, 0, 4073, 1, 0, 0, 0, 0, 0, 0, 2, 4076, 0, 0, 0, 5, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5761, 4055, 4112, 2359, 4414, 3886, 4082, 4099] avg=4096.0000 max=5761 max_rank=0 min=2359 min_rank=3 imbalance_ratio=1.406494 max_load_reduction=27007 max_load_reduction_pct=0.824188 moved_to_echo=16481 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 48, 3293, 18, 0, 0, 1100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 3, 0, 0, 0, 16, 13, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 2, 4044, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4012, 5, 0, 0, 3, 6, 3239, 0, 0, 0, 0, 0, 0, 0, 0, 41, 174, 0, 0, 62, 3, 0, 0, 324, 241, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3780, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 49, 4051, 1, 0, 0, 0, 0, 0, 0, 0, 4018, 1, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4459, 4107, 4064, 7265, 932, 3780, 4122, 4039] avg=4096.0000 max=7265 max_rank=3 min=932 min_rank=4 imbalance_ratio=1.773682 max_load_reduction=25503 max_load_reduction_pct=0.778290 moved_to_echo=12873 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3728, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 17, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3407, 0, 0, 0, 46, 0, 916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3755, 0, 0, 0, 0, 0, 0, 367, 246, 24, 0, 0, 0, 0, 0, 0, 22, 0, 0, 2, 0, 0, 3850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 1, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3757, 4116, 4097, 4369, 4392, 3874, 4089, 4074] avg=4096.0000 max=4392 max_rank=4 min=3757 min_rank=0 imbalance_ratio=1.072266 max_load_reduction=28376 max_load_reduction_pct=0.865967 moved_to_echo=16429 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[15, 0, 3675, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3934, 0, 0, 0, 0, 16, 3, 0, 0, 0, 0, 0, 0, 0, 30, 0, 8, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3451, 1, 0, 0, 0, 0, 790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 2, 0, 0, 0, 0, 362, 246, 21, 0, 0, 0, 0, 0, 0, 0, 5, 0, 53, 0, 0, 3843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 1, 0, 0, 0, 0, 0, 0, 0, 4025, 0, 22, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3718, 3953, 4135, 4242, 4711, 3901, 4058, 4050] avg=4096.0000 max=4711 max_rank=4 min=3718 min_rank=0 imbalance_ratio=1.150146 max_load_reduction=28057 max_load_reduction_pct=0.856232 moved_to_echo=16720 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3722, 0, 0, 0, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 27, 0, 1, 19, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 0, 0, 0, 0, 0, 0, 0, 1, 1109, 0, 0, 0, 140, 5, 3809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2892, 0, 0, 0, 0, 0, 0, 370, 250, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 3845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 4, 0, 0, 0, 0, 0, 0, 0, 3969, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3869, 4162, 4094, 5064, 3619, 3916, 4073, 3971] avg=4096.0000 max=5064 max_rank=3 min=3619 min_rank=4 imbalance_ratio=1.236328 max_load_reduction=27704 max_load_reduction_pct=0.845459 moved_to_echo=15579 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[52, 0, 3665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 17, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 808, 0, 0, 0, 0, 0, 1348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 0, 2, 0, 0, 0, 0, 361, 246, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1679, 1, 0, 3850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 306, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3717, 4118, 4097, 2156, 4667, 5530, 4097, 4386] avg=4096.0000 max=5530 max_rank=5 min=2156 min_rank=3 imbalance_ratio=1.350098 max_load_reduction=27238 max_load_reduction_pct=0.831238 moved_to_echo=18680 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 14848, 128, 0, 0, 256, 128, 17792, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 23680, 0, 128, 128, 128, 3712, 0, 2944, 2048, 512, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 29312, 128, 0, 128, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 2048, 128, 0, 30720, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 0, 0, 32640, 128, 128, 0, 128, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 0, 0, 128, 128, 128, 128, 32640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 32384, 128, 128, 128, 128, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2421, 0, 4095, 0, 0, 0, 1489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 232, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4067, 0, 4084] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 8, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 316, 0, 0, 2, 1, 0, 4088, 0, 0, 0, 3988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4087, 0, 0, 0, 175, 83, 0, 0, 0, 1, 0, 0, 0, 0, 3966, 0, 0, 4084, 4076, 7, 3731] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 7, 1, 0, 0, 1, 4049, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 55, 7, 0, 0, 0, 0, 11, 0, 1, 4085, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 46, 4088, 0, 0, 4094, 4092, 5, 4032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3839, 0, 4093, 0, 0, 1, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 48, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2, 8, 0, 14, 0, 106, 0, 0, 0, 0, 0, 2123, 0, 4061, 0, 0, 0, 3937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 66, 50, 0, 0, 1, 0, 0, 108, 0, 0, 1874, 0, 516, 3, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4093, 3974, 63, 3582] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 9, 3, 0, 0, 0, 0, 8, 0, 0, 0, 154, 0, 0, 0, 1159, 0, 3993, 0, 0, 0, 2140, 3, 38, 0, 3, 0, 0, 0, 25, 0, 4056, 0, 0, 0, 0, 0, 0, 55, 514, 0, 0, 3878, 0, 114, 0, 712, 0, 0, 0, 0, 0, 0, 87, 0, 0, 4034, 0, 0, 4096, 3994, 41, 3652] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 3570, 0, 4096, 0, 0, 0, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 7, 0, 16, 2, 0, 4, 1, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 4096, 0, 0, 4093, 4095, 0, 4086] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 4093, 15, 97, 0, 0, 0, 4009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 215, 0, 0, 0, 0, 0, 0, 0, 162, 0, 3926, 0, 8, 0, 1, 41, 0, 0, 0, 0, 0, 0, 131, 0, 4096, 0, 0, 4089, 3695, 0, 4088] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3870, 544, 43, 0, 0, 0, 0, 0, 1877, 0, 0, 0, 0, 0, 1489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 232, 0, 0, 4092, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3903, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4457, 3366, 4091, 4045, 4332, 4128, 4238] avg=4096.0000 max=4457 max_rank=1 min=3366 min_rank=2 imbalance_ratio=1.088135 max_load_reduction=28311 max_load_reduction_pct=0.863983 moved_to_echo=16743 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 0, 37, 0, 0, 0, 0, 3966, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 316, 0, 0, 2, 3864, 0, 43, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 4045, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4087, 0, 0, 0, 175, 8, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 1, 0, 0, 0, 3911, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 7, 3731, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4026, 4225, 3989, 4079, 4060, 4277, 4219, 3893] avg=4096.0000 max=4277 max_rank=5 min=3893 min_rank=7 imbalance_ratio=1.044189 max_load_reduction=28491 max_load_reduction_pct=0.869476 moved_to_echo=16449 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 11, 4088, 10, 4, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1, 0, 0, 1, 3829, 911, 43, 0, 0, 0, 0, 0, 3138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 55, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 1, 4085, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 3927, 223, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 155, 5, 4032, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4792, 3138, 4090, 4104, 4105, 4188, 4238] avg=4096.0000 max=4792 max_rank=1 min=3138 min_rank=2 imbalance_ratio=1.169922 max_load_reduction=27976 max_load_reduction_pct=0.853760 moved_to_echo=16635 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2, 8, 0, 4096, 10, 5, 0, 0, 0, 0, 0, 14, 0, 106, 0, 0, 0, 0, 0, 3838, 477, 43, 0, 0, 0, 0, 0, 1646, 0, 0, 0, 0, 0, 3937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 4039, 66, 50, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 0, 0, 1874, 0, 516, 3, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3813, 223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 63, 3582, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4478, 5583, 4088, 4156, 2509, 4037, 3796] avg=4096.0000 max=5583 max_rank=2 min=2509 min_rank=5 imbalance_ratio=1.363037 max_load_reduction=27185 max_load_reduction_pct=0.829620 moved_to_echo=14498 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 9, 3, 0, 0, 0, 0, 4034, 10, 5, 0, 0, 0, 0, 0, 8, 0, 0, 0, 154, 0, 0, 0, 3775, 260, 43, 0, 0, 0, 0, 0, 899, 0, 0, 0, 0, 0, 2140, 3, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 3, 0, 0, 0, 25, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4005, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 514, 0, 0, 3878, 0, 114, 0, 712, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 3833, 218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 41, 3652, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4240, 3042, 4157, 4060, 5226, 4138, 3844] avg=4096.0000 max=5226 max_rank=5 min=3042 min_rank=2 imbalance_ratio=1.275879 max_load_reduction=27542 max_load_reduction_pct=0.840515 moved_to_echo=17268 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3869, 864, 43, 0, 0, 0, 0, 0, 2975, 0, 0, 0, 0, 1, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4096, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3931, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 48, 4094, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4776, 3183, 4091, 4045, 4110, 4155, 4297] avg=4096.0000 max=4776 max_rank=1 min=3183 min_rank=2 imbalance_ratio=1.166016 max_load_reduction=27992 max_load_reduction_pct=0.854248 moved_to_echo=16607 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 14, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 3872, 807, 47, 0, 0, 0, 0, 0, 2763, 0, 0, 0, 0, 0, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 4013, 0, 0, 7, 0, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 4084, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 3933, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4738, 3271, 4088, 4038, 4104, 4180, 4234] avg=4096.0000 max=4738 max_rank=1 min=3271 min_rank=2 imbalance_ratio=1.156738 max_load_reduction=28030 max_load_reduction_pct=0.855408 moved_to_echo=16556 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 92, 921, 43, 0, 0, 0, 0, 0, 3172, 15, 0, 0, 0, 0, 4009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 4045, 215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 0, 3926, 0, 8, 0, 1, 8, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 131, 3546, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 1062, 7196, 4084, 4260, 4105, 3723, 4228] avg=4096.0000 max=7196 max_rank=2 min=1062 min_rank=1 imbalance_ratio=1.756836 max_load_reduction=25572 max_load_reduction_pct=0.780396 moved_to_echo=16316 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[16512, 128, 0, 0, 0, 128, 16384, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 512, 0, 128, 128, 27136, 4864, 384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[896, 256, 128, 30208, 0, 640, 128, 896, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 0, 0, 1280, 256, 31360, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[0, 128, 128, 128, 0, 128, 128, 128, 32640, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 0, 128, 0, 128, 256, 30848, 1664, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[32384, 384, 128, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 0, 0, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 8, 4008, 529, 0, 0, 0, 4026, 0, 0, 0, 0, 4, 2356, 1, 0, 14, 4072, 0, 0, 1924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3685, 0, 0, 0, 0, 0, 4096, 0, 4090, 15, 0, 0, 3824, 0, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3962, 0, 0, 0, 0, 0, 1179, 0, 17, 0, 4080, 0, 0, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3047, 0, 216, 0, 0, 0, 21, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 4096, 0, 4083, 0, 0, 0, 3723, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3, 0, 18, 3877, 121, 0, 0, 0, 3884, 0, 0, 6, 0, 0, 0, 0, 11, 0, 3579, 0, 0, 3171, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 1924, 0, 64, 2, 0, 0, 642, 0, 5, 0, 0, 4084, 0, 0, 0, 1, 0, 4071, 0, 4096, 0, 0, 13, 3166, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3, 0, 4086, 2626, 0, 0, 0, 4063, 26, 0, 0, 0, 0, 4050, 0, 0, 0, 4095, 0, 0, 361, 0, 60, 0, 0, 14, 0, 0, 0, 2, 0, 0, 815, 0, 0, 6, 3, 0, 0, 1, 0, 11, 0, 353, 0, 0, 4, 0, 0, 4086, 0, 4010, 1, 0, 0, 4092, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 72, 0, 0, 0, 3579, 0, 0, 0, 0, 0, 583, 0, 2, 0, 4094, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 4023, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 36, 0, 0, 0, 4091, 109, 0, 0, 0, 310, 0, 0, 0, 0, 0, 499, 0, 0, 0, 3978, 1, 0, 3683, 1, 129, 0, 0, 0, 0, 0, 0, 0, 1, 9, 3775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 4094, 0, 4089, 7, 0, 0, 3860, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 13, 9, 3, 10, 0, 2, 0, 2409, 372, 0, 0, 1, 17, 0, 0, 1, 0, 28, 70, 0, 0, 1, 4076, 0, 1, 4095, 4, 240, 0, 0, 154, 0, 170, 0, 0, 0, 0, 4063, 29, 0, 0, 0, 0, 0, 8, 0, 1, 60, 4027, 0, 812, 0, 0, 0, 4084, 0, 3920, 16, 0, 0, 4071, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4091, 3, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 28, 0, 4, 0, 4095, 0, 0, 3580, 0, 0, 0, 0, 10, 4, 0, 0, 0, 0, 0, 3148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1447, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 0, 4081, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 8, 3854, 258, 0, 0, 0, 0, 0, 0, 651, 529, 0, 0, 0, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2356, 1, 0, 14, 3037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1924, 0, 0, 0, 0, 0, 1966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 107, 0, 1035, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3079, 937, 0, 0, 0, 0, 0, 0, 0, 3685, 0, 0, 0, 0, 0, 759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236, 15, 0, 0, 3824, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 5206, 5412, 3890, 1602, 4016, 4444, 4078] avg=4096.0000 max=5412 max_rank=2 min=1602 min_rank=4 imbalance_ratio=1.321289 max_load_reduction=27356 max_load_reduction_pct=0.834839 moved_to_echo=14140 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 3847, 258, 0, 0, 0, 0, 0, 0, 665, 0, 0, 0, 0, 3962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1179, 0, 17, 0, 3043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 0, 0, 0, 0, 0, 2010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3047, 0, 1037, 464, 0, 0, 0, 0, 0, 0, 216, 0, 0, 0, 21, 0, 0, 0, 3079, 957, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236, 0, 0, 0, 3723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4627, 4239, 2182, 4548, 4273, 4835, 3959] avg=4096.0000 max=4835 max_rank=6 min=2182 min_rank=3 imbalance_ratio=1.180420 max_load_reduction=27933 max_load_reduction_pct=0.852448 moved_to_echo=17615 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3, 0, 18, 3860, 256, 0, 0, 0, 0, 0, 0, 629, 121, 0, 0, 0, 3884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 11, 0, 2669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3171, 8, 0, 0, 0, 0, 1902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 1924, 0, 910, 440, 0, 0, 0, 0, 0, 0, 64, 2, 0, 0, 642, 0, 5, 0, 3060, 906, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 1, 0, 755, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236, 0, 0, 13, 3166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4137, 4634, 2686, 5081, 3296, 4679, 4840, 3415] avg=4096.0000 max=5081 max_rank=3 min=2686 min_rank=2 imbalance_ratio=1.240479 max_load_reduction=27687 max_load_reduction_pct=0.844940 moved_to_echo=16230 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3, 0, 3778, 257, 0, 0, 0, 0, 0, 0, 663, 2626, 0, 0, 0, 4063, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 0, 3054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 361, 0, 60, 0, 0, 14, 2005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 815, 0, 1041, 463, 0, 0, 0, 0, 0, 0, 0, 6, 3, 0, 0, 1, 0, 11, 3071, 955, 0, 0, 0, 0, 0, 0, 0, 353, 0, 0, 4, 0, 0, 758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 232, 1, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4038, 7378, 7104, 2440, 2321, 4047, 1115, 4325] avg=4096.0000 max=7378 max_rank=1 min=1115 min_rank=6 imbalance_ratio=1.801270 max_load_reduction=25390 max_load_reduction_pct=0.774841 moved_to_echo=11808 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 13, 9, 3, 10, 0, 2, 0, 3694, 257, 0, 0, 0, 0, 0, 0, 391, 372, 0, 0, 1, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 28, 70, 0, 0, 1, 3040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 4, 240, 0, 0, 154, 1182, 0, 0, 0, 0, 0, 0, 0, 0, 170, 0, 0, 0, 0, 4063, 29, 1036, 273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 1, 3070, 563, 0, 0, 0, 0, 0, 0, 60, 4027, 0, 812, 0, 0, 0, 757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 226, 16, 0, 0, 4071, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3988, 781, 3140, 5676, 5571, 3642, 5656, 4314] avg=4096.0000 max=5676 max_rank=3 min=781 min_rank=1 imbalance_ratio=1.385742 max_load_reduction=27092 max_load_reduction_pct=0.826782 moved_to_echo=19183 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 3857, 258, 0, 0, 0, 0, 0, 0, 665, 72, 0, 0, 0, 3579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 583, 0, 2, 0, 3053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 0, 2010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4000, 0, 1041, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3079, 957, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 0, 759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236, 0, 0, 0, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4316, 3638, 2078, 5505, 4036, 4821, 4259] avg=4096.0000 max=5505 max_rank=4 min=2078 min_rank=3 imbalance_ratio=1.343994 max_load_reduction=27263 max_load_reduction_pct=0.832001 moved_to_echo=18621 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 36, 0, 0, 0, 3856, 259, 0, 0, 0, 0, 0, 0, 651, 109, 0, 0, 0, 310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 0, 0, 0, 2964, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3683, 1, 129, 0, 0, 0, 2010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 9, 3775, 0, 1014, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3079, 961, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 233, 7, 0, 0, 3860, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4151, 1070, 3463, 5824, 5268, 4040, 4851, 4101] avg=4096.0000 max=5824 max_rank=3 min=1070 min_rank=1 imbalance_ratio=1.421875 max_load_reduction=26944 max_load_reduction_pct=0.822266 moved_to_echo=18260 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 3856, 258, 0, 0, 0, 0, 0, 0, 664, 3, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 4, 0, 3054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3580, 0, 0, 0, 0, 10, 2007, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 3148, 0, 1041, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3079, 956, 0, 0, 0, 0, 0, 0, 0, 1447, 0, 0, 0, 0, 0, 759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4756, 3086, 5597, 4657, 4035, 2206, 4317] avg=4096.0000 max=5597 max_rank=3 min=2206 min_rank=6 imbalance_ratio=1.366455 max_load_reduction=27171 max_load_reduction_pct=0.829193 moved_to_echo=15215 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 256, 0, 128, 128, 128, 20992, 128, 8192, 3584, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4992, 3840, 0, 0, 128, 23936, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 17152, 128, 512, 0, 0, 256, 15104, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[384, 128, 128, 0, 768, 128, 128, 128, 24704, 7296, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 25856, 0, 896, 128, 128, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 128, 8832, 128, 128, 128, 23936, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 1920, 128, 0, 128, 30848, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 128, 128, 128, 128, 128, 30720, 2176, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33792 slack_slots=31744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33792 slack_slots=31744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[4091, 45, 1, 0, 0, 1417, 0, 10, 8, 122, 3063, 0, 0, 0, 98, 0, 0, 3944, 4045, 974, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 3841, 0, 0, 0, 0, 7, 0, 8, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 9, 14, 9, 2764, 4095, 0, 0, 107, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[552, 0, 1720, 0, 0, 790, 0, 0, 0, 0, 1932, 3, 0, 0, 0, 0, 0, 4090, 4095, 68, 0, 5, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 12, 3010, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 4088, 4096, 4032, 0, 0, 0, 0, 14, 0, 0, 0, 146] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1475, 0, 3953, 0, 0, 0, 0, 0, 0, 0, 2170, 0, 0, 0, 0, 0, 0, 4096, 3899, 90, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 754, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4039, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[4084, 0, 3160, 0, 0, 3, 1, 4, 0, 176, 3963, 0, 1, 5, 0, 0, 0, 4077, 4093, 15, 0, 0, 0, 0, 60, 0, 0, 0, 4091, 0, 0, 0, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 2, 0, 0, 3906, 4091, 2, 0, 0, 8, 0, 0, 0, 2, 13] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[71, 3, 3089, 517, 0, 0, 12, 0, 0, 346, 2366, 5, 24, 0, 0, 36, 0, 4051, 3757, 2, 0, 0, 9, 0, 72, 0, 0, 0, 4082, 159, 0, 1, 4025, 0, 0, 0, 0, 0, 764, 0, 13, 0, 0, 0, 0, 0, 2, 47, 0, 0, 1, 1, 1010, 3501, 4056, 0, 3, 1, 0, 679, 0, 0, 0, 63] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[4051, 0, 1738, 0, 0, 0, 0, 0, 0, 244, 1438, 0, 0, 0, 0, 0, 0, 4096, 4089, 112, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3957, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 746, 4094, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[93, 0, 158, 0, 1, 61, 0, 0, 0, 0, 3933, 0, 0, 0, 0, 0, 0, 4028, 4096, 0, 0, 1, 0, 3, 0, 0, 1, 0, 4093, 0, 1, 5, 4086, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 4054, 4091, 0, 0, 3, 0, 27, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[2710, 0, 21, 6, 0, 1815, 0, 0, 0, 0, 4041, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4086, 6, 0, 0, 0, 0, 124, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3798, 666, 2960, 0, 0, 0, 0, 2, 0, 0, 0, 41] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3403, 45, 1, 0, 0, 1417, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 8, 122, 3063, 0, 0, 0, 98, 0, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3960, 974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3841, 0, 0, 0, 0, 7, 0, 8, 311, 688, 85, 32, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 3944, 22, 0, 0, 0, 0, 0, 0, 0, 0, 9, 14, 9, 1526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 0, 0, 0, 0, 0, 0, 4073, 31, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4876, 4187, 4934, 4036, 4972, 3994, 1558, 4211] avg=4096.0000 max=4972 max_rank=4 min=1558 min_rank=6 imbalance_ratio=1.213867 max_load_reduction=27796 max_load_reduction_pct=0.848267 moved_to_echo=14735 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1227, 0, 3953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2170, 0, 0, 0, 0, 0, 1309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3817, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 754, 0, 0, 0, 0, 0, 2, 0, 454, 248, 82, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4075, 46, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5180, 3479, 3907, 4064, 1572, 4117, 6326, 4123] avg=4096.0000 max=6326 max_rank=6 min=1572 min_rank=4 imbalance_ratio=1.544434 max_load_reduction=26442 max_load_reduction_pct=0.806946 moved_to_echo=16138 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[460, 0, 1720, 0, 0, 790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1932, 3, 0, 0, 0, 0, 1328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 68, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 3010, 0, 0, 0, 0, 1, 0, 3, 461, 92, 86, 32, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 4090, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 2261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 146, 4011, 46, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2970, 3263, 4082, 4076, 3685, 4126, 6349, 4217] avg=4096.0000 max=6349 max_rank=6 min=2970 min_rank=0 imbalance_ratio=1.550049 max_load_reduction=26419 max_load_reduction_pct=0.806244 moved_to_echo=18377 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[60, 3, 3089, 517, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 346, 2366, 5, 24, 0, 0, 36, 1135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3678, 2, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 4050, 159, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 0, 0, 0, 0, 0, 764, 0, 394, 11, 79, 32, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 2, 47, 4051, 22, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1010, 1932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 679, 0, 0, 0, 63, 4034, 40, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3681, 3912, 3689, 4282, 5305, 4135, 2944, 4820] avg=4096.0000 max=5305 max_rank=4 min=2944 min_rank=6 imbalance_ratio=1.295166 max_load_reduction=27463 max_load_reduction_pct=0.838104 moved_to_echo=17204 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3397, 0, 3160, 0, 0, 3, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 3963, 0, 1, 5, 0, 0, 1266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4007, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 0, 0, 0, 0, 0, 0, 0, 439, 687, 86, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4077, 21, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2157, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 2, 13, 4070, 44, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6565, 5411, 4022, 4119, 2250, 4103, 2161, 4137] avg=4096.0000 max=6565 max_rank=0 min=2161 min_rank=6 imbalance_ratio=1.602783 max_load_reduction=26203 max_load_reduction_pct=0.799652 moved_to_echo=12651 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3369, 0, 1738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 244, 1438, 0, 0, 0, 0, 0, 1327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4003, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3957, 0, 0, 0, 0, 0, 0, 0, 460, 682, 86, 32, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 4096, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 746, 2261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 46, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5107, 3009, 4115, 4064, 5217, 4132, 3007, 4117] avg=4096.0000 max=5217 max_rank=4 min=3007 min_rank=6 imbalance_ratio=1.273682 max_load_reduction=27551 max_load_reduction_pct=0.840790 moved_to_echo=16473 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[73, 0, 158, 0, 1, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3933, 0, 0, 0, 0, 0, 1318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4005, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4055, 0, 1, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 4, 460, 20, 91, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4028, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 2226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 27, 0, 0, 0, 0, 4069, 50, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[293, 5251, 4009, 4062, 4699, 4050, 6255, 4149] avg=4096.0000 max=6255 max_rank=6 min=293 min_rank=0 imbalance_ratio=1.527100 max_load_reduction=26513 max_load_reduction_pct=0.809113 moved_to_echo=19153 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2254, 0, 21, 6, 0, 1815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 0, 0, 0, 0, 0, 215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 6, 0, 0, 0, 0, 124, 204, 74, 456, 86, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3798, 370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 41, 2944, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4256, 4010, 4065, 5068, 4111, 4168, 2994] avg=4096.0000 max=5068 max_rank=4 min=2994 min_rank=7 imbalance_ratio=1.237305 max_load_reduction=27700 max_load_reduction_pct=0.845337 moved_to_echo=16341 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[14336, 128, 13952, 640, 128, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 128, 128, 17792, 14976, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[24832, 128, 0, 0, 0, 128, 896, 256, 3072, 2944, 768, 384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 31616, 1280, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 128, 128, 128, 128, 32512, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 768, 0, 0, 128, 384, 31360, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[256, 0, 128, 0, 32512, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 896, 22912, 128, 128, 128, 128, 128, 8832, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True + [2026-09-21 08:24:25] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 269.9 | throughput per GPU (TFLOP/s/GPU): 119.4 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.077660E+01 | loss scale: 1.0 | grad norm: 21.368 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1012, 595, 250, 4, 1, 1002, 1275, 155, 0, 1353, 115, 0, 18, 75, 93, 245, 20, 328, 340, 1388, 10, 258, 302, 48, 307, 61, 36, 637, 759, 1474, 309, 1044, 185, 246, 719, 437, 144, 1866, 471, 907, 814, 1239, 1533, 11, 2339, 1229, 5, 253, 763, 484, 986, 43, 1025, 783, 458, 37, 1420, 7, 499, 10, 48, 61, 210, 22] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1432, 602, 204, 3, 25, 939, 1149, 234, 16, 2010, 15, 2, 17, 71, 218, 106, 50, 497, 400, 1579, 21, 164, 251, 12, 333, 37, 35, 515, 873, 972, 286, 898, 242, 225, 621, 783, 125, 1616, 271, 791, 399, 727, 1310, 13, 2234, 1362, 21, 195, 606, 744, 967, 58, 1174, 1062, 709, 26, 1308, 18, 362, 6, 9, 161, 595, 62] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1299, 486, 235, 2, 4, 784, 1405, 105, 7, 1545, 43, 1, 72, 62, 235, 352, 82, 862, 762, 1586, 21, 234, 300, 15, 243, 20, 60, 310, 968, 897, 474, 1113, 360, 217, 708, 361, 169, 1561, 366, 691, 655, 1015, 1227, 53, 1600, 824, 45, 330, 832, 818, 851, 39, 956, 967, 775, 198, 992, 17, 532, 45, 10, 176, 711, 83] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1122, 589, 116, 7, 9, 944, 1284, 135, 14, 1609, 14, 1, 56, 229, 101, 266, 2, 666, 500, 1680, 13, 381, 164, 38, 537, 32, 30, 358, 908, 1093, 500, 841, 121, 154, 779, 416, 80, 1624, 367, 943, 446, 1112, 910, 28, 2022, 1569, 28, 254, 995, 629, 842, 26, 1082, 961, 491, 61, 1578, 44, 433, 23, 16, 117, 355, 23] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1117, 567, 178, 6, 6, 576, 1369, 220, 16, 1619, 98, 0, 28, 67, 362, 248, 35, 634, 717, 2022, 22, 353, 147, 10, 169, 26, 48, 417, 764, 974, 303, 1086, 229, 236, 501, 302, 111, 1807, 364, 774, 572, 979, 1210, 30, 1786, 1235, 32, 397, 951, 884, 1006, 14, 773, 1172, 491, 81, 1163, 22, 674, 8, 28, 42, 660, 30] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1116, 660, 355, 57, 10, 1002, 1451, 130, 18, 2412, 37, 2, 14, 146, 108, 152, 56, 173, 290, 1440, 17, 301, 244, 86, 303, 125, 27, 565, 629, 1317, 228, 675, 100, 148, 581, 476, 67, 1241, 294, 594, 1140, 823, 1229, 40, 2239, 1580, 24, 175, 751, 563, 956, 121, 942, 1462, 807, 80, 1001, 10, 552, 13, 54, 101, 432, 26] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[673, 598, 168, 16, 8, 768, 1321, 154, 6, 1959, 77, 0, 5, 214, 110, 654, 12, 610, 472, 1363, 38, 707, 302, 11, 388, 15, 51, 250, 819, 744, 492, 787, 214, 202, 814, 394, 171, 1962, 443, 785, 552, 966, 1202, 21, 2378, 1365, 5, 429, 958, 814, 955, 21, 668, 1212, 793, 96, 767, 7, 396, 3, 29, 36, 205, 113] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1341, 636, 312, 0, 10, 558, 1555, 125, 69, 1974, 21, 5, 11, 172, 187, 330, 77, 796, 468, 1367, 13, 289, 297, 28, 219, 56, 29, 421, 553, 831, 511, 1119, 365, 215, 611, 564, 228, 1216, 138, 720, 407, 1132, 972, 65, 2514, 1511, 31, 231, 784, 604, 826, 136, 1081, 956, 405, 49, 1390, 21, 218, 37, 7, 99, 828, 27] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1122, 589, 116, 7, 9, 944, 1083, 135, 0, 0, 0, 0, 0, 0, 0, 0, 14, 1609, 14, 1, 56, 229, 101, 266, 961, 389, 119, 0, 0, 0, 0, 0, 2, 666, 500, 1680, 13, 381, 164, 38, 335, 349, 160, 0, 0, 0, 0, 0, 537, 32, 30, 358, 908, 1093, 500, 841, 72, 41, 0, 0, 0, 0, 0, 0, 121, 154, 779, 416, 80, 1289, 367, 943, 0, 0, 0, 0, 0, 0, 0, 0, 446, 1112, 910, 28, 0, 1101, 28, 254, 0, 0, 0, 0, 0, 0, 0, 0, 995, 629, 842, 26, 1010, 0, 491, 61, 0, 0, 0, 0, 0, 0, 0, 0, 1578, 44, 433, 23, 16, 117, 355, 23, 1633, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4005, 3759, 4288, 4412, 4149, 3879, 4054, 4222] avg=4096.0000 max=4412 max_rank=3 min=3759 min_rank=1 imbalance_ratio=1.077148 max_load_reduction=28356 max_load_reduction_pct=0.865356 moved_to_echo=16304 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1012, 595, 250, 4, 1, 1002, 1075, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1353, 115, 0, 18, 75, 93, 245, 783, 450, 93, 0, 0, 0, 0, 0, 20, 328, 340, 1388, 10, 258, 302, 48, 385, 273, 159, 0, 0, 0, 0, 0, 307, 61, 36, 637, 759, 1474, 309, 1044, 68, 41, 0, 0, 0, 0, 0, 0, 185, 246, 719, 437, 144, 1481, 471, 907, 0, 0, 0, 0, 0, 0, 0, 0, 814, 1239, 1533, 11, 0, 863, 5, 253, 0, 0, 0, 0, 0, 0, 0, 0, 763, 484, 986, 43, 957, 0, 458, 37, 0, 0, 0, 0, 0, 0, 0, 0, 1420, 7, 499, 10, 48, 61, 210, 22, 1889, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 3225, 3511, 4736, 4590, 4718, 3728, 4166] avg=4096.0000 max=4736 max_rank=3 min=3225 min_rank=1 imbalance_ratio=1.156250 max_load_reduction=28032 max_load_reduction_pct=0.855469 moved_to_echo=17202 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1432, 602, 204, 3, 25, 939, 969, 234, 0, 0, 0, 0, 0, 0, 0, 0, 16, 2010, 15, 2, 17, 71, 218, 106, 1062, 430, 103, 0, 0, 0, 0, 0, 50, 497, 400, 1579, 21, 164, 251, 12, 334, 303, 143, 0, 0, 0, 0, 0, 333, 37, 35, 515, 873, 972, 286, 898, 78, 37, 0, 0, 0, 0, 0, 0, 242, 225, 621, 783, 125, 1282, 271, 791, 0, 0, 0, 0, 0, 0, 0, 0, 399, 727, 1310, 13, 0, 956, 21, 195, 0, 0, 0, 0, 0, 0, 0, 0, 606, 744, 967, 58, 1096, 0, 709, 26, 0, 0, 0, 0, 0, 0, 0, 0, 1308, 18, 362, 6, 9, 161, 595, 62, 1804, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4408, 4050, 3754, 4064, 4340, 3621, 4206, 4325] avg=4096.0000 max=4408 max_rank=0 min=3621 min_rank=5 imbalance_ratio=1.076172 max_load_reduction=28360 max_load_reduction_pct=0.865479 moved_to_echo=16492 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1299, 486, 235, 2, 4, 784, 1185, 105, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1545, 43, 1, 72, 62, 235, 352, 967, 309, 62, 0, 0, 0, 0, 0, 82, 862, 762, 1586, 21, 234, 300, 15, 322, 183, 175, 0, 0, 0, 0, 0, 243, 20, 60, 310, 968, 897, 474, 1113, 64, 45, 0, 0, 0, 0, 0, 0, 360, 217, 708, 361, 169, 1239, 366, 691, 0, 0, 0, 0, 0, 0, 0, 0, 655, 1015, 1227, 53, 0, 579, 45, 330, 0, 0, 0, 0, 0, 0, 0, 0, 832, 818, 851, 39, 892, 0, 775, 198, 0, 0, 0, 0, 0, 0, 0, 0, 992, 17, 532, 45, 10, 176, 711, 83, 1291, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 3655, 4542, 4194, 4111, 3904, 4405, 3857] avg=4096.0000 max=4542 max_rank=2 min=3655 min_rank=1 imbalance_ratio=1.108887 max_load_reduction=28226 max_load_reduction_pct=0.861389 moved_to_echo=16277 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[673, 598, 168, 16, 8, 768, 1114, 154, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1959, 77, 0, 5, 214, 110, 654, 1212, 458, 104, 0, 0, 0, 0, 0, 12, 610, 472, 1363, 38, 707, 302, 11, 405, 304, 165, 0, 0, 0, 0, 0, 388, 15, 51, 250, 819, 744, 492, 787, 44, 42, 0, 0, 0, 0, 0, 0, 214, 202, 814, 394, 171, 1557, 443, 785, 0, 0, 0, 0, 0, 0, 0, 0, 552, 966, 1202, 21, 0, 957, 5, 429, 0, 0, 0, 0, 0, 0, 0, 0, 958, 814, 955, 21, 624, 0, 793, 96, 0, 0, 0, 0, 0, 0, 0, 0, 767, 7, 396, 3, 29, 36, 205, 113, 1920, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3499, 4799, 4389, 3632, 4580, 4132, 4261, 3476] avg=4096.0000 max=4799 max_rank=1 min=3476 min_rank=7 imbalance_ratio=1.171631 max_load_reduction=27969 max_load_reduction_pct=0.853546 moved_to_echo=16449 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1117, 567, 178, 6, 6, 576, 1154, 220, 0, 0, 0, 0, 0, 0, 0, 0, 16, 1619, 98, 0, 28, 67, 362, 248, 1172, 344, 94, 0, 0, 0, 0, 0, 35, 634, 717, 2022, 22, 353, 147, 10, 373, 275, 171, 0, 0, 0, 0, 0, 169, 26, 48, 417, 764, 974, 303, 1086, 51, 44, 0, 0, 0, 0, 0, 0, 229, 236, 501, 302, 111, 1434, 364, 774, 0, 0, 0, 0, 0, 0, 0, 0, 572, 979, 1210, 30, 0, 866, 32, 397, 0, 0, 0, 0, 0, 0, 0, 0, 951, 884, 1006, 14, 722, 0, 491, 81, 0, 0, 0, 0, 0, 0, 0, 0, 1163, 22, 674, 8, 28, 42, 660, 30, 1442, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3824, 4048, 4759, 3882, 3951, 4086, 4149, 4069] avg=4096.0000 max=4759 max_rank=2 min=3824 min_rank=0 imbalance_ratio=1.161865 max_load_reduction=28009 max_load_reduction_pct=0.854767 moved_to_echo=16255 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1116, 660, 355, 57, 10, 1002, 1215, 130, 0, 0, 0, 0, 0, 0, 0, 0, 18, 2412, 37, 2, 14, 146, 108, 152, 1462, 432, 125, 0, 0, 0, 0, 0, 56, 173, 290, 1440, 17, 301, 244, 86, 260, 356, 186, 0, 0, 0, 0, 0, 303, 125, 27, 565, 629, 1317, 228, 675, 68, 50, 0, 0, 0, 0, 0, 0, 100, 148, 581, 476, 67, 981, 294, 594, 0, 0, 0, 0, 0, 0, 0, 0, 1140, 823, 1229, 40, 0, 1099, 24, 175, 0, 0, 0, 0, 0, 0, 0, 0, 751, 563, 956, 121, 874, 0, 807, 80, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 10, 552, 13, 54, 101, 432, 26, 1807, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4545, 4908, 3409, 3987, 3241, 4530, 4152, 3996] avg=4096.0000 max=4908 max_rank=1 min=3241 min_rank=4 imbalance_ratio=1.198242 max_load_reduction=27860 max_load_reduction_pct=0.850220 moved_to_echo=15919 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1341, 636, 312, 0, 10, 558, 1311, 125, 0, 0, 0, 0, 0, 0, 0, 0, 69, 1974, 21, 5, 11, 172, 187, 330, 956, 484, 115, 0, 0, 0, 0, 0, 77, 796, 468, 1367, 13, 289, 297, 28, 251, 336, 194, 0, 0, 0, 0, 0, 219, 56, 29, 421, 553, 831, 511, 1119, 72, 50, 0, 0, 0, 0, 0, 0, 365, 215, 611, 564, 228, 965, 138, 720, 0, 0, 0, 0, 0, 0, 0, 0, 407, 1132, 972, 65, 0, 1060, 31, 231, 0, 0, 0, 0, 0, 0, 0, 0, 784, 604, 826, 136, 1009, 0, 405, 49, 0, 0, 0, 0, 0, 0, 0, 0, 1390, 21, 218, 37, 7, 99, 828, 27, 2030, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4293, 4324, 4116, 3861, 3806, 3898, 3813, 4657] avg=4096.0000 max=4657 max_rank=7 min=3806 min_rank=4 imbalance_ratio=1.136963 max_load_reduction=28111 max_load_reduction_pct=0.857880 moved_to_echo=16174 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[6656, 5632, 7424, 512, 7296, 0, 4992, 640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[2560, 384, 384, 3584, 6400, 8320, 3200, 7680, 640, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[384, 4608, 3968, 12544, 256, 2688, 2048, 256, 2688, 2432, 1408, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[9216, 4736, 1920, 128, 128, 6656, 9216, 1280, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4992, 8064, 9600, 384, 0, 7552, 256, 2304, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1920, 1664, 5376, 3840, 1152, 10240, 2816, 6272, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[256, 14592, 512, 128, 256, 1152, 1536, 2432, 8576, 3328, 896, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[9728, 256, 3712, 256, 256, 896, 4096, 512, 13824, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1, 241, 4025, 1, 0, 1, 0, 0, 1, 1, 5, 88, 4077, 0, 781, 0, 3577, 0, 110, 2, 0, 877, 0, 1, 0, 66, 1, 0, 1101, 0, 7, 2895, 188, 17, 0, 0, 0, 0, 69, 3939, 285, 3413, 1527, 0, 65, 0, 0, 1, 421, 3546, 0, 0, 0, 0, 0, 217, 400, 0, 71, 0, 746, 0, 4, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 3974, 0, 0, 0, 0, 0, 73, 0, 0, 82, 3632, 0, 3, 1, 1328, 0, 435, 2, 0, 132, 18, 1, 0, 0, 0, 0, 3721, 0, 71, 3315, 137, 0, 0, 0, 0, 0, 155, 3666, 80, 4019, 1709, 0, 285, 0, 1, 0, 10, 3176, 0, 0, 8, 0, 6, 162, 744, 1, 1755, 0, 66, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 15, 3651, 0, 0, 66, 1, 0, 4, 0, 112, 59, 4083, 0, 1, 0, 3023, 0, 514, 2, 0, 34, 228, 75, 0, 11, 0, 0, 3461, 0, 3, 442, 1262, 0, 0, 0, 0, 0, 301, 3968, 100, 4062, 1284, 0, 16, 0, 5, 0, 174, 3475, 4, 2, 0, 0, 0, 170, 1332, 0, 780, 1, 47, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1, 44, 3289, 2, 1, 12, 21, 0, 33, 15, 1, 1887, 3865, 0, 212, 23, 621, 1, 413, 65, 1336, 2921, 27, 71, 0, 107, 185, 45, 795, 65, 57, 284, 95, 85, 7, 75, 2, 44, 52, 3730, 552, 4061, 107, 69, 1799, 3, 0, 0, 7, 3528, 160, 176, 0, 2, 0, 110, 368, 4, 771, 0, 515, 23, 23, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 1, 3944, 0, 0, 13, 0, 21, 55, 0, 158, 270, 4017, 7, 610, 0, 1712, 0, 34, 2, 14, 1901, 0, 0, 0, 2, 0, 166, 1599, 3, 0, 2094, 93, 1, 1, 0, 0, 0, 176, 3870, 963, 4064, 670, 0, 4, 0, 52, 0, 10, 3943, 1, 0, 0, 0, 0, 1823, 285, 0, 169, 0, 20, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 509, 3746, 0, 0, 0, 2, 0, 0, 4, 0, 1126, 4082, 1, 2, 0, 920, 17, 131, 24, 0, 35, 10, 72, 4, 3, 0, 0, 3557, 0, 129, 3583, 54, 0, 0, 0, 0, 0, 153, 4091, 27, 3935, 829, 0, 74, 0, 9, 0, 81, 2891, 0, 3, 1, 0, 0, 18, 1719, 0, 884, 4, 37, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[338, 1, 4076, 0, 0, 0, 4, 0, 9, 3, 3, 5, 4095, 0, 107, 0, 648, 0, 757, 0, 0, 1627, 16, 456, 0, 284, 0, 198, 2461, 4, 0, 3453, 164, 0, 0, 0, 1, 1, 101, 4078, 1842, 3057, 443, 1, 111, 0, 0, 2, 606, 3362, 0, 1, 7, 1, 0, 234, 197, 0, 8, 2, 3, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 1, 3819, 0, 0, 0, 160, 0, 126, 1, 0, 1423, 3926, 19, 19, 1, 2591, 1, 2, 5, 406, 119, 11, 10, 0, 0, 0, 11, 3141, 0, 2, 3672, 1001, 0, 0, 0, 2, 0, 34, 3609, 0, 4020, 55, 0, 550, 0, 5, 12, 114, 1789, 17, 0, 0, 1, 29, 75, 789, 0, 982, 0, 206, 0, 12, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 241, 4025, 1, 0, 1, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 1, 1, 5, 88, 3265, 0, 781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3577, 0, 110, 2, 0, 877, 0, 1, 697, 0, 0, 0, 0, 0, 0, 0, 0, 66, 1, 0, 645, 0, 7, 2895, 0, 0, 0, 0, 0, 0, 0, 0, 188, 17, 0, 0, 0, 0, 69, 3627, 0, 0, 0, 0, 0, 0, 0, 0, 285, 2143, 1527, 0, 65, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 421, 3546, 0, 0, 0, 0, 0, 217, 212, 94, 0, 0, 0, 0, 0, 0, 400, 0, 71, 0, 746, 0, 4, 0, 1270, 456, 21, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4369, 4141, 5264, 3614, 3901, 4021, 4490, 2968] avg=4096.0000 max=5264 max_rank=2 min=2968 min_rank=7 imbalance_ratio=1.285156 max_load_reduction=27504 max_load_reduction_pct=0.839355 moved_to_echo=15380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3974, 0, 0, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 82, 2909, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1328, 0, 435, 2, 0, 132, 18, 1, 620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2178, 0, 71, 3315, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 0, 0, 0, 155, 3376, 0, 0, 0, 0, 0, 0, 0, 0, 80, 2524, 1709, 0, 285, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3176, 0, 0, 8, 0, 6, 162, 197, 84, 0, 0, 0, 0, 0, 0, 744, 1, 1755, 0, 66, 0, 0, 0, 1495, 1543, 19, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 3068, 2536, 5564, 3668, 4599, 3643, 5623] avg=4096.0000 max=5623 max_rank=7 min=2536 min_rank=2 imbalance_ratio=1.372803 max_load_reduction=27145 max_load_reduction_pct=0.828400 moved_to_echo=17533 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 3819, 0, 0, 0, 160, 0, 91, 0, 0, 0, 0, 0, 0, 0, 126, 1, 0, 1423, 3144, 19, 19, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2591, 1, 2, 5, 406, 119, 11, 10, 671, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1838, 0, 2, 3672, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 0, 0, 0, 2, 0, 34, 3324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2524, 55, 0, 550, 0, 5, 12, 0, 0, 0, 0, 0, 0, 0, 0, 114, 1789, 17, 0, 0, 1, 29, 75, 194, 91, 0, 0, 0, 0, 0, 0, 789, 0, 982, 0, 206, 0, 12, 0, 1496, 1303, 20, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4071, 4733, 3816, 5523, 4361, 3146, 2310, 4808] avg=4096.0000 max=5523 max_rank=3 min=2310 min_rank=6 imbalance_ratio=1.348389 max_load_reduction=27245 max_load_reduction_pct=0.831451 moved_to_echo=14625 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 44, 3289, 2, 1, 12, 21, 0, 98, 0, 0, 0, 0, 0, 0, 0, 33, 15, 1, 1887, 3085, 0, 212, 23, 0, 0, 0, 0, 0, 0, 0, 0, 621, 1, 413, 65, 1336, 2921, 27, 71, 664, 0, 0, 0, 0, 0, 0, 0, 0, 107, 185, 45, 461, 65, 57, 284, 0, 0, 0, 0, 0, 0, 0, 0, 95, 85, 7, 75, 2, 44, 52, 3428, 0, 0, 0, 0, 0, 0, 0, 0, 552, 2546, 107, 69, 1799, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3528, 160, 176, 0, 2, 0, 110, 204, 93, 0, 0, 0, 0, 0, 0, 368, 4, 771, 0, 515, 23, 23, 1, 1515, 334, 23, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3468, 5256, 6119, 1204, 3788, 5076, 4280, 3577] avg=4096.0000 max=6119 max_rank=2 min=1204 min_rank=3 imbalance_ratio=1.493896 max_load_reduction=26649 max_load_reduction_pct=0.813263 moved_to_echo=16721 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 509, 3746, 0, 0, 0, 2, 0, 104, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1126, 3269, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 920, 17, 131, 24, 0, 35, 10, 72, 697, 0, 0, 0, 0, 0, 0, 0, 4, 3, 0, 0, 2082, 0, 129, 3583, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 153, 3767, 0, 0, 0, 0, 0, 0, 0, 0, 27, 2471, 829, 0, 74, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 2891, 0, 3, 1, 0, 0, 18, 220, 95, 0, 0, 0, 0, 0, 0, 1719, 0, 884, 4, 37, 0, 1, 0, 1464, 1475, 21, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4361, 4402, 1906, 5801, 3974, 3410, 3309, 5605] avg=4096.0000 max=5801 max_rank=3 min=1906 min_rank=2 imbalance_ratio=1.416260 max_load_reduction=26967 max_load_reduction_pct=0.822968 moved_to_echo=16298 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 3944, 0, 0, 13, 0, 21, 98, 0, 0, 0, 0, 0, 0, 0, 55, 0, 158, 270, 3217, 7, 610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1712, 0, 34, 2, 14, 1901, 0, 0, 686, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 166, 936, 3, 0, 2094, 0, 0, 0, 0, 0, 0, 0, 0, 93, 1, 1, 0, 0, 0, 176, 3564, 0, 0, 0, 0, 0, 0, 0, 0, 963, 2552, 670, 0, 4, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3943, 1, 0, 0, 0, 0, 1823, 208, 93, 0, 0, 0, 0, 0, 0, 285, 0, 169, 0, 20, 0, 0, 0, 1512, 663, 21, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4317, 4349, 3201, 3835, 4241, 6078, 2670] avg=4096.0000 max=6078 max_rank=6 min=2670 min_rank=7 imbalance_ratio=1.483887 max_load_reduction=26690 max_load_reduction_pct=0.814514 moved_to_echo=16824 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 15, 3651, 0, 0, 66, 1, 0, 100, 0, 0, 0, 0, 0, 0, 0, 4, 0, 112, 59, 3269, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3023, 0, 514, 2, 0, 34, 228, 75, 698, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 2026, 0, 3, 442, 0, 0, 0, 0, 0, 0, 0, 0, 1262, 0, 0, 0, 0, 0, 301, 3655, 0, 0, 0, 0, 0, 0, 0, 0, 100, 2551, 1284, 0, 16, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174, 3475, 4, 2, 0, 0, 0, 170, 213, 95, 0, 0, 0, 0, 0, 0, 1332, 0, 780, 1, 47, 0, 0, 0, 1511, 1435, 21, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3833, 3445, 4574, 2482, 5218, 3956, 4133, 5127] avg=4096.0000 max=5218 max_rank=4 min=2482 min_rank=3 imbalance_ratio=1.273926 max_load_reduction=27550 max_load_reduction_pct=0.840759 moved_to_echo=18434 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[338, 1, 4076, 0, 0, 0, 4, 0, 103, 0, 0, 0, 0, 0, 0, 0, 9, 3, 3, 5, 3279, 0, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 648, 0, 757, 0, 0, 1627, 16, 456, 700, 0, 0, 0, 0, 0, 0, 0, 0, 284, 0, 198, 1440, 4, 0, 3453, 0, 0, 0, 0, 0, 0, 0, 0, 164, 0, 0, 0, 1, 1, 101, 3756, 0, 0, 0, 0, 0, 0, 0, 0, 1842, 1920, 443, 1, 111, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 606, 3362, 0, 1, 7, 1, 0, 234, 219, 95, 0, 0, 0, 0, 0, 0, 197, 0, 8, 2, 3, 0, 1, 0, 1137, 1021, 21, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4522, 3406, 4204, 5379, 4023, 4319, 4525, 2390] avg=4096.0000 max=5379 max_rank=3 min=2390 min_rank=7 imbalance_ratio=1.313232 max_load_reduction=27389 max_load_reduction_pct=0.835846 moved_to_echo=15257 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[5888, 128, 5504, 128, 1664, 128, 128, 128, 11520, 8320, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[14464, 128, 2432, 128, 1792, 7680, 384, 768, 5504, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 512, 256, 512, 11648, 128, 384, 19840, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3968, 19328, 6656, 128, 2944, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[1536, 25728, 256, 256, 128, 128, 128, 2816, 1792, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[384, 128, 384, 4992, 25472, 128, 1792, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3072, 128, 128, 128, 128, 128, 1152, 28544, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[384, 896, 30592, 128, 128, 128, 256, 128, 896, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33792 slack_slots=31744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[197, 1, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 23, 0, 0, 0, 0, 8, 3925, 5, 4040, 2, 0, 0, 0, 5, 0, 2, 0, 0, 0, 0, 4083, 0, 0, 413, 0, 7, 0, 0, 0, 0, 0, 0, 0, 2, 3736, 4041, 0, 0, 4082, 0, 36, 0, 0, 132, 0, 3830, 0, 0, 0, 3, 0, 111, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[11, 0, 1, 0, 194, 0, 4, 0, 0, 3951, 0, 0, 0, 0, 6, 0, 537, 3615, 0, 2902, 7, 0, 0, 0, 0, 0, 2, 0, 0, 541, 0, 3696, 0, 0, 0, 0, 52, 0, 1, 0, 0, 0, 0, 0, 0, 4055, 4046, 0, 15, 4035, 1, 876, 0, 1, 49, 0, 3863, 68, 0, 0, 0, 43, 193, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 4, 0, 0, 4085, 0, 1172, 0, 0, 0, 0, 27, 3402, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 0, 3790, 0, 35, 0, 0, 78, 4, 1, 0, 0, 0, 0, 0, 8, 4040, 4080, 0, 0, 4069, 9, 83, 0, 0, 0, 0, 3689, 0, 6, 0, 17, 1, 29, 64] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[5, 0, 0, 0, 192, 12, 0, 1, 0, 4096, 0, 44, 0, 0, 0, 0, 2726, 324, 1, 895, 12, 0, 0, 0, 0, 0, 0, 0, 77, 526, 2, 3873, 78, 77, 0, 0, 0, 137, 0, 0, 0, 0, 0, 0, 0, 4088, 4088, 0, 45, 4066, 0, 3905, 0, 0, 0, 0, 3498, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[10, 0, 0, 28, 0, 0, 1, 0, 28, 4057, 0, 911, 0, 0, 0, 58, 32, 2999, 0, 2229, 8, 167, 0, 0, 0, 0, 437, 0, 38, 0, 60, 3961, 0, 0, 0, 0, 0, 0, 41, 0, 1, 225, 46, 0, 3, 3527, 4053, 6, 693, 4096, 2, 51, 7, 24, 88, 0, 2062, 5, 0, 0, 95, 0, 2719, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[10, 0, 96, 0, 48, 0, 0, 0, 0, 4047, 0, 68, 0, 0, 0, 0, 56, 4096, 24, 1987, 0, 0, 0, 0, 0, 0, 2, 0, 214, 240, 0, 4092, 0, 4, 18, 1, 0, 0, 0, 0, 0, 0, 0, 35, 0, 3082, 4026, 0, 1, 4001, 0, 2158, 3, 0, 0, 0, 4080, 82, 0, 0, 0, 0, 291, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[139, 0, 45, 0, 0, 1, 6, 0, 0, 3883, 0, 1525, 0, 0, 0, 210, 6, 3547, 12, 1222, 1, 0, 0, 0, 0, 0, 3, 15, 845, 277, 0, 4076, 0, 3, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3732, 0, 110, 3985, 0, 629, 2, 129, 0, 0, 4047, 0, 1, 1, 0, 0, 55, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 120, 0, 174, 0, 0, 0, 0, 4047, 0, 21, 0, 0, 0, 0, 3988, 346, 0, 1742, 41, 0, 0, 0, 0, 0, 1, 0, 239, 393, 0, 4034, 0, 0, 19, 3, 0, 0, 132, 0, 0, 0, 0, 0, 0, 4023, 3965, 0, 0, 4091, 0, 2082, 0, 0, 0, 0, 3221, 4, 0, 0, 0, 0, 81, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[197, 1, 0, 0, 0, 0, 0, 0, 2639, 1358, 449, 279, 0, 0, 0, 0, 0, 3635, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1064, 5, 4040, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2, 0, 0, 0, 0, 3727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 413, 0, 7, 0, 0, 0, 3814, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3736, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2724, 0, 36, 0, 0, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3830, 0, 0, 0, 3, 0, 111, 0, 77, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4923, 3658, 5119, 3734, 4456, 3965, 2892, 4021] avg=4096.0000 max=5119 max_rank=2 min=2892 min_rank=6 imbalance_ratio=1.249756 max_load_reduction=27649 max_load_reduction_pct=0.843781 moved_to_echo=15334 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[11, 0, 1, 0, 194, 0, 4, 0, 2431, 1342, 434, 253, 0, 0, 0, 0, 0, 3517, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 979, 0, 2902, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 541, 0, 3373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 1, 0, 3819, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 2693, 1, 876, 0, 1, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3863, 68, 0, 0, 0, 43, 193, 3, 70, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4670, 3523, 4425, 3916, 4077, 4282, 3635, 4240] avg=4096.0000 max=4670 max_rank=0 min=3523 min_rank=1 imbalance_ratio=1.140137 max_load_reduction=28098 max_load_reduction_pct=0.857483 moved_to_echo=16234 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 0, 0, 192, 12, 0, 1, 217, 1352, 450, 265, 0, 0, 0, 0, 0, 3646, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2726, 89, 1, 895, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 526, 2, 3535, 0, 0, 0, 0, 0, 0, 0, 0, 78, 77, 0, 0, 0, 137, 0, 0, 3858, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 2714, 0, 3905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3498, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2494, 3690, 3723, 4140, 4168, 4318, 6664, 3571] avg=4096.0000 max=6664 max_rank=6 min=2494 min_rank=0 imbalance_ratio=1.626953 max_load_reduction=26104 max_load_reduction_pct=0.796631 moved_to_echo=18721 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 4, 0, 2287, 1353, 449, 259, 0, 0, 0, 0, 0, 3636, 0, 1172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 922, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 0, 3460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 78, 4, 1, 0, 3851, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4040, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2716, 9, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3689, 0, 6, 0, 17, 1, 29, 64, 71, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4353, 4808, 5015, 3468, 4162, 4277, 2808, 3877] avg=4096.0000 max=5015 max_rank=2 min=2808 min_rank=6 imbalance_ratio=1.224365 max_load_reduction=27753 max_load_reduction_pct=0.846954 moved_to_echo=15124 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[139, 0, 45, 0, 0, 1, 6, 0, 2385, 1325, 427, 279, 0, 0, 0, 0, 0, 3456, 0, 1525, 0, 0, 0, 210, 0, 0, 0, 0, 0, 0, 0, 0, 6, 961, 12, 1222, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 15, 845, 277, 0, 3720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 165, 0, 0, 0, 0, 0, 3522, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 2660, 0, 629, 2, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 1, 1, 0, 0, 55, 0, 77, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4607, 5191, 2202, 4860, 3891, 4306, 3530, 4181] avg=4096.0000 max=5191 max_rank=1 min=2202 min_rank=2 imbalance_ratio=1.267334 max_load_reduction=27577 max_load_reduction_pct=0.841583 moved_to_echo=15908 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 96, 0, 48, 0, 0, 0, 2754, 1331, 445, 280, 0, 0, 0, 0, 0, 3602, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 1110, 24, 1987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 214, 240, 0, 3735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 18, 1, 0, 0, 0, 0, 3800, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 3082, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2670, 0, 2158, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 82, 0, 0, 0, 0, 291, 6, 77, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4964, 3670, 3177, 4191, 4055, 3343, 4832, 4536] avg=4096.0000 max=4964 max_rank=0 min=3177 min_rank=2 imbalance_ratio=1.211914 max_load_reduction=27804 max_load_reduction_pct=0.848511 moved_to_echo=16766 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 0, 28, 0, 0, 1, 0, 2021, 1366, 449, 273, 0, 0, 0, 0, 28, 3608, 0, 911, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 32, 805, 0, 2229, 8, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 437, 0, 38, 0, 60, 3609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 3830, 173, 0, 0, 0, 0, 0, 0, 1, 225, 46, 0, 3, 3527, 223, 6, 0, 0, 0, 0, 0, 0, 0, 0, 693, 2730, 2, 51, 7, 24, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2062, 5, 0, 0, 95, 0, 2719, 0, 79, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4148, 4605, 3241, 4144, 4044, 4031, 3595, 4960] avg=4096.0000 max=4960 max_rank=7 min=3241 min_rank=2 imbalance_ratio=1.210938 max_load_reduction=27808 max_load_reduction_pct=0.848633 moved_to_echo=16630 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 120, 0, 174, 0, 0, 0, 232, 1361, 445, 276, 0, 0, 0, 0, 0, 3602, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3988, 95, 0, 1742, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 239, 393, 0, 3682, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 3, 0, 0, 132, 0, 3742, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2730, 0, 2082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3221, 4, 0, 0, 0, 0, 81, 0, 76, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2609, 3623, 5866, 4315, 3915, 4246, 4812, 3382] avg=4096.0000 max=5866 max_rank=2 min=2609 min_rank=0 imbalance_ratio=1.432129 max_load_reduction=26902 max_load_reduction_pct=0.820984 moved_to_echo=16355 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[384, 128, 384, 128, 640, 128, 128, 128, 14976, 10880, 3584, 2176, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[7424, 6144, 128, 19200, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[896, 21760, 128, 9856, 128, 256, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 512, 128, 1536, 2048, 128, 28928, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[28416, 256, 128, 128, 128, 128, 3584, 128, 640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 28800, 0, 3840, 0, 0, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 640, 128, 256, 256, 256, 0, 30336, 1280, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 256, 128, 128, 128, 30720, 1920, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4090, 0, 0, 3, 3926, 0, 0, 10, 0, 0, 0, 2663, 0, 1680, 3991, 0, 70, 0, 1, 19, 0, 0, 14, 0, 4, 25, 0, 3, 0, 146, 470, 3824, 1, 0, 0, 0, 0, 3573, 0, 0, 26, 4061, 35, 0, 1, 0, 1, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 49, 4, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4020, 0, 0, 321, 3909, 0, 0, 0, 0, 0, 0, 830, 0, 1881, 3959, 0, 1, 7, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3089, 4084, 0, 0, 2, 0, 0, 4096, 6, 0, 10, 2403, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4076, 71, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[8, 0, 4096, 0, 8, 1, 4093, 0, 0, 0, 0, 0, 0, 27, 1, 3941, 4084, 0, 29, 0, 0, 0, 0, 22, 1, 0, 23, 0, 0, 10, 0, 1, 1510, 4089, 0, 3, 0, 14, 0, 4096, 0, 0, 7, 2579, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 1, 0, 0, 0, 0, 4090, 16, 0, 0, 15] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[3, 0, 3859, 0, 0, 15, 3594, 18, 0, 0, 0, 0, 0, 67, 0, 4022, 3229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1549, 0, 457, 71, 4032, 154, 18, 0, 0, 0, 4090, 0, 0, 264, 3116, 0, 0, 0, 0, 6, 0, 0, 0, 0, 47, 2, 0, 0, 5, 0, 4084, 11, 0, 0, 55] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4094, 0, 0, 2203, 4084, 1, 0, 0, 3, 0, 0, 127, 0, 2617, 4045, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 218, 0, 2, 2657, 4091, 30, 0, 0, 0, 0, 4089, 0, 0, 0, 391, 0, 0, 0, 1, 0, 0, 4, 0, 4, 0, 0, 0, 0, 0, 0, 4088, 3, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[3, 1, 3868, 0, 0, 13, 4042, 154, 0, 91, 0, 1, 0, 2962, 2, 1378, 3958, 0, 0, 0, 0, 0, 0, 0, 2, 1, 96, 3, 0, 861, 0, 1, 800, 4094, 13, 0, 0, 0, 0, 2800, 0, 0, 0, 3532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4091, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 2522, 0, 287, 2286, 3794, 2, 0, 0, 0, 0, 4, 9, 0, 3083, 3928, 14, 0, 7, 107, 73, 0, 0, 631, 0, 15, 41, 4, 1, 0, 1, 3485, 4083, 0, 0, 0, 0, 0, 2824, 0, 0, 0, 65, 0, 1, 0, 0, 137, 0, 0, 0, 0, 54, 1702, 0, 0, 0, 0, 3546, 60, 0, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4081, 0, 0, 0, 3816, 39, 0, 0, 138, 0, 0, 1434, 0, 3139, 4057, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 3950, 0, 0, 0, 0, 0, 2909, 0, 0, 113, 3949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 941, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3661, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 2663, 0, 1680, 354, 0, 0, 0, 0, 0, 0, 0, 3991, 0, 70, 0, 1, 19, 0, 0, 72, 67, 0, 0, 0, 0, 0, 0, 14, 0, 4, 25, 0, 3, 0, 146, 3403, 179, 0, 0, 0, 0, 0, 0, 470, 0, 1, 0, 0, 0, 0, 2566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 4061, 35, 0, 1, 0, 1007, 523, 3, 0, 0, 0, 0, 0, 1, 0, 71, 0, 0, 0, 0, 0, 3645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3939, 49, 4, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3664, 4707, 4220, 3774, 3037, 5656, 3717, 3993] avg=4096.0000 max=5656 max_rank=5 min=3037 min_rank=4 imbalance_ratio=1.380859 max_load_reduction=27112 max_load_reduction_pct=0.827393 moved_to_echo=16403 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3598, 0, 0, 321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 830, 0, 1881, 348, 0, 0, 0, 0, 0, 0, 0, 3959, 0, 1, 7, 0, 0, 0, 0, 71, 69, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3389, 191, 0, 0, 0, 0, 0, 0, 3089, 0, 0, 0, 2, 0, 0, 2942, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 10, 2403, 0, 0, 0, 0, 1154, 520, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4007, 71, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3919, 3059, 4107, 3581, 6033, 4096, 3894, 4079] avg=4096.0000 max=6033 max_rank=4 min=3059 min_rank=1 imbalance_ratio=1.472900 max_load_reduction=26735 max_load_reduction_pct=0.815887 moved_to_echo=18102 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 3454, 0, 0, 15, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 4022, 334, 0, 0, 0, 0, 0, 0, 0, 3229, 0, 0, 0, 0, 0, 0, 0, 68, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1549, 0, 457, 3116, 189, 0, 0, 0, 0, 0, 0, 71, 0, 154, 18, 0, 0, 0, 2938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 264, 3116, 0, 0, 0, 0, 1152, 478, 3, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 47, 2, 0, 3843, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4015, 11, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3490, 4423, 3366, 5311, 3181, 5013, 3898, 4086] avg=4096.0000 max=5311 max_rank=3 min=3181 min_rank=4 imbalance_ratio=1.296631 max_load_reduction=27457 max_load_reduction_pct=0.837921 moved_to_echo=16178 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3664, 0, 0, 2203, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 127, 0, 2617, 355, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 0, 16, 0, 0, 0, 0, 72, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 218, 0, 2, 3540, 191, 0, 0, 0, 0, 0, 0, 2657, 0, 30, 0, 0, 0, 0, 2937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 391, 0, 0, 0, 1, 1152, 544, 3, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 0, 0, 0, 3900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4019, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5868, 3102, 4202, 3951, 5624, 2091, 3908, 4022] avg=4096.0000 max=5868 max_rank=0 min=2091 min_rank=5 imbalance_ratio=1.432617 max_load_reduction=26900 max_load_reduction_pct=0.820923 moved_to_echo=15645 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[3, 1, 3462, 0, 0, 13, 0, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 1, 0, 2962, 2, 1378, 335, 0, 0, 0, 0, 0, 0, 0, 3958, 0, 0, 0, 0, 0, 0, 0, 68, 69, 0, 0, 0, 0, 0, 0, 2, 1, 96, 3, 0, 861, 0, 1, 3505, 192, 0, 0, 0, 0, 0, 0, 800, 0, 13, 0, 0, 0, 0, 2011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3532, 0, 0, 0, 0, 789, 537, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3902, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3633, 4769, 4095, 4661, 2824, 4861, 3902, 4023] avg=4096.0000 max=4861 max_rank=5 min=2824 min_rank=4 imbalance_ratio=1.186768 max_load_reduction=27907 max_load_reduction_pct=0.851654 moved_to_echo=15610 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 3666, 0, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 1, 3941, 355, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 29, 0, 0, 0, 0, 22, 72, 69, 0, 0, 0, 0, 0, 0, 1, 0, 23, 0, 0, 10, 0, 1, 3549, 192, 0, 0, 0, 0, 0, 0, 1510, 0, 0, 3, 0, 14, 0, 2942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2579, 0, 0, 0, 0, 1154, 544, 3, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 1, 0, 3897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4021, 16, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3683, 4324, 4276, 3776, 4469, 4287, 3901, 4052] avg=4096.0000 max=4469 max_rank=4 min=3683 min_rank=0 imbalance_ratio=1.091064 max_load_reduction=28299 max_load_reduction_pct=0.863617 moved_to_echo=16709 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2245, 0, 287, 2286, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 9, 0, 3083, 223, 0, 0, 0, 0, 0, 0, 0, 3928, 14, 0, 7, 107, 73, 0, 0, 50, 62, 0, 0, 0, 0, 0, 0, 631, 0, 15, 41, 4, 1, 0, 1, 3290, 192, 0, 0, 0, 0, 0, 0, 3485, 0, 0, 0, 0, 0, 0, 2025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 1, 0, 0, 799, 504, 4, 0, 0, 0, 0, 0, 137, 0, 0, 0, 0, 54, 1702, 0, 3891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3484, 60, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4820, 3319, 4241, 4175, 5510, 1373, 5784, 3546] avg=4096.0000 max=5784 max_rank=6 min=1373 min_rank=5 imbalance_ratio=1.412109 max_load_reduction=26984 max_load_reduction_pct=0.823486 moved_to_echo=16213 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3652, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 0, 0, 1434, 0, 3139, 354, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 63, 0, 0, 0, 0, 0, 72, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 3309, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 3949, 0, 0, 0, 0, 819, 507, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3691, 5065, 4261, 3539, 2090, 5391, 3764, 4967] avg=4096.0000 max=5391 max_rank=5 min=2090 min_rank=4 imbalance_ratio=1.316162 max_load_reduction=27377 max_load_reduction_pct=0.835480 moved_to_echo=16212 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[12160, 0, 256, 128, 128, 128, 0, 20480, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 0, 512, 20096, 128, 128, 128, 128, 8064, 4224, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 256, 128, 128, 8192, 128, 21760, 2688, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[768, 128, 256, 128, 128, 2688, 0, 640, 27136, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 27520, 0, 384, 4864, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[256, 128, 128, 128, 128, 128, 1792, 0, 30848, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[31360, 128, 256, 128, 128, 128, 0, 128, 640, 640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 31616, 1152, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 3, 1, 3508, 177, 0, 4052, 0, 40, 0, 3, 0, 44, 0, 0, 0, 4091, 0, 0, 0, 0, 4095, 4068, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 26, 0, 1, 0, 1, 0, 2, 0, 3, 4052, 4088, 14, 0, 0, 5, 0, 0, 0, 21, 0, 4095, 0, 0, 0, 0, 367] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[2, 0, 1, 0, 15, 0, 3596, 0, 264, 0, 0, 290, 3222, 0, 0, 336, 3838, 0, 3, 2, 2, 4096, 3628, 15, 234, 0, 6, 5, 522, 210, 0, 164, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 11, 5, 6, 0, 0, 3986, 3748, 0, 0, 2, 0, 363, 71, 0, 25, 13, 3436, 121, 0, 523, 0, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[11, 1, 11, 3314, 6, 0, 4083, 0, 420, 0, 0, 0, 2, 0, 0, 1, 4024, 0, 0, 0, 0, 4008, 3510, 0, 0, 0, 0, 35, 29, 0, 0, 279, 0, 0, 0, 0, 1, 0, 2, 12, 0, 1, 14, 0, 2, 8, 4, 3695, 3835, 0, 0, 0, 1, 47, 1270, 0, 0, 0, 4074, 0, 0, 23, 0, 45] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[2, 0, 0, 4093, 4, 0, 4059, 23, 0, 0, 0, 1, 100, 0, 0, 0, 4029, 0, 6, 0, 0, 3975, 4086, 1, 0, 0, 0, 66, 0, 0, 0, 463, 0, 0, 6, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 2, 3505, 3889, 0, 0, 0, 0, 0, 333, 0, 4, 0, 4095, 1, 0, 0, 0, 17] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 3990, 205, 4, 3938, 1, 230, 0, 0, 0, 0, 0, 0, 0, 4089, 15, 0, 0, 0, 4096, 4072, 0, 0, 0, 0, 0, 35, 0, 0, 4, 0, 0, 0, 1, 0, 0, 1, 2, 0, 0, 318, 0, 0, 0, 7, 3088, 3818, 0, 0, 0, 1, 0, 1282, 32, 0, 0, 3527, 0, 0, 12, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4029, 0, 0, 4073, 0, 0, 0, 0, 0, 846, 0, 0, 4, 4095, 0, 0, 0, 0, 4080, 3943, 0, 0, 0, 0, 82, 39, 0, 0, 0, 0, 1, 28, 4, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 2, 1883, 4087, 0, 0, 0, 0, 0, 143, 0, 0, 0, 4094, 0, 0, 0, 0, 1297] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4087, 0, 8, 4091, 0, 13, 0, 0, 0, 493, 0, 0, 0, 4036, 0, 1, 0, 3, 4080, 3903, 7, 48, 0, 0, 531, 38, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 98, 0, 0, 0, 0, 3116, 4096, 0, 6, 4, 0, 7, 14, 0, 0, 1, 4056, 0, 0, 4, 0, 12] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 3992, 0, 0, 3561, 0, 79, 0, 0, 0, 14, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4079, 0, 0, 0, 0, 2, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 7, 0, 0, 0, 715, 4096, 0, 7, 0, 0, 0, 3828, 0, 0, 0, 4095, 0, 0, 0, 0, 71] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 1, 3508, 177, 0, 681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 3, 0, 44, 0, 0, 0, 3294, 100, 0, 0, 0, 0, 0, 0, 182, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 26, 0, 4095, 16, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 2, 0, 3, 4052, 815, 164, 77, 0, 0, 0, 0, 0, 3273, 14, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 3931, 0, 0, 0, 0, 367, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4370, 3481, 4250, 3794, 4147, 5115, 3292, 4319] avg=4096.0000 max=5115 max_rank=5 min=3292 min_rank=6 imbalance_ratio=1.248779 max_load_reduction=27653 max_load_reduction_pct=0.843903 moved_to_echo=16873 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[11, 1, 11, 3314, 6, 0, 686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 420, 0, 0, 0, 2, 0, 0, 1, 3319, 98, 0, 0, 0, 0, 0, 0, 179, 0, 0, 0, 0, 0, 3510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 29, 0, 0, 279, 3731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 12, 4008, 16, 0, 0, 0, 0, 0, 0, 0, 1, 14, 0, 2, 8, 4, 3695, 765, 163, 78, 0, 0, 0, 0, 0, 3070, 0, 0, 0, 1, 47, 1270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3911, 0, 0, 23, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4029, 3840, 3689, 4074, 4039, 4730, 4388, 3979] avg=4096.0000 max=4730 max_rank=5 min=3689 min_rank=2 imbalance_ratio=1.154785 max_load_reduction=28038 max_load_reduction_pct=0.855652 moved_to_echo=17136 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3990, 205, 4, 662, 1, 0, 0, 0, 0, 0, 0, 0, 0, 230, 0, 0, 0, 0, 0, 0, 0, 3201, 100, 0, 0, 0, 0, 0, 0, 182, 15, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 4, 3791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 2, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 318, 0, 0, 0, 7, 3088, 762, 141, 75, 0, 0, 0, 0, 0, 3056, 0, 0, 0, 1, 0, 1282, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3386, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4862, 3531, 4269, 3830, 4116, 4391, 4371, 3398] avg=4096.0000 max=4862 max_rank=0 min=3398 min_rank=7 imbalance_ratio=1.187012 max_load_reduction=27906 max_load_reduction_pct=0.851624 moved_to_echo=16276 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 4093, 4, 0, 682, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 100, 0, 0, 0, 3300, 99, 0, 0, 0, 0, 0, 0, 179, 0, 6, 0, 0, 0, 4086, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 463, 3735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 8, 0, 3975, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3505, 776, 164, 77, 0, 0, 0, 0, 0, 3113, 0, 0, 0, 0, 0, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3931, 1, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4804, 3500, 4272, 4264, 4005, 4524, 3446, 3953] avg=4096.0000 max=4804 max_rank=0 min=3446 min_rank=6 imbalance_ratio=1.172852 max_load_reduction=27964 max_load_reduction_pct=0.853394 moved_to_echo=15928 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4087, 0, 8, 687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 493, 0, 0, 0, 3326, 99, 0, 0, 0, 0, 0, 0, 179, 0, 1, 0, 3, 0, 3903, 7, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 531, 38, 2, 0, 0, 3742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 16, 0, 0, 0, 0, 0, 0, 13, 0, 98, 0, 0, 0, 0, 3116, 817, 163, 78, 0, 0, 0, 0, 0, 3279, 0, 6, 4, 0, 7, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3893, 0, 0, 4, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4782, 3931, 4093, 4361, 4096, 4285, 3310, 3910] avg=4096.0000 max=4782 max_rank=0 min=3310 min_rank=6 imbalance_ratio=1.167480 max_load_reduction=27986 max_load_reduction_pct=0.854065 moved_to_echo=15601 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4029, 0, 0, 685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 846, 0, 0, 4, 3311, 100, 0, 0, 0, 0, 0, 0, 182, 0, 0, 0, 0, 0, 3943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 39, 0, 0, 0, 3797, 0, 0, 0, 0, 0, 0, 0, 0, 1, 28, 4, 0, 0, 0, 0, 4080, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 2, 1883, 815, 164, 77, 0, 0, 0, 0, 0, 3272, 0, 0, 0, 0, 0, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3930, 0, 0, 0, 0, 1297, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4714, 4261, 4125, 3918, 4129, 2979, 3415, 5227] avg=4096.0000 max=5227 max_rank=7 min=2979 min_rank=5 imbalance_ratio=1.276123 max_load_reduction=27541 max_load_reduction_pct=0.840485 moved_to_echo=15750 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 1, 0, 15, 0, 599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 264, 0, 0, 290, 3222, 0, 0, 336, 2926, 98, 0, 0, 0, 0, 0, 0, 158, 0, 3, 2, 2, 0, 3628, 15, 0, 0, 0, 0, 0, 0, 0, 0, 234, 0, 6, 5, 522, 210, 0, 164, 3562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 20, 0, 0, 0, 0, 0, 0, 0, 0, 11, 5, 6, 0, 0, 3986, 752, 142, 71, 0, 0, 0, 0, 0, 2996, 0, 0, 2, 0, 363, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 13, 3294, 121, 0, 523, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[617, 7136, 3808, 4703, 4119, 4973, 3432, 3980] avg=4096.0000 max=7136 max_rank=1 min=617 min_rank=0 imbalance_ratio=1.742188 max_load_reduction=25632 max_load_reduction_pct=0.782227 moved_to_echo=16504 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3992, 0, 0, 598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 0, 14, 0, 0, 0, 2895, 100, 0, 0, 0, 0, 0, 0, 183, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 25, 0, 0, 0, 3797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 715, 817, 164, 68, 0, 0, 0, 0, 0, 3279, 0, 7, 0, 0, 0, 3828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3931, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4590, 3088, 4262, 3824, 4117, 1771, 7114, 4002] avg=4096.0000 max=7114 max_rank=6 min=1771 min_rank=5 imbalance_ratio=1.736816 max_load_reduction=25654 max_load_reduction_pct=0.782898 moved_to_echo=17004 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1152, 0, 128, 384, 4736, 0, 0, 384, 25600, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 27136, 512, 128, 5376, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 512, 128, 128, 128, 128, 24064, 6400, 1280, 640, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[25344, 128, 128, 128, 128, 512, 7040, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[1536, 128, 128, 128, 128, 0, 31360, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 30208, 128, 0, 640, 0, 1920, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[0, 128, 128, 128, 128, 0, 128, 128, 32640, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[384, 0, 128, 768, 768, 256, 0, 1024, 29952, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4093, 4070, 0, 4020, 0, 0, 0, 3, 0, 0, 0, 4083, 2, 0, 0, 64, 0, 1, 0, 0, 4091, 0, 1808, 0, 1384, 0, 0, 0, 0, 1034, 0, 0, 0, 0, 3958, 0, 1, 4078, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 53, 8, 10, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4072, 4005, 0, 4073, 0, 0, 0, 0, 0, 0, 1, 1222, 12, 0, 0, 2, 107, 0, 0, 0, 3777, 4, 348, 7, 3367, 0, 2, 165, 0, 3471, 0, 0, 0, 0, 3953, 0, 0, 4044, 42, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 26, 0, 2, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 17, 4096, 3981, 0, 1687, 0, 0, 0, 0, 0, 0, 0, 3361, 40, 0, 7, 2169, 78, 599, 0, 0, 2395, 20, 0, 49, 2369, 0, 0, 0, 0, 3824, 0, 0, 0, 0, 4007, 0, 51, 3883, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4085, 0, 362, 0, 0, 6, 0, 0, 31, 0, 4056, 0, 0, 0, 76, 0, 0, 0, 0, 4076, 1283, 3622, 0, 652, 0, 0, 0, 0, 2167, 5, 0, 0, 0, 4084, 0, 0, 4089, 0, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 3, 3, 0, 0, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 3, 4087, 4009, 0, 2408, 2, 0, 0, 1, 0, 0, 28, 456, 0, 0, 0, 0, 0, 3, 0, 0, 4029, 66, 141, 780, 3983, 0, 0, 462, 0, 4054, 11, 7, 0, 23, 4079, 15, 0, 4096, 13, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 1, 1, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 1, 4094, 4086, 0, 4084, 0, 0, 0, 1, 0, 0, 0, 4046, 0, 0, 0, 489, 1, 3, 0, 0, 4062, 614, 1708, 184, 1059, 0, 3, 26, 0, 99, 0, 0, 0, 22, 4069, 0, 1, 4095, 0, 3, 0, 1, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4028, 4070, 0, 0, 46, 0, 67, 134, 0, 0, 0, 3925, 202, 0, 0, 0, 0, 0, 0, 0, 1419, 59, 3, 0, 3825, 0, 6, 0, 0, 3988, 0, 0, 29, 92, 3802, 0, 0, 3914, 7, 0, 191, 0, 1, 0, 0, 0, 1, 0, 0, 2, 30, 0, 0, 0, 7, 2885, 35, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 4090, 0, 0, 0, 0, 0, 28, 0, 1020, 0, 0, 0, 0, 0, 98, 0, 0, 3986, 56, 2909, 6, 7, 0, 0, 0, 0, 4058, 0, 0, 0, 0, 4091, 0, 69, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 58, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1505, 0, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4083, 2, 0, 418, 127, 0, 0, 0, 0, 0, 0, 0, 64, 0, 1, 0, 0, 3964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1808, 0, 1384, 0, 0, 0, 0, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3958, 0, 1, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4021, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4093, 5, 0, 0, 0, 0, 0, 0, 0, 53, 8, 10, 0, 0, 0, 0, 2509, 438, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5525, 4633, 4029, 3370, 4011, 4078, 4104, 3018] avg=4096.0000 max=5525 max_rank=0 min=3018 min_rank=7 imbalance_ratio=1.348877 max_load_reduction=27243 max_load_reduction_pct=0.831390 moved_to_echo=15211 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1481, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1222, 12, 0, 1403, 117, 0, 0, 0, 0, 0, 0, 0, 2, 107, 0, 0, 0, 3660, 4, 0, 0, 0, 0, 0, 0, 0, 0, 348, 7, 3367, 0, 2, 165, 0, 596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3953, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 3988, 55, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 4072, 5, 0, 0, 0, 0, 0, 0, 0, 0, 57, 26, 0, 2, 0, 1, 2469, 1472, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5554, 2755, 3773, 4485, 4004, 4085, 4085, 4027] avg=4096.0000 max=5554 max_rank=0 min=2755 min_rank=1 imbalance_ratio=1.355957 max_load_reduction=27214 max_load_reduction_pct=0.830505 moved_to_echo=16201 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 17, 0, 1472, 0, 1687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3361, 40, 0, 1546, 74, 0, 0, 0, 0, 0, 0, 7, 2169, 78, 599, 0, 0, 2321, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 2369, 0, 0, 0, 0, 657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4007, 0, 51, 49, 0, 0, 0, 0, 0, 0, 0, 0, 134, 0, 0, 0, 0, 0, 0, 0, 3829, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2454, 1621, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3176, 5021, 5194, 3075, 4107, 4018, 4101, 4076] avg=4096.0000 max=5194 max_rank=2 min=3075 min_rank=3 imbalance_ratio=1.268066 max_load_reduction=27574 max_load_reduction_pct=0.841492 moved_to_echo=16302 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 1511, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4046, 0, 0, 40, 126, 0, 0, 0, 0, 0, 0, 0, 489, 1, 3, 0, 0, 3936, 614, 0, 0, 0, 0, 0, 0, 0, 0, 1708, 184, 1059, 0, 3, 26, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4069, 0, 1, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 8, 0, 0, 4038, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 1, 2519, 41, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5596, 4213, 5043, 2998, 4144, 4106, 4099, 2569] avg=4096.0000 max=5596 max_rank=0 min=2569 min_rank=7 imbalance_ratio=1.366211 max_load_reduction=27172 max_load_reduction_pct=0.829224 moved_to_echo=14918 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 1483, 0, 2408, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 28, 456, 0, 0, 1639, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3904, 66, 0, 0, 0, 0, 0, 0, 0, 0, 141, 780, 3983, 0, 0, 462, 0, 696, 0, 0, 0, 0, 0, 0, 0, 0, 11, 7, 0, 23, 4079, 15, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 1, 0, 0, 0, 4039, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4087, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 2471, 1719, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3896, 2249, 3973, 6062, 4187, 4108, 4101, 4192] avg=4096.0000 max=6062 max_rank=3 min=2249 min_rank=1 imbalance_ratio=1.479980 max_load_reduction=26706 max_load_reduction_pct=0.815002 moved_to_echo=16588 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1511, 0, 362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 31, 0, 4056, 0, 0, 876, 126, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0, 3950, 1283, 0, 0, 0, 0, 0, 0, 0, 0, 3622, 0, 652, 0, 0, 0, 0, 372, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4084, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 4032, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 2, 0, 2518, 919, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1873, 5095, 5309, 4646, 4141, 4154, 4105, 3445] avg=4096.0000 max=5309 max_rank=2 min=1873 min_rank=0 imbalance_ratio=1.296143 max_load_reduction=27459 max_load_reduction_pct=0.837982 moved_to_echo=15845 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1497, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 134, 0, 0, 0, 3925, 202, 0, 1615, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1372, 59, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3825, 0, 6, 0, 0, 679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 92, 3802, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 191, 0, 1, 0, 0, 0, 3864, 61, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 30, 0, 0, 0, 4028, 11, 0, 0, 0, 0, 0, 0, 7, 2885, 35, 0, 0, 0, 0, 0, 2512, 1694, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1543, 5990, 1431, 4513, 3962, 4124, 4072, 7133] avg=4096.0000 max=7133 max_rank=7 min=1431 min_rank=2 imbalance_ratio=1.741455 max_load_reduction=25635 max_load_reduction_pct=0.782318 moved_to_echo=19291 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1515, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 1020, 0, 0, 1640, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 3862, 56, 0, 0, 0, 0, 0, 0, 0, 0, 2909, 6, 7, 0, 0, 0, 0, 697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 69, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 4, 0, 0, 58, 0, 0, 0, 0, 2525, 1721, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5605, 2812, 4016, 3619, 4212, 4095, 4101, 4308] avg=4096.0000 max=5605 max_rank=0 min=2812 min_rank=1 imbalance_ratio=1.368408 max_load_reduction=27163 max_load_reduction_pct=0.828949 moved_to_echo=16716 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 0, 12032, 0, 20736, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 2944, 128, 128, 128, 128, 128, 128, 20096, 9728, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 256, 0, 128, 128, 22272, 256, 0, 9216, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[10624, 1152, 16768, 0, 128, 768, 0, 3968, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 2816, 256, 768, 0, 0, 27008, 2176, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 0, 128, 128, 0, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[256, 128, 256, 128, 128, 128, 0, 0, 31872, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 256, 32128, 128, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[10, 2646, 0, 0, 1, 152, 0, 2, 0, 0, 0, 0, 10, 4, 4080, 50, 47, 0, 0, 13, 0, 0, 206, 0, 0, 3556, 0, 5, 0, 0, 13, 3522, 0, 0, 0, 0, 0, 0, 0, 2213, 0, 1, 4085, 0, 4014, 6, 1118, 0, 0, 0, 4025, 0, 0, 2873, 1, 0, 114, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 2188, 157, 0, 0, 1, 5, 0, 0, 0, 0, 0, 1, 0, 4087, 0, 2, 0, 0, 0, 1, 0, 1, 0, 0, 4087, 0, 0, 0, 1, 4057, 152, 0, 0, 2, 0, 377, 0, 0, 1627, 0, 0, 4096, 0, 4093, 0, 1, 0, 0, 0, 4094, 28, 2, 2880, 765, 0, 54, 0, 0, 0, 1, 0, 0, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 3619, 524, 0, 539, 0, 0, 0, 0, 11, 0, 0, 188, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3114, 2, 0, 0, 4, 29, 2, 3, 0, 0, 0, 144, 0, 0, 515, 0, 0, 4082, 0, 4082, 0, 3353, 37, 4, 0, 3498, 91, 0, 1038, 0, 0, 3769, 0, 0, 0, 0, 0, 0, 24] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 2121, 28, 0, 6, 0, 0, 0, 0, 0, 0, 0, 943, 0, 4028, 21, 0, 1, 0, 0, 6, 0, 0, 25, 0, 4066, 0, 0, 0, 0, 2322, 6, 0, 0, 0, 0, 0, 0, 0, 2961, 1, 3, 4095, 0, 4095, 3, 71, 0, 859, 0, 4086, 7, 1, 0, 2983, 0, 26, 0, 0, 1, 0, 1, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 3465, 334, 0, 11, 2986, 30, 0, 0, 0, 0, 0, 1195, 271, 3548, 0, 5, 0, 0, 0, 0, 0, 1771, 0, 0, 1327, 0, 23, 0, 0, 1320, 3252, 9, 158, 0, 0, 0, 0, 459, 173, 0, 0, 3620, 0, 4093, 0, 34, 0, 450, 0, 3553, 276, 0, 0, 224, 0, 162, 0, 0, 0, 15, 1, 0, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 4013, 1046, 0, 6, 0, 94, 0, 0, 3, 0, 0, 44, 0, 3999, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 2, 0, 2905, 101, 0, 0, 0, 0, 10, 0, 0, 1192, 0, 0, 4037, 0, 4006, 1, 80, 0, 64, 0, 4046, 0, 0, 7, 0, 0, 3010, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 4019, 1741, 0, 0, 2, 0, 0, 0, 10, 0, 0, 0, 0, 3555, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3984, 1, 0, 0, 1, 3496, 58, 0, 65, 6, 0, 97, 0, 0, 221, 239, 0, 4077, 0, 4088, 0, 84, 0, 61, 0, 4088, 10, 0, 2761, 0, 0, 103, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 1672, 82, 0, 0, 437, 0, 0, 0, 0, 0, 0, 262, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 15, 1479, 0, 0, 0, 0, 113, 0, 0, 124, 0, 0, 4096, 0, 4096, 0, 3766, 0, 0, 0, 4044, 0, 0, 46, 259, 0, 4085, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[10, 2646, 0, 0, 1, 152, 0, 2, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4, 3855, 50, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 13, 0, 0, 206, 0, 3782, 0, 0, 0, 0, 0, 0, 0, 0, 1256, 0, 5, 0, 0, 13, 3522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2213, 2300, 269, 232, 0, 0, 0, 0, 0, 0, 1, 3043, 0, 0, 6, 1118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2188, 0, 0, 2873, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 0, 1, 0, 0, 0, 0, 0, 1568, 1042, 118, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2918, 3919, 4048, 4796, 5014, 4168, 5062, 2843] avg=4096.0000 max=5062 max_rank=6 min=2843 min_rank=7 imbalance_ratio=1.235840 max_load_reduction=27706 max_load_reduction_pct=0.845520 moved_to_echo=17087 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 2188, 157, 0, 0, 1, 5, 0, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 1, 0, 3857, 0, 0, 0, 0, 0, 0, 0, 0, 1444, 0, 0, 0, 1, 4057, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 377, 0, 0, 1627, 2643, 273, 236, 0, 0, 0, 0, 0, 0, 0, 3051, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2226, 28, 2, 2880, 765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 1, 0, 0, 8, 1595, 1045, 119, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2458, 3862, 3861, 5654, 5158, 3052, 5901, 2822] avg=4096.0000 max=5901 max_rank=6 min=2458 min_rank=0 imbalance_ratio=1.440674 max_load_reduction=26867 max_load_reduction_pct=0.819916 moved_to_echo=16933 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3619, 524, 0, 539, 0, 0, 0, 108, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 188, 0, 3869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3847, 0, 0, 0, 0, 0, 0, 0, 0, 1100, 2, 0, 0, 4, 29, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 144, 0, 0, 515, 2014, 234, 235, 0, 0, 0, 0, 0, 0, 0, 3040, 0, 0, 0, 3353, 37, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1901, 91, 0, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3769, 0, 0, 0, 0, 0, 0, 24, 1363, 1042, 119, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4790, 4068, 3847, 1137, 3145, 6430, 3034, 6317] avg=4096.0000 max=6430 max_rank=5 min=1137 min_rank=3 imbalance_ratio=1.569824 max_load_reduction=26338 max_load_reduction_pct=0.803772 moved_to_echo=18926 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 2121, 28, 0, 6, 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 943, 0, 3805, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6, 0, 0, 25, 3859, 0, 0, 0, 0, 0, 0, 0, 0, 1436, 0, 0, 0, 0, 2322, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2961, 2630, 273, 236, 0, 0, 0, 0, 0, 1, 3, 3050, 0, 0, 3, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 859, 0, 2221, 7, 1, 0, 2983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 1, 0, 1, 2, 0, 1592, 1045, 117, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2261, 4769, 3891, 3764, 6100, 3128, 6071, 2784] avg=4096.0000 max=6100 max_rank=4 min=2261 min_rank=0 imbalance_ratio=1.489258 max_load_reduction=26668 max_load_reduction_pct=0.813843 moved_to_echo=18083 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4013, 1046, 0, 6, 0, 94, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 44, 0, 3778, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3774, 0, 0, 0, 0, 0, 0, 0, 0, 1444, 0, 0, 2, 0, 2905, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 1192, 2644, 270, 232, 0, 0, 0, 0, 0, 0, 0, 3007, 0, 0, 1, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 2199, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3010, 0, 0, 0, 0, 0, 0, 0, 1577, 1030, 116, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5264, 3839, 3774, 4452, 4348, 3088, 2270, 5733] avg=4096.0000 max=5733 max_rank=7 min=2270 min_rank=6 imbalance_ratio=1.399658 max_load_reduction=27035 max_load_reduction_pct=0.825043 moved_to_echo=15439 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4019, 1741, 0, 0, 2, 0, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 3359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3853, 0, 0, 0, 0, 0, 0, 0, 0, 1407, 1, 0, 0, 1, 3496, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 6, 0, 97, 0, 0, 221, 2577, 273, 235, 0, 0, 0, 0, 0, 239, 0, 3037, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 2222, 10, 0, 2761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 0, 0, 0, 0, 0, 0, 0, 1593, 1040, 103, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5855, 3369, 3854, 4963, 3474, 3360, 5054, 2839] avg=4096.0000 max=5855 max_rank=0 min=2839 min_rank=7 imbalance_ratio=1.429443 max_load_reduction=26913 max_load_reduction_pct=0.821320 moved_to_echo=14727 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3465, 334, 0, 11, 2986, 30, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1195, 271, 3345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1771, 0, 3857, 0, 0, 0, 0, 0, 0, 0, 0, 466, 0, 23, 0, 0, 1320, 3252, 0, 0, 0, 0, 0, 0, 0, 0, 9, 158, 0, 0, 0, 0, 459, 173, 861, 241, 236, 0, 0, 0, 0, 0, 0, 0, 2691, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 450, 0, 1923, 276, 0, 0, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 0, 0, 0, 15, 1, 0, 3, 1389, 929, 106, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6923, 4811, 5633, 5061, 2137, 2725, 2873, 2605] avg=4096.0000 max=6923 max_rank=0 min=2137 min_rank=4 imbalance_ratio=1.690186 max_load_reduction=25845 max_load_reduction_pct=0.788727 moved_to_echo=10340 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1672, 82, 0, 0, 437, 0, 0, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 262, 0, 3869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3860, 0, 0, 0, 0, 0, 0, 0, 0, 1447, 0, 0, 0, 0, 15, 1479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 0, 0, 124, 2649, 270, 236, 0, 0, 0, 0, 0, 0, 0, 3051, 0, 0, 0, 3766, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2198, 0, 0, 46, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 1576, 1045, 119, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2299, 4131, 3860, 2941, 3392, 6817, 2503, 6825] avg=4096.0000 max=6825 max_rank=7 min=2299 min_rank=0 imbalance_ratio=1.666260 max_load_reduction=25943 max_load_reduction_pct=0.791718 moved_to_echo=19537 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[11392, 0, 128, 128, 128, 128, 128, 128, 12288, 8320, 1024, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 256, 128, 0, 768, 0, 512, 9088, 18432, 2176, 1920, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1536, 0, 17152, 512, 128, 9728, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 10112, 128, 128, 128, 128, 14208, 8576, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 23808, 3968, 0, 640, 3584, 256, 128, 896, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 128, 24064, 0, 0, 128, 8576, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 128, 0, 2048, 128, 30720, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 2688, 384, 29824, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33792 slack_slots=31744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4026, 5, 4088, 0, 0, 4096, 0, 3358, 0, 1, 0, 0, 0, 0, 0, 2, 30, 0, 0, 0, 0, 0, 0, 0, 0, 141, 0, 0, 4094, 0, 4094, 5, 0, 0, 0, 4, 2, 0, 0, 0, 0, 0, 8, 0, 1, 4088, 0, 0, 0, 0, 0, 0, 0, 557, 0, 1, 0, 0, 0, 3896, 0, 271] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 41, 3928, 0, 3854, 0, 279, 4054, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 4094, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 3322, 0, 0, 1, 0, 0, 0, 12, 45, 0, 0, 0, 0, 0, 3833, 1, 887] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4092, 0, 4069, 0, 2051, 4023, 0, 2748, 9, 0, 0, 0, 12, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 3, 72, 3, 0, 4072, 0, 4094, 1, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 129, 3974, 0, 0, 115, 0, 0, 0, 1, 170, 7, 0, 1, 0, 0, 2575, 0, 536] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[18, 147, 4012, 0, 4095, 0, 184, 923, 0, 3977, 0, 5, 0, 0, 0, 0, 0, 0, 0, 258, 1, 0, 67, 0, 0, 0, 0, 268, 0, 0, 3612, 0, 4096, 1853, 0, 1, 0, 0, 279, 0, 0, 0, 0, 0, 2, 0, 2757, 3933, 0, 0, 39, 0, 2, 0, 0, 61, 0, 0, 0, 0, 20, 1863, 0, 295] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4087, 47, 2962, 0, 2461, 4070, 3, 3398, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 496, 0, 0, 0, 0, 70, 0, 0, 4033, 0, 4092, 406, 0, 12, 0, 0, 0, 0, 9, 0, 0, 1, 0, 0, 211, 4030, 0, 10, 0, 1, 1, 0, 0, 128, 0, 1, 0, 0, 0, 1999, 0, 239] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 1, 3718, 111, 6, 0, 885, 4006, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2619, 0, 1, 0, 0, 7, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 3511, 0, 0, 0, 0, 0, 0, 242, 2869, 0, 0, 1, 0, 0, 1429, 0, 982] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4081, 12, 2529, 0, 5, 4096, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 3, 11, 0, 0, 3, 0, 0, 0, 0, 636, 0, 0, 4086, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 0, 0, 4, 0, 0, 0, 0, 1201, 0, 0, 0, 0, 23, 3550, 0, 280] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4096, 11, 428, 0, 77, 4096, 0, 282, 0, 0, 0, 0, 0, 0, 0, 0, 431, 0, 0, 0, 3114, 0, 0, 0, 0, 126, 0, 0, 4091, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 0, 4095, 0, 3681] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 4088, 0, 0, 615, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3358, 0, 1, 0, 0, 0, 0, 563, 334, 1, 0, 0, 0, 0, 0, 0, 2, 30, 0, 0, 0, 0, 0, 2918, 604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 0, 0, 4002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3760, 5, 0, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 1, 3879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 557, 3422, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3896, 0, 271, 208, 92, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4708, 4257, 3554, 4143, 3771, 3888, 3979, 4468] avg=4096.0000 max=4708 max_rank=0 min=3554 min_rank=2 imbalance_ratio=1.149414 max_load_reduction=28060 max_load_reduction_pct=0.856323 moved_to_echo=16106 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 41, 0, 0, 3854, 0, 279, 609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 557, 334, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2888, 589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3760, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 3152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 12, 45, 3339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3833, 1, 887, 169, 91, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4783, 4959, 3478, 3991, 3874, 3305, 3397, 4981] avg=4096.0000 max=4981 max_rank=7 min=3305 min_rank=5 imbalance_ratio=1.216064 max_load_reduction=27787 max_load_reduction_pct=0.847992 moved_to_echo=15557 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 111, 6, 0, 885, 601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 551, 334, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2619, 0, 2854, 557, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7, 0, 0, 4003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 3332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 242, 2869, 3161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1429, 0, 982, 178, 92, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1604, 4965, 6030, 4011, 3762, 3442, 6272, 2682] avg=4096.0000 max=6272 max_rank=6 min=1604 min_rank=0 imbalance_ratio=1.531250 max_load_reduction=26496 max_load_reduction_pct=0.808594 moved_to_echo=16158 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 47, 2962, 0, 2461, 611, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3398, 0, 0, 0, 0, 0, 0, 560, 334, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 496, 0, 2899, 614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 3943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3758, 406, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 1, 0, 0, 211, 3824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 1, 1, 0, 0, 128, 3473, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1999, 0, 239, 205, 90, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6081, 4296, 4010, 4013, 4176, 4045, 3613, 2534] avg=4096.0000 max=6081 max_rank=0 min=2534 min_rank=7 imbalance_ratio=1.484619 max_load_reduction=26687 max_load_reduction_pct=0.814423 moved_to_echo=14368 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4069, 0, 2051, 604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2748, 9, 0, 0, 0, 12, 1, 553, 334, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 2866, 614, 0, 0, 0, 0, 0, 0, 0, 0, 3, 72, 3, 0, 3981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3760, 1, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 3771, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 0, 1, 170, 3478, 0, 0, 0, 0, 0, 0, 0, 7, 0, 1, 0, 0, 2575, 0, 536, 202, 91, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6724, 3658, 3482, 4059, 3769, 3900, 3764, 3412] avg=4096.0000 max=6724 max_rank=0 min=3412 min_rank=7 imbalance_ratio=1.641602 max_load_reduction=26044 max_load_reduction_pct=0.794800 moved_to_echo=14845 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 12, 2529, 0, 5, 615, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 563, 334, 1, 0, 0, 0, 0, 0, 0, 0, 3, 11, 0, 0, 3, 0, 2918, 613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 636, 0, 0, 3995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3762, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 1201, 3468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 3550, 0, 280, 206, 91, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3161, 4998, 3548, 4631, 3763, 3844, 4673, 4150] avg=4096.0000 max=4998 max_rank=1 min=3161 min_rank=0 imbalance_ratio=1.220215 max_load_reduction=27770 max_load_reduction_pct=0.847473 moved_to_echo=16430 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[18, 147, 0, 0, 4095, 0, 184, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3977, 0, 5, 0, 0, 0, 0, 131, 338, 4, 0, 0, 0, 0, 0, 0, 0, 0, 258, 1, 0, 67, 0, 660, 603, 0, 0, 0, 0, 0, 0, 0, 0, 0, 268, 0, 0, 3527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3758, 1853, 0, 1, 0, 0, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2757, 3726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 2, 0, 0, 61, 3409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 1863, 0, 295, 203, 85, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4576, 4455, 1589, 3795, 5891, 6485, 3511, 2466] avg=4096.0000 max=6485 max_rank=5 min=1589 min_rank=2 imbalance_ratio=1.583252 max_load_reduction=26283 max_load_reduction_pct=0.802094 moved_to_echo=18353 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 11, 428, 0, 77, 615, 0, 0, 0, 0, 0, 0, 0, 0, 0, 282, 0, 0, 0, 0, 0, 0, 563, 334, 1, 0, 0, 0, 0, 0, 0, 0, 431, 0, 0, 0, 3114, 0, 2918, 614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 0, 0, 3999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 3482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3681, 207, 92, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1131, 1180, 7077, 4125, 3762, 3859, 3559, 8075] avg=4096.0000 max=8075 max_rank=7 min=1131 min_rank=0 imbalance_ratio=1.971436 max_load_reduction=24693 max_load_reduction_pct=0.753571 moved_to_echo=19255 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 256, 0, 256, 22144, 0, 6016, 4480, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 512, 384, 128, 0, 6400, 0, 20992, 4864, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 128, 23296, 128, 7296, 1664, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 26112, 128, 128, 0, 0, 128, 128, 4096, 2688, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 256, 128, 128, 0, 256, 5120, 27264, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[30208, 2432, 0, 128, 128, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 0, 128, 128, 0, 3456, 29440, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 1408, 128, 0, 31488, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 305, 2926, 0, 4, 0, 0, 0, 2, 0, 38, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 18, 0, 0, 2994, 4040, 0, 0, 0, 0, 0, 0, 0, 1, 118, 0, 0, 0, 1, 1900, 0, 0, 0, 0, 0, 4070, 0, 0, 4064] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 3925, 55, 0, 0, 0, 0, 103, 0, 0, 53, 0, 0, 0, 4081, 0, 0, 0, 0, 75, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4010, 0, 4087, 0, 0, 0, 0, 3269, 4096, 0, 0, 1, 0, 17, 0, 0, 0, 315, 368, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 3976, 0, 0, 273] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 38, 2648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 55, 4077, 0, 0, 0, 0, 3094, 4096, 0, 40, 0, 0, 11, 0, 0, 24, 0, 0, 0, 0, 0, 3537, 25, 0, 0, 0, 0, 4033, 0, 0, 2928] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 2875, 0, 0, 3, 0, 0, 0, 0, 0, 41, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4085, 0, 0, 0, 0, 1238, 4093, 6, 0, 0, 0, 0, 0, 0, 1, 10, 51, 0, 2, 0, 4085, 0, 0, 0, 1, 0, 4096, 0, 0, 3984] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 5, 0, 1, 0, 0, 1, 4088, 0, 0, 3, 0, 6, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3987, 17, 4088, 4, 0, 0, 0, 3969, 4090, 1, 0, 48, 0, 2, 0, 0, 0, 3211, 1206, 4, 0, 0, 3877, 0, 0, 0, 0, 0, 4090, 0, 1, 48] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4043, 0, 0, 0, 0, 0, 0, 7, 0, 61, 0, 0, 0, 3928, 0, 10, 0, 0, 0, 0, 30, 0, 0, 3223, 0, 279, 0, 0, 0, 4033, 238, 3832, 0, 0, 0, 0, 1005, 4089, 0, 0, 11, 0, 0, 0, 0, 0, 105, 12, 96, 0, 0, 3434, 0, 26, 27, 0, 0, 4096, 0, 0, 183] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 3699, 24, 0, 0, 0, 0, 18, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 10, 0, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4085, 0, 0, 0, 0, 309, 4096, 0, 0, 0, 0, 0, 0, 0, 7, 68, 0, 0, 0, 0, 4054, 11, 4, 0, 0, 0, 4093, 0, 0, 4092] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 45, 2329, 0, 0, 0, 0, 5, 0, 0, 262, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 4096, 0, 0, 0, 0, 1419, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 69, 14, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 4083, 0, 0, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 305, 2926, 0, 4, 0, 985, 130, 3, 0, 0, 0, 0, 0, 0, 0, 2, 0, 38, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2438, 1198, 35, 0, 0, 0, 0, 0, 1913, 0, 0, 0, 18, 0, 0, 2994, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 118, 0, 0, 0, 1, 1770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1632, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4353, 4100, 4100, 3671, 4925, 4033, 1890, 5696] avg=4096.0000 max=5696 max_rank=7 min=1890 min_rank=6 imbalance_ratio=1.390625 max_load_reduction=27072 max_load_reduction_pct=0.826172 moved_to_echo=16544 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 38, 2648, 0, 0, 0, 978, 242, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2416, 1189, 35, 0, 0, 0, 0, 0, 1899, 55, 0, 0, 0, 0, 0, 3094, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 40, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 3295, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1617, 0, 0, 2928, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3909, 4042, 4100, 3640, 5048, 4140, 3344, 4545] avg=4096.0000 max=5048 max_rank=4 min=3344 min_rank=6 imbalance_ratio=1.232422 max_load_reduction=27720 max_load_reduction_pct=0.845947 moved_to_echo=17077 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2875, 0, 0, 3, 0, 985, 280, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 0, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2454, 1197, 35, 0, 0, 0, 0, 0, 1911, 0, 0, 0, 0, 0, 0, 1238, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 10, 51, 0, 2, 0, 3805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1642, 0, 0, 3984, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4146, 4091, 4108, 3686, 3149, 4092, 3869, 5627] avg=4096.0000 max=5627 max_rank=7 min=3149 min_rank=4 imbalance_ratio=1.373779 max_load_reduction=27141 max_load_reduction_pct=0.828278 moved_to_echo=16737 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 20, 0, 0, 0, 0, 959, 266, 3, 0, 0, 0, 0, 0, 0, 0, 5, 0, 1, 0, 0, 1, 4054, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 3, 0, 6, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2450, 1166, 34, 0, 0, 0, 0, 0, 1862, 17, 0, 4, 0, 0, 0, 3969, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 1, 0, 48, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3211, 1206, 4, 0, 0, 3611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1640, 0, 1, 48, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1248, 4061, 4101, 3651, 5852, 4134, 8032, 1689] avg=4096.0000 max=8032 max_rank=6 min=1248 min_rank=0 imbalance_ratio=1.960938 max_load_reduction=24736 max_load_reduction_pct=0.754883 moved_to_echo=19707 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3925, 55, 0, 0, 0, 965, 278, 3, 0, 0, 0, 0, 0, 0, 103, 0, 0, 53, 0, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 75, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 2382, 1173, 34, 0, 0, 0, 0, 0, 1872, 0, 0, 0, 0, 0, 0, 3269, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 1, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 315, 368, 0, 0, 0, 3782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1594, 0, 0, 273, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5226, 4209, 4160, 3593, 5141, 4107, 4465, 1867] avg=4096.0000 max=5226 max_rank=0 min=1867 min_rank=7 imbalance_ratio=1.275879 max_load_reduction=27542 max_load_reduction_pct=0.840515 moved_to_echo=15580 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3699, 24, 0, 0, 0, 984, 278, 3, 0, 0, 0, 0, 0, 0, 18, 0, 0, 1, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 10, 0, 6, 5, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2452, 1196, 35, 0, 0, 0, 0, 0, 1910, 0, 0, 0, 0, 0, 0, 309, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 68, 0, 0, 0, 0, 3776, 11, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1641, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4988, 4069, 4121, 3683, 2219, 4089, 3862, 5737] avg=4096.0000 max=5737 max_rank=7 min=2219 min_rank=4 imbalance_ratio=1.400635 max_load_reduction=27031 max_load_reduction_pct=0.824921 moved_to_echo=15907 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4043, 0, 0, 0, 0, 974, 239, 6, 0, 0, 0, 0, 0, 0, 0, 7, 0, 61, 0, 0, 0, 3800, 0, 0, 0, 0, 0, 0, 0, 3928, 0, 10, 0, 0, 0, 0, 30, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3223, 0, 279, 0, 0, 0, 2456, 1184, 32, 0, 0, 0, 0, 0, 1875, 238, 0, 0, 0, 0, 0, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 12, 96, 0, 0, 3195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 27, 0, 0, 1640, 0, 0, 183, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5262, 3868, 3978, 7174, 3118, 4084, 3408, 1876] avg=4096.0000 max=7174 max_rank=3 min=1876 min_rank=7 imbalance_ratio=1.751465 max_load_reduction=25594 max_load_reduction_pct=0.781067 moved_to_echo=12486 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 45, 2329, 0, 0, 0, 978, 281, 3, 0, 0, 0, 0, 0, 0, 5, 0, 0, 262, 0, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2446, 1189, 35, 0, 0, 0, 0, 0, 1897, 0, 0, 0, 0, 0, 0, 1419, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 14, 1, 0, 0, 3814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1637, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3636, 4328, 4100, 3670, 3316, 4089, 3898, 5731] avg=4096.0000 max=5731 max_rank=7 min=3316 min_rank=4 imbalance_ratio=1.399170 max_load_reduction=27037 max_load_reduction_pct=0.825104 moved_to_echo=17034 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 3328, 0, 384, 0, 0, 0, 19584, 9600, 384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[15232, 384, 0, 128, 128, 0, 0, 17408, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 0, 14976, 8064, 0, 128, 0, 7808, 2048, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 3968, 1664, 128, 128, 128, 27136, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 0, 13056, 0, 128, 19712, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 0, 512, 0, 0, 128, 32256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32640, 128, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[32640, 0, 128, 128, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[11, 1, 4095, 5, 0, 2, 0, 0, 4050, 0, 24, 29, 0, 3314, 0, 4080, 0, 0, 0, 0, 4096, 4094, 4, 0, 52, 0, 0, 3, 0, 3, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 331, 0, 0, 0, 0, 1, 0, 0, 0, 6, 385, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 1, 4088, 0, 0, 0, 0, 7, 4081, 0, 0, 69, 17, 2509, 0, 3957, 5, 0, 0, 0, 4096, 3966, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 4078, 0, 1590, 0, 34, 0, 0, 0, 0, 0, 3, 5, 160, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 1248, 0, 0, 2, 0, 0, 4095, 0, 31, 0, 0, 2966, 1, 4092, 0, 18, 0, 8, 4095, 4088, 17, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4080, 0, 0, 0, 0, 1, 0, 0, 0, 4, 1, 0, 4079, 0, 3930, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 14, 3200, 0, 0, 0, 0, 0, 3934, 0, 8, 0, 0, 327, 0, 4044, 154, 0, 0, 1, 4096, 4009, 234, 0, 52, 0, 0, 0, 0, 53, 594, 0, 4028, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 45, 4096, 0, 3801, 0, 6, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4096, 2, 2, 0, 0, 0, 4087, 0, 15, 0, 26, 185, 0, 4093, 0, 0, 0, 0, 4082, 3903, 0, 0, 0, 0, 0, 1, 0, 0, 1, 3, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 4061, 3, 4085, 0, 0, 0, 12, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 3957, 0, 0, 0, 0, 0, 4083, 0, 167, 0, 3, 1699, 0, 4094, 97, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2271, 0, 0, 0, 5, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4, 0, 3303, 1, 237, 0, 0, 43, 4077, 0, 0, 392, 15, 3213, 0, 4007, 0, 0, 0, 0, 4096, 4092, 0, 0, 96, 0, 65, 0, 0, 81, 0, 0, 2974, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4096, 0, 943, 0, 0, 0, 0, 986, 0, 0, 0, 19, 22, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1, 0, 3936, 0, 0, 0, 0, 0, 3683, 0, 0, 58, 0, 2917, 0, 3959, 319, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1438, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[11, 1, 4095, 5, 0, 2, 0, 0, 569, 0, 0, 0, 0, 0, 0, 0, 1867, 0, 24, 29, 0, 3314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4040, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 3, 0, 3, 0, 0, 3936, 69, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 98, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1792, 0, 331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6, 2304, 1516, 144, 0, 0, 0, 0, 0, 385, 0, 0, 0, 0, 0, 0, 0, 4027, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4683, 5234, 4044, 4063, 4238, 2123, 3971, 4412] avg=4096.0000 max=5234 max_rank=1 min=2123 min_rank=5 imbalance_ratio=1.277832 max_load_reduction=27534 max_load_reduction_pct=0.840271 moved_to_echo=14744 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4088, 0, 0, 0, 0, 7, 573, 0, 0, 0, 0, 0, 0, 0, 1882, 0, 0, 69, 17, 2509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 3913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 3818, 69, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 99, 53, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 1784, 0, 1590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 3, 5, 2294, 1527, 139, 0, 0, 0, 0, 0, 160, 0, 0, 0, 0, 0, 0, 0, 4027, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4669, 4477, 3918, 3889, 4248, 3378, 4002, 4187] avg=4096.0000 max=4669 max_rank=0 min=3378 min_rank=5 imbalance_ratio=1.139893 max_load_reduction=28099 max_load_reduction_pct=0.857513 moved_to_echo=15815 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 14, 3200, 0, 0, 0, 0, 0, 553, 0, 0, 0, 0, 0, 0, 0, 1814, 0, 8, 0, 0, 327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 0, 0, 1, 0, 3956, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 53, 594, 0, 3902, 69, 0, 0, 0, 0, 0, 0, 4028, 0, 0, 0, 0, 0, 0, 3, 95, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 1792, 0, 3801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 69, 0, 0, 0, 0, 0, 0, 2304, 1472, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3767, 2149, 4345, 4670, 4179, 5638, 3993, 4027] avg=4096.0000 max=5638 max_rank=5 min=2149 min_rank=1 imbalance_ratio=1.376465 max_load_reduction=27130 max_load_reduction_pct=0.827942 moved_to_echo=17837 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1248, 0, 0, 2, 0, 0, 575, 0, 0, 0, 0, 0, 0, 0, 1889, 0, 31, 0, 0, 2966, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 8, 0, 4034, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 3949, 69, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 1, 0, 0, 99, 54, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 1785, 0, 3930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 2294, 1532, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1825, 4887, 4077, 4027, 4234, 5720, 3972, 4026] avg=4096.0000 max=5720 max_rank=5 min=1825 min_rank=0 imbalance_ratio=1.396484 max_load_reduction=27048 max_load_reduction_pct=0.825439 moved_to_echo=17952 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 2, 2, 0, 0, 0, 574, 0, 0, 0, 0, 0, 0, 0, 1885, 0, 15, 0, 26, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 3, 3949, 68, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 99, 52, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 1777, 3, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 9, 2284, 1529, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4014, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4674, 2111, 3851, 4022, 4239, 5879, 3978, 4014] avg=4096.0000 max=5879 max_rank=5 min=2111 min_rank=1 imbalance_ratio=1.435303 max_load_reduction=26889 max_load_reduction_pct=0.820587 moved_to_echo=18110 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3957, 0, 0, 0, 0, 0, 574, 0, 0, 0, 0, 0, 0, 0, 1882, 0, 167, 0, 3, 1699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 0, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 3951, 69, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 99, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1792, 0, 2271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 0, 0, 0, 2304, 1528, 143, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4026, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4531, 3751, 4139, 4023, 4253, 4063, 3981, 4027] avg=4096.0000 max=4531 max_rank=0 min=3751 min_rank=1 imbalance_ratio=1.106201 max_load_reduction=28237 max_load_reduction_pct=0.861725 moved_to_echo=16324 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 3303, 1, 237, 0, 0, 43, 577, 0, 0, 0, 0, 0, 0, 0, 1865, 0, 0, 392, 15, 3213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 65, 0, 0, 81, 0, 0, 3867, 70, 0, 0, 0, 0, 0, 0, 2974, 0, 0, 0, 0, 0, 0, 0, 105, 59, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 1788, 0, 943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 986, 0, 0, 0, 19, 2308, 1530, 140, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 4026, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4165, 5485, 4033, 4179, 3138, 2737, 4983, 4048] avg=4096.0000 max=5485 max_rank=1 min=2737 min_rank=5 imbalance_ratio=1.339111 max_load_reduction=27283 max_load_reduction_pct=0.832611 moved_to_echo=14906 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 3936, 0, 0, 0, 0, 0, 517, 0, 0, 0, 0, 0, 0, 0, 1699, 0, 0, 58, 0, 2917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 319, 0, 0, 0, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 3821, 69, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 89, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1792, 0, 1438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 0, 2304, 1378, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4454, 4674, 4361, 3895, 4239, 3230, 3888, 4027] avg=4096.0000 max=4674 max_rank=1 min=3230 min_rank=5 imbalance_ratio=1.141113 max_load_reduction=28094 max_load_reduction_pct=0.857361 moved_to_echo=15384 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 1024, 0, 128, 128, 128, 18432, 12032, 1152, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 14336, 128, 18432, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[14848, 0, 256, 640, 128, 17152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 28032, 128, 256, 128, 0, 128, 4608, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[640, 0, 0, 0, 0, 0, 0, 0, 32256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[31616, 0, 0, 128, 0, 128, 0, 128, 896, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[640, 128, 0, 128, 0, 32000, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[256, 0, 128, 128, 0, 256, 640, 128, 31232, 640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 30, 0, 98, 0, 9, 0, 3070, 0, 0, 0, 0, 0, 0, 4017, 0, 0, 0, 3868, 4051, 0, 0, 0, 0, 0, 4088, 0, 2, 0, 0, 0, 16, 27, 3200, 0, 0, 0, 1, 0, 0, 0, 174, 0, 0, 0, 0, 2, 2543, 0, 0, 0, 0, 0, 0, 0, 3480, 5, 0, 0, 0, 4087] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 8, 0, 20, 6, 78, 0, 62, 0, 941, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 4046, 4096, 0, 2, 6, 0, 0, 4096, 1, 0, 0, 0, 0, 56, 0, 4017, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 3468, 0, 0, 0, 0, 0, 0, 0, 3698, 0, 0, 0, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 1, 0, 0, 0, 2, 4096, 0, 4, 7, 4075, 4092, 0, 56, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 64, 0, 4039, 0, 0, 0, 1, 0, 0, 0, 16, 0, 0, 0, 0, 0, 4, 8, 0, 0, 0, 0, 0, 0, 4040, 0, 0, 0, 3, 4090] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 6, 0, 0, 0, 70, 0, 733, 0, 1894, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4036, 4080, 0, 21, 69, 0, 0, 4096, 0, 0, 0, 0, 0, 1486, 0, 3616, 0, 0, 4, 0, 0, 0, 0, 7, 0, 0, 0, 4, 0, 517, 5, 0, 0, 0, 0, 0, 0, 3937, 0, 0, 0, 0, 4090] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 9, 0, 0, 0, 7, 0, 2, 0, 4081, 1, 0, 8, 0, 0, 0, 3796, 0, 0, 0, 4096, 4092, 0, 1, 0, 0, 0, 4074, 0, 19, 0, 0, 0, 1347, 0, 742, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 2160, 225, 0, 0, 0, 0, 0, 0, 3994, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 363, 1, 4095, 1, 0, 0, 0, 0, 0, 4046, 0, 0, 1, 4080, 4095, 0, 0, 0, 3, 0, 4092, 0, 4, 0, 0, 0, 1900, 0, 876, 0, 0, 0, 0, 0, 0, 1, 742, 3, 0, 0, 0, 8, 287, 0, 0, 1, 9, 0, 0, 0, 4080, 0, 0, 1, 0, 4079] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 452, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 3981, 4009, 0, 2257, 47, 0, 0, 4089, 0, 0, 0, 0, 0, 3818, 0, 3068, 0, 0, 1, 0, 0, 0, 0, 211, 0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 0, 2859, 0, 0, 0, 0, 3815] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 9, 2313, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1852, 0, 0, 0, 0, 0, 0, 0, 4014, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 30, 0, 98, 0, 3972, 0, 0, 0, 0, 0, 0, 0, 9, 0, 3070, 0, 0, 0, 0, 0, 1133, 256, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 3868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3784, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 27, 3200, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 174, 0, 0, 0, 3849, 79, 0, 0, 0, 0, 0, 0, 0, 2, 2543, 0, 0, 0, 0, 0, 2618, 108, 0, 0, 0, 0, 0, 0, 0, 0, 3480, 5, 0, 0, 0, 336, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4468, 3928, 3786, 3291, 4103, 5271, 3821] avg=4096.0000 max=5271 max_rank=6 min=3291 min_rank=4 imbalance_ratio=1.286865 max_load_reduction=27497 max_load_reduction_pct=0.839142 moved_to_echo=16486 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 1, 0, 0, 0, 1134, 256, 0, 0, 0, 0, 0, 0, 2, 62, 0, 4, 7, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 3792, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 4039, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 16, 0, 0, 0, 3924, 79, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8, 0, 0, 0, 0, 2620, 110, 0, 0, 0, 0, 0, 0, 0, 0, 4040, 0, 0, 0, 3, 336, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4013, 5464, 4150, 3849, 4151, 4020, 2742, 4379] avg=4096.0000 max=5464 max_rank=1 min=2742 min_rank=6 imbalance_ratio=1.333984 max_load_reduction=27304 max_load_reduction_pct=0.833252 moved_to_echo=15292 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8, 0, 20, 6, 78, 0, 4016, 0, 0, 0, 0, 0, 0, 0, 62, 0, 941, 0, 0, 0, 0, 0, 1135, 256, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6, 0, 0, 3792, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 4017, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 3897, 80, 0, 0, 0, 0, 0, 0, 0, 0, 3468, 0, 0, 0, 0, 0, 2624, 110, 0, 0, 0, 0, 0, 0, 0, 0, 3698, 0, 0, 0, 0, 336, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 2394, 4106, 3801, 4121, 3982, 6202, 4034] avg=4096.0000 max=6202 max_rank=6 min=2394 min_rank=1 imbalance_ratio=1.514160 max_load_reduction=26566 max_load_reduction_pct=0.810730 moved_to_echo=18339 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 9, 0, 0, 0, 7, 0, 4013, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4081, 1, 0, 8, 0, 0, 1135, 255, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3771, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1347, 0, 742, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 3637, 79, 0, 0, 0, 0, 0, 0, 0, 0, 2160, 225, 0, 0, 0, 0, 2624, 102, 0, 0, 0, 0, 0, 0, 0, 0, 3994, 0, 0, 0, 0, 337, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4029, 5482, 4153, 3791, 2137, 3734, 5111, 4331] avg=4096.0000 max=5482 max_rank=1 min=2137 min_rank=4 imbalance_ratio=1.338379 max_load_reduction=27286 max_load_reduction_pct=0.832703 moved_to_echo=15313 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4016, 0, 0, 0, 0, 0, 0, 0, 363, 1, 4095, 1, 0, 0, 0, 0, 1131, 256, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 1, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3788, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1900, 0, 876, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 742, 3, 0, 0, 3876, 79, 0, 0, 0, 0, 0, 0, 0, 8, 287, 0, 0, 1, 9, 0, 2613, 109, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 1, 0, 335, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4016, 5847, 4142, 3795, 2824, 4701, 3027, 4416] avg=4096.0000 max=5847 max_rank=1 min=2824 min_rank=4 imbalance_ratio=1.427490 max_load_reduction=26921 max_load_reduction_pct=0.821564 moved_to_echo=14968 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6, 0, 0, 0, 70, 0, 4002, 0, 0, 0, 0, 0, 0, 0, 733, 0, 1894, 0, 0, 0, 0, 0, 1134, 256, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 1, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 69, 0, 0, 3792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1486, 0, 3616, 0, 0, 4, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 3924, 78, 0, 0, 0, 0, 0, 0, 4, 0, 517, 5, 0, 0, 0, 0, 2620, 110, 0, 0, 0, 0, 0, 0, 0, 0, 3937, 0, 0, 0, 0, 336, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4017, 4099, 3882, 5154, 4009, 3256, 4273] avg=4096.0000 max=5154 max_rank=4 min=3256 min_rank=6 imbalance_ratio=1.258301 max_load_reduction=27614 max_load_reduction_pct=0.842712 moved_to_echo=16692 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 452, 0, 3932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 1061, 262, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 3981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2257, 47, 0, 0, 3773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3818, 0, 3068, 0, 0, 1, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 211, 0, 0, 0, 3925, 77, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 2450, 116, 0, 0, 0, 0, 0, 0, 0, 0, 2859, 0, 0, 0, 0, 304, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4386, 1383, 4032, 6077, 6941, 4213, 2573, 3163] avg=4096.0000 max=6941 max_rank=4 min=1383 min_rank=1 imbalance_ratio=1.694580 max_load_reduction=25827 max_load_reduction_pct=0.788177 moved_to_echo=16890 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 4017, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2313, 0, 0, 0, 0, 0, 1135, 256, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3924, 79, 0, 0, 0, 0, 0, 0, 0, 0, 1852, 0, 0, 0, 0, 0, 2624, 110, 0, 0, 0, 0, 0, 0, 0, 0, 4014, 0, 0, 0, 0, 337, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4018, 3713, 4158, 3787, 4149, 4006, 4586, 4351] avg=4096.0000 max=4586 max_rank=6 min=3713 min_rank=1 imbalance_ratio=1.119629 max_load_reduction=28182 max_load_reduction_pct=0.860046 moved_to_echo=17092 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 10880, 256, 0, 128, 128, 0, 20864, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[1280, 128, 20608, 128, 128, 128, 0, 0, 9088, 2176, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 8704, 128, 23680, 0, 0, 128, 512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 30208, 128, 0, 128, 128, 2688, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 128, 128, 128, 128, 768, 0, 32000, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[2432, 128, 128, 0, 30336, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 0, 128, 1280, 128, 0, 0, 30976, 640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 512, 0, 128, 128, 32384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 762, 0, 3266, 0, 3, 0, 3, 0, 0, 0, 98, 0, 0, 0, 0, 4089, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4094, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4093, 4087, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4072, 0, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1584, 70, 2264, 0, 0, 0, 3, 0, 0, 0, 58, 0, 0, 7, 0, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3464, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4004, 3960, 0, 0, 0, 0, 0, 0, 1, 1215, 0, 0, 0, 0, 0, 3912, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 15, 0, 1274, 0, 0, 0, 958, 0, 0, 0, 470, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 40, 0, 8, 0, 0, 0, 4094, 4061, 0, 209, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4058, 4096, 4015, 0, 0, 0, 45, 0, 0, 2, 1979, 0, 0, 0, 0, 0, 3347, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1049, 0, 2976, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 4091, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4064, 4078, 0, 148, 0, 0, 1, 0, 0, 0, 0, 15, 0, 0, 0, 4094, 4077, 4096, 0, 0, 0, 34, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4032, 0, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 17, 0, 2946, 0, 0, 0, 1160, 2, 1, 0, 21, 0, 0, 0, 0, 4095, 0, 5, 0, 3, 0, 1, 6, 0, 65, 1, 3730, 4090, 10, 235, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 4092, 4092, 0, 1, 1, 2, 1, 0, 0, 17, 0, 1, 0, 21, 0, 4071, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[6, 0, 0, 0, 727, 5, 3150, 0, 0, 1, 251, 0, 0, 0, 0, 31, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 71, 53, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4083, 4068, 4096, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 3924, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1383, 0, 4081, 0, 0, 0, 178, 0, 16, 0, 0, 51, 0, 19, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 903, 4033, 356, 3244, 0, 0, 1, 0, 0, 0, 0, 5, 0, 0, 0, 4036, 3415, 3714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3237, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 17, 0, 2852, 8, 0, 0, 8, 0, 0, 0, 57, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 4096, 0, 832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 388, 0, 0, 0, 0, 0, 4032, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 762, 0, 3266, 0, 541, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 0, 98, 0, 3688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 2, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 2769, 701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 3283, 406, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4072, 0, 3, 264, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4569, 3792, 4094, 4609, 3473, 4196, 3695, 4340] avg=4096.0000 max=4609 max_rank=3 min=3473 min_rank=4 imbalance_ratio=1.125244 max_load_reduction=28159 max_load_reduction_pct=0.859344 moved_to_echo=15704 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 15, 0, 1274, 0, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 958, 0, 0, 0, 470, 0, 3655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 40, 0, 8, 0, 0, 0, 4094, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 209, 0, 0, 0, 0, 1, 0, 2746, 689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 2, 1979, 3225, 403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3347, 0, 0, 262, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1825, 5083, 4099, 4656, 3645, 4197, 5654, 3609] avg=4096.0000 max=5654 max_rank=6 min=1825 min_rank=0 imbalance_ratio=1.380371 max_load_reduction=27114 max_load_reduction_pct=0.827454 moved_to_echo=17105 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1584, 70, 2264, 0, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 58, 0, 3684, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4040, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3464, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2770, 679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4004, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1215, 3181, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3912, 0, 0, 264, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4459, 3745, 4050, 3982, 3449, 4104, 4803, 4176] avg=4096.0000 max=4803 max_rank=6 min=3449 min_rank=4 imbalance_ratio=1.172607 max_load_reduction=27965 max_load_reduction_pct=0.853424 moved_to_echo=16532 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 17, 0, 2946, 0, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1160, 2, 1, 0, 21, 0, 3676, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 5, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 65, 1, 3730, 517, 0, 0, 0, 0, 0, 0, 0, 0, 10, 235, 1, 0, 0, 0, 0, 0, 2766, 702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 1, 0, 0, 17, 3287, 404, 0, 0, 0, 0, 0, 0, 0, 1, 0, 21, 0, 4071, 0, 0, 264, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3503, 4860, 4106, 4320, 3714, 4195, 3713, 4357] avg=4096.0000 max=4860 max_rank=1 min=3503 min_rank=0 imbalance_ratio=1.186523 max_load_reduction=27908 max_load_reduction_pct=0.851685 moved_to_echo=15979 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1049, 0, 2976, 0, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3688, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4091, 0, 0, 0, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 0, 0, 1, 0, 0, 0, 2758, 702, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 4077, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 3, 0, 3290, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 0, 4, 263, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4564, 3689, 4099, 4579, 3609, 4196, 3733, 4299] avg=4096.0000 max=4579 max_rank=3 min=3609 min_rank=4 imbalance_ratio=1.117920 max_load_reduction=28189 max_load_reduction_pct=0.860260 moved_to_echo=15837 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 0, 0, 727, 5, 3150, 0, 541, 0, 0, 0, 0, 0, 0, 0, 0, 1, 251, 0, 0, 0, 0, 31, 3679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 517, 0, 0, 0, 0, 0, 0, 0, 0, 71, 53, 0, 0, 0, 7, 0, 0, 2767, 702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 3290, 404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3924, 0, 0, 264, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4429, 3962, 4099, 4613, 3600, 4172, 3705, 4188] avg=4096.0000 max=4613 max_rank=3 min=3600 min_rank=4 imbalance_ratio=1.126221 max_load_reduction=28155 max_load_reduction_pct=0.859222 moved_to_echo=15665 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1383, 0, 4081, 0, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 0, 16, 0, 0, 51, 3636, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 4096, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 903, 491, 0, 0, 0, 0, 0, 0, 0, 0, 356, 3244, 0, 0, 1, 0, 0, 0, 2732, 641, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 3415, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2985, 400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3237, 0, 0, 266, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6001, 3881, 4122, 1394, 6974, 3508, 3385, 3503] avg=4096.0000 max=6974 max_rank=4 min=1394 min_rank=3 imbalance_ratio=1.702637 max_load_reduction=25794 max_load_reduction_pct=0.787170 moved_to_echo=17370 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 17, 0, 2852, 8, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 57, 0, 3691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 832, 0, 0, 0, 0, 0, 0, 2770, 702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 388, 3287, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 0, 0, 264, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3418, 3756, 4099, 4615, 4304, 4200, 4080, 4296] avg=4096.0000 max=4615 max_rank=3 min=3418 min_rank=0 imbalance_ratio=1.126709 max_load_reduction=28153 max_load_reduction_pct=0.859161 moved_to_echo=16880 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 128, 0, 128, 3712, 25856, 3328, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[512, 4736, 128, 0, 128, 128, 128, 0, 22144, 5632, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 5632, 128, 22912, 128, 4352, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 2688, 128, 128, 0, 768, 128, 29440, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 128, 128, 28544, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 0, 30720, 0, 128, 2176, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 0, 0, 32000, 896, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 0, 32768, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[3356, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 10, 0, 0, 0, 4089, 0, 3920, 0, 0, 0, 1, 0, 847, 0, 5, 0, 1, 0, 0, 4090, 0, 0, 0, 0, 0, 4096, 0, 0, 4091, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4086, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 6, 0, 0, 3946, 0, 4048, 0, 0, 0, 0, 1, 2571, 0, 0, 0, 18, 0, 0, 4095, 0, 0, 0, 5, 3, 4096, 182, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 8, 0, 0, 0, 4095, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[3568, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 12, 0, 0, 3, 0, 0, 4021, 3, 4093, 0, 0, 0, 0, 0, 595, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 3, 1, 0, 4094, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 4087, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[469, 0, 0, 1744, 0, 0, 0, 28, 0, 278, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3909, 0, 4096, 0, 0, 0, 0, 0, 1819, 0, 241, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 1, 0, 0, 0, 3827, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[4053, 0, 0, 0, 0, 0, 33, 0, 0, 23, 0, 0, 0, 0, 0, 0, 5, 26, 0, 0, 0, 0, 0, 3953, 0, 2805, 0, 0, 0, 0, 96, 3638, 0, 1281, 0, 30, 0, 0, 4096, 0, 0, 0, 0, 0, 4075, 0, 0, 3994, 0, 0, 1, 0, 0, 49, 0, 0, 0, 4096, 13, 0, 0, 0, 501, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[491, 1, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 1, 0, 19, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 3545, 0, 1, 0, 0, 0, 0, 4093, 2, 0, 0, 0, 0, 4096, 0, 0, 4096, 157, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 12, 0, 0, 0, 3891, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[227, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4022, 0, 4096, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 3958, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1073, 1, 0, 0, 0, 0, 0, 0, 1, 14, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 0, 3166, 0, 1, 0, 0, 0, 3949, 0, 18, 0, 0, 0, 0, 4094, 0, 1, 0, 1, 0, 4094, 31, 0, 4095, 20, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 8, 0, 0, 0, 4074, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1502, 0, 0, 0, 0, 0, 0, 0, 2011, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 3559, 484, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 3946, 62, 14, 0, 0, 0, 0, 0, 0, 0, 1553, 0, 0, 0, 0, 1, 2571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 3898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3, 0, 182, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 525, 8, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3648, 4083, 4028, 4125, 3916, 4269, 4071, 4628] avg=4096.0000 max=4628 max_rank=7 min=3648 min_rank=0 imbalance_ratio=1.129883 max_load_reduction=28140 max_load_reduction_pct=0.858765 moved_to_echo=16884 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3568, 1, 0, 0, 0, 0, 0, 0, 2033, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 3568, 489, 25, 0, 0, 0, 0, 0, 0, 12, 0, 0, 3, 0, 0, 4021, 62, 14, 0, 0, 0, 0, 0, 0, 3, 1571, 0, 0, 0, 0, 0, 595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 527, 0, 0, 1, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5737, 4086, 4112, 2169, 3896, 4083, 4070, 4615] avg=4096.0000 max=5737 max_rank=0 min=2169 min_rank=3 imbalance_ratio=1.400635 max_load_reduction=27031 max_load_reduction_pct=0.824921 moved_to_echo=16664 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3356, 0, 0, 73, 0, 0, 0, 0, 1947, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3567, 468, 25, 0, 0, 0, 0, 0, 0, 8, 0, 10, 0, 0, 0, 4089, 62, 14, 0, 0, 0, 0, 0, 0, 0, 1505, 0, 0, 0, 1, 0, 847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 1, 0, 0, 3893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 527, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5511, 4060, 4183, 2353, 3899, 4077, 4072, 4613] avg=4096.0000 max=5511 max_rank=0 min=2353 min_rank=3 imbalance_ratio=1.345459 max_load_reduction=27257 max_load_reduction_pct=0.831818 moved_to_echo=16661 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[469, 0, 0, 1744, 0, 0, 0, 28, 2035, 135, 0, 0, 0, 0, 0, 0, 0, 278, 0, 0, 0, 7, 0, 0, 3545, 489, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3909, 62, 14, 0, 0, 0, 0, 0, 0, 0, 1572, 0, 0, 0, 0, 0, 1819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 241, 0, 0, 0, 0, 3899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 523, 1, 0, 0, 0, 3827, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4411, 4344, 3985, 3391, 4140, 4081, 4065, 4351] avg=4096.0000 max=4411 max_rank=0 min=3391 min_rank=3 imbalance_ratio=1.076904 max_load_reduction=28357 max_load_reduction_pct=0.865387 moved_to_echo=16637 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[491, 1, 0, 0, 0, 0, 0, 0, 2034, 135, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 0, 0, 3569, 489, 25, 0, 0, 0, 0, 0, 0, 1, 0, 19, 0, 0, 0, 4096, 62, 14, 0, 0, 0, 0, 0, 0, 0, 1571, 0, 0, 0, 0, 0, 3545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3896, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 157, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 527, 12, 0, 0, 0, 3891, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2661, 4160, 4192, 5116, 3899, 4082, 4228, 4430] avg=4096.0000 max=5116 max_rank=3 min=2661 min_rank=0 imbalance_ratio=1.249023 max_load_reduction=27652 max_load_reduction_pct=0.843872 moved_to_echo=16639 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4053, 0, 0, 0, 0, 0, 33, 0, 1398, 137, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 3574, 340, 26, 0, 0, 0, 0, 0, 5, 26, 0, 0, 0, 0, 0, 3953, 66, 14, 0, 0, 0, 0, 0, 0, 0, 1067, 0, 0, 0, 0, 96, 3638, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1281, 0, 30, 0, 0, 3893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 49, 0, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 522, 13, 0, 0, 0, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5621, 3963, 4064, 4801, 5204, 3980, 4099, 1036] avg=4096.0000 max=5621 max_rank=0 min=1036 min_rank=7 imbalance_ratio=1.372314 max_load_reduction=27147 max_load_reduction_pct=0.828461 moved_to_echo=14319 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[227, 0, 0, 0, 0, 0, 0, 0, 2035, 135, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3569, 489, 25, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4022, 62, 14, 0, 0, 0, 0, 0, 0, 0, 1572, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 527, 1, 0, 0, 0, 3958, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2397, 4084, 4099, 5648, 3899, 4082, 4073, 4486] avg=4096.0000 max=5648 max_rank=3 min=2397 min_rank=0 imbalance_ratio=1.378906 max_load_reduction=27120 max_load_reduction_pct=0.827637 moved_to_echo=16540 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1073, 1, 0, 0, 0, 0, 0, 0, 1573, 135, 0, 0, 0, 0, 0, 0, 1, 14, 1, 0, 0, 0, 1, 0, 3568, 378, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 62, 14, 0, 0, 0, 0, 0, 0, 0, 1215, 0, 1, 0, 0, 0, 3949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 3897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 31, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 1, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 527, 8, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2782, 3988, 4105, 5165, 3915, 4114, 4090, 4609] avg=4096.0000 max=5165 max_rank=3 min=2782 min_rank=0 imbalance_ratio=1.260986 max_load_reduction=27603 max_load_reduction_pct=0.842377 moved_to_echo=16728 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[14848, 128, 0, 1920, 0, 0, 128, 128, 15104, 1152, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 11648, 0, 128, 0, 128, 128, 21120, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 4224, 128, 0, 128, 0, 28544, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 512, 128, 0, 0, 128, 128, 0, 28544, 3712, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 1664, 0, 128, 0, 0, 31232, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 128, 0, 0, 32128, 512, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[256, 0, 128, 128, 0, 128, 0, 128, 32640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 0, 256, 0, 32640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 3897, 0, 0, 0, 0, 3, 0, 0, 0, 0, 174, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 442, 0, 0, 4095, 0, 0, 20, 0, 0, 0, 4062, 0, 0, 2, 0, 0, 0, 0, 3, 4, 0, 2, 0, 0, 4096, 0, 3690] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 3652, 33, 0, 0, 0, 0, 61, 0, 4, 0, 0, 0, 0, 4096, 0, 0, 3997, 11, 0, 0, 0, 0, 0, 0, 4096, 102, 0, 367, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3432, 1, 0, 0, 0, 0, 0, 0, 0, 0, 445, 0, 0, 4096, 0, 0, 3937, 32, 0, 0, 0, 414, 0, 0, 4096, 0, 0, 55, 1, 0, 0, 3852, 0, 0, 0, 0, 0, 0, 0, 120, 7, 5, 0, 2, 0, 4095, 0, 4082] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3, 10, 8, 0, 0, 0, 0, 4091, 1, 0, 0, 4049, 0, 0, 0, 0, 0, 38, 5, 1, 0, 120, 0, 0, 4092, 1, 0, 3969, 0, 0, 1, 1, 0, 1, 0, 4094, 0, 0, 1, 0, 0, 0, 4071, 18, 0, 0, 0, 0, 0, 0, 5, 5, 1, 0, 0, 4, 4092, 0, 4086] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 3992, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 106, 0, 0, 4090, 0, 0, 14, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 4052, 0, 4090] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3994, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 75, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 4096, 14, 0, 0, 0, 0, 0, 0, 4096, 53, 0, 0, 0, 0, 0, 4089, 6, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 4078, 0, 4030] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 2572, 0, 0, 0, 0, 0, 2284, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3814, 176, 0, 0, 0, 0, 0, 0, 4096, 455, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 29, 0, 4096, 0, 2939] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 497, 0, 0, 0, 0, 0, 1, 0, 0, 0, 56, 0, 0, 4096, 0, 0, 4042, 0, 0, 0, 0, 3602, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4090] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 3897, 0, 0, 0, 0, 3, 0, 0, 341, 100, 75, 0, 0, 0, 0, 0, 0, 0, 174, 0, 0, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3576, 0, 0, 0, 0, 442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 3970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3, 3906, 179, 2, 0, 0, 0, 0, 0, 4, 0, 2, 0, 0, 190, 0, 3690, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4103, 4416, 4170, 4018, 3990, 4092, 3886] avg=4096.0000 max=4416 max_rank=2 min=3886 min_rank=7 imbalance_ratio=1.078125 max_load_reduction=28352 max_load_reduction_pct=0.865234 moved_to_echo=15986 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3, 10, 8, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 0, 0, 0, 0, 38, 5, 331, 100, 75, 0, 0, 0, 0, 0, 1, 0, 120, 0, 0, 3992, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3464, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3979, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 5, 3902, 174, 2, 0, 0, 0, 0, 0, 5, 1, 0, 0, 4, 190, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4109, 4598, 4114, 3467, 3980, 4101, 4286] avg=4096.0000 max=4598 max_rank=2 min=3467 min_rank=4 imbalance_ratio=1.122559 max_load_reduction=28170 max_load_reduction_pct=0.859680 moved_to_echo=15834 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 3652, 33, 0, 0, 0, 0, 61, 0, 333, 100, 76, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3489, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 367, 0, 0, 0, 3993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3906, 175, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 190, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4088, 4255, 4000, 3500, 4462, 4084, 4286] avg=4096.0000 max=4462 max_rank=5 min=3500 min_rank=4 imbalance_ratio=1.089355 max_load_reduction=28306 max_load_reduction_pct=0.863831 moved_to_echo=16332 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 3, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 3992, 5, 0, 0, 0, 0, 0, 0, 341, 100, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3576, 0, 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 1, 4001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3864, 179, 3, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 188, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4114, 4514, 3996, 3682, 4016, 4046, 4309] avg=4096.0000 max=4514 max_rank=2 min=3682 min_rank=4 imbalance_ratio=1.102051 max_load_reduction=28254 max_load_reduction_pct=0.862244 moved_to_echo=16053 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3994, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 75, 0, 341, 100, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3576, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 3995, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 3889, 179, 3, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 189, 0, 4030, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4012, 4685, 4000, 3590, 4048, 4077, 4262] avg=4096.0000 max=4685 max_rank=2 min=3590 min_rank=4 imbalance_ratio=1.143799 max_load_reduction=28083 max_load_reduction_pct=0.857025 moved_to_echo=15977 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 3432, 1, 0, 0, 0, 0, 0, 0, 328, 100, 71, 0, 0, 0, 0, 0, 0, 0, 445, 0, 0, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3437, 32, 0, 0, 0, 414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 1, 0, 0, 3765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 3905, 172, 2, 0, 0, 0, 0, 0, 7, 5, 0, 2, 0, 190, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4112, 3932, 4441, 3883, 3821, 4199, 4286] avg=4096.0000 max=4441 max_rank=3 min=3821 min_rank=5 imbalance_ratio=1.084229 max_load_reduction=28327 max_load_reduction_pct=0.864471 moved_to_echo=16189 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 2, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 2572, 0, 0, 0, 0, 0, 2284, 0, 321, 101, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3322, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 455, 0, 0, 0, 0, 0, 3995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3911, 171, 2, 0, 0, 0, 0, 0, 0, 19, 0, 29, 0, 185, 0, 2939, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4116, 5357, 3995, 3498, 4450, 4084, 3172] avg=4096.0000 max=5357 max_rank=2 min=3172 min_rank=7 imbalance_ratio=1.307861 max_load_reduction=27411 max_load_reduction_pct=0.836517 moved_to_echo=15204 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 497, 0, 0, 0, 0, 0, 1, 0, 337, 100, 76, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3528, 0, 0, 0, 0, 3602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3906, 177, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 190, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4114, 1011, 4052, 7130, 4001, 4085, 4281] avg=4096.0000 max=7130 max_rank=4 min=1011 min_rank=2 imbalance_ratio=1.740723 max_load_reduction=25638 max_load_reduction_pct=0.782410 moved_to_echo=19497 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[28032, 256, 0, 128, 128, 4608, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[26240, 128, 0, 0, 0, 128, 2560, 128, 2688, 896, 640, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 0, 0, 0, 0, 128, 31232, 1408, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 128, 1536, 0, 31104, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 32640, 128, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 640, 0, 512, 128, 0, 128, 31744, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 0, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 896, 0, 128, 32000, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3, 0, 0, 4096, 4096, 0, 0, 0, 4077, 0, 0, 4094, 9, 0, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4095, 0, 2, 0, 0, 0, 0, 0, 0, 12, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4090, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 62, 0, 4065, 0, 0, 4096, 0, 0, 4096, 0, 0, 17, 0, 0, 0, 3959, 0, 0, 0, 0, 4053, 0, 31, 0, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 4089, 0, 0, 4096, 4, 0, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4042, 3, 0, 0, 0, 9, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 4093, 4094, 0, 0, 0, 4055, 0, 0, 4094, 0, 0, 4091, 0, 0, 0, 13, 0, 1, 3891, 1, 0, 0, 1, 4063, 0, 33, 0, 0, 0, 162, 0, 1, 85, 1, 1, 1, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 2, 0, 0, 4073, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 11, 0, 4044, 0, 0, 4096, 0, 0, 4072, 0, 0, 0, 0, 1, 0, 3842, 0, 0, 0, 0, 4092, 0, 63, 0, 0, 0, 236, 0, 0, 1, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 4045, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4089, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3762, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 5, 0, 0, 334, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 4096, 0, 0, 4096, 45, 0, 4096, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 2472, 0, 110, 0, 0, 0, 0, 0, 0, 1487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 227, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3985, 11, 0, 0, 0, 0, 0, 0, 12, 0, 0, 1, 0, 0, 0, 0, 3854, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 11, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4088, 4099, 4322, 3998, 3967, 4088, 4107] avg=4096.0000 max=4322 max_rank=3 min=3967 min_rank=5 imbalance_ratio=1.055176 max_load_reduction=28446 max_load_reduction_pct=0.868103 moved_to_echo=16160 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 62, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 219, 0, 0, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 136, 0, 0, 3985, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3729, 100, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 4085, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 0, 11, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4138, 4107, 4272, 4163, 3831, 4112, 4050] avg=4096.0000 max=4272 max_rank=3 min=3831 min_rank=5 imbalance_ratio=1.042969 max_load_reduction=28496 max_load_reduction_pct=0.869629 moved_to_echo=16156 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 215, 1, 0, 0, 1, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 162, 0, 1, 3982, 12, 0, 0, 0, 0, 0, 0, 85, 1, 1, 1, 4, 1, 0, 0, 3665, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4082, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 4073, 1, 0, 11, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4067, 4098, 4281, 4190, 3857, 4086, 4094] avg=4096.0000 max=4281 max_rank=3 min=3857 min_rank=5 imbalance_ratio=1.045166 max_load_reduction=28487 max_load_reduction_pct=0.869354 moved_to_echo=16227 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 1, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 227, 0, 0, 0, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 9, 0, 0, 0, 3985, 11, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 3858, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 11, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4101, 4087, 4269, 4008, 4009, 4086, 4113] avg=4096.0000 max=4269 max_rank=3 min=4008 min_rank=4 imbalance_ratio=1.042236 max_load_reduction=28499 max_load_reduction_pct=0.869720 moved_to_echo=16216 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 11, 0, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 213, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 236, 0, 0, 3985, 11, 0, 0, 0, 0, 0, 0, 1, 0, 0, 18, 0, 0, 0, 0, 3618, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 0, 11, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4066, 4066, 4306, 4295, 3737, 4140, 4062] avg=4096.0000 max=4306 max_rank=3 min=3737 min_rank=5 imbalance_ratio=1.051270 max_load_reduction=28462 max_load_reduction_pct=0.868591 moved_to_echo=16234 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 209, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5, 0, 0, 3985, 12, 0, 0, 0, 0, 0, 0, 334, 0, 0, 5, 0, 0, 0, 0, 3543, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 10, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4100, 4090, 4305, 4003, 3982, 4084, 4108] avg=4096.0000 max=4305 max_rank=3 min=3982 min_rank=5 imbalance_ratio=1.051025 max_load_reduction=28463 max_load_reduction_pct=0.868622 moved_to_echo=16177 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 227, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3985, 12, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3857, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 11, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4107, 4090, 4323, 3998, 3958, 4084, 4112] avg=4096.0000 max=4323 max_rank=3 min=3958 min_rank=5 imbalance_ratio=1.055420 max_load_reduction=28445 max_load_reduction_pct=0.868073 moved_to_echo=16152 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 218, 0, 0, 0, 0, 2472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 0, 0, 0, 0, 0, 3991, 12, 0, 0, 0, 0, 0, 0, 1487, 0, 0, 0, 0, 0, 0, 0, 3839, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 16, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4101, 4131, 2690, 4113, 5427, 4088, 4122] avg=4096.0000 max=5427 max_rank=5 min=2690 min_rank=3 imbalance_ratio=1.324951 max_load_reduction=27341 max_load_reduction_pct=0.834381 moved_to_echo=17750 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 256, 0, 0, 128, 640, 0, 128, 32000, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 1792, 128, 0, 0, 128, 31104, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[2048, 128, 128, 128, 128, 128, 128, 0, 30080, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 0, 0, 32640, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 0, 32768, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 0, 0, 128, 0, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33792 slack_slots=31744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 93, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 0, 0, 4049, 0, 4096, 0, 0, 1, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[10, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4082, 4045, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 44, 0, 0, 1, 0, 0, 0, 297, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 3782, 0, 0, 4096, 15, 4096, 0, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 4062, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4063, 0, 0, 0, 0, 0, 1, 1, 3931, 0, 0, 0, 0, 0, 0, 13, 4096, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 181, 0, 0, 290, 67, 0, 1, 0, 0, 0, 0, 0, 169, 0, 10, 3571, 0, 0, 4075, 0, 4082, 0, 27, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[2, 4052, 12, 0, 0, 0, 1, 3, 0, 0, 0, 4030, 4093, 0, 0, 0, 0, 0, 0, 8, 4087, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 52, 0, 2, 0, 0, 0, 6, 1430, 0, 0, 27, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 2639, 0, 0, 4082, 2, 4093, 0, 66, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 189, 0, 4012, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 1885, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 7, 23, 0, 0, 2078, 0, 0, 4060, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 4078, 0, 0, 0, 0, 0, 0, 9, 0, 0, 3933, 4091, 0, 8, 0, 0, 0, 4069, 0, 322, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 188, 0, 0, 0, 0, 0, 1005, 0, 0, 0, 0, 0, 14, 0, 0, 2784, 0, 0, 4073, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1, 4079, 0, 0, 0, 1, 0, 0, 31, 0, 0, 4087, 4095, 0, 10, 0, 0, 0, 2, 0, 3998, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 0, 0, 2, 0, 3, 0, 0, 0, 0, 0, 31, 0, 0, 4020, 0, 0, 4091, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4072, 0, 2, 0, 0, 0, 0, 30, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1541, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 1, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 93, 3068, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 3914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 0, 1003, 8, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 5, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4120, 4073, 4083, 3339, 3938, 5050, 4056] avg=4096.0000 max=5050 max_rank=6 min=3339 min_rank=4 imbalance_ratio=1.232910 max_load_reduction=27718 max_load_reduction_pct=0.845886 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[10, 4096, 0, 1, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 1, 0, 0, 0, 297, 3030, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 3914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3782, 0, 990, 8, 0, 0, 0, 0, 0, 0, 0, 4096, 15, 5, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4107, 4073, 4099, 3549, 3924, 4780, 4117] avg=4096.0000 max=4780 max_rank=6 min=3549 min_rank=4 imbalance_ratio=1.166992 max_load_reduction=27988 max_load_reduction_pct=0.854126 moved_to_echo=16370 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4062, 1, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 3909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 181, 3043, 176, 0, 0, 0, 0, 0, 0, 0, 0, 290, 67, 0, 1, 0, 0, 3901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 0, 10, 3571, 0, 995, 8, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 5, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4121, 3911, 4112, 3431, 4259, 4753, 4107] avg=4096.0000 max=4753 max_rank=6 min=3431 min_rank=4 imbalance_ratio=1.160400 max_load_reduction=28015 max_load_reduction_pct=0.854950 moved_to_echo=16550 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2, 4052, 12, 0, 0, 0, 1, 3, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4030, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 52, 0, 2, 0, 0, 0, 6, 1430, 3066, 177, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 1, 1, 1, 0, 3916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2639, 0, 1002, 8, 0, 0, 0, 0, 0, 0, 0, 4082, 2, 0, 0, 66, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4055, 4072, 4078, 4733, 3946, 3649, 4153] avg=4096.0000 max=4733 max_rank=4 min=3649 min_rank=6 imbalance_ratio=1.155518 max_load_reduction=28035 max_load_reduction_pct=0.855560 moved_to_echo=16481 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4091, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 0, 3990, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 1885, 3068, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 3914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 23, 0, 0, 2078, 0, 1003, 8, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4121, 4181, 4099, 5133, 3948, 3119, 4065] avg=4096.0000 max=5133 max_rank=4 min=3119 min_rank=6 imbalance_ratio=1.253174 max_load_reduction=27635 max_load_reduction_pct=0.843353 moved_to_echo=16265 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 4079, 0, 0, 0, 1, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 4087, 25, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 3067, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 0, 0, 3914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 4020, 0, 1003, 8, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4153, 3978, 4097, 3371, 3919, 5062, 4096] avg=4096.0000 max=5062 max_rank=6 min=3371 min_rank=4 imbalance_ratio=1.235840 max_load_reduction=27706 max_load_reduction_pct=0.845520 moved_to_echo=16448 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 25, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2547, 3050, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1541, 0, 997, 8, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4123, 4103, 4099, 5774, 3914, 2546, 4101] avg=4096.0000 max=5774 max_rank=4 min=2546 min_rank=6 imbalance_ratio=1.409668 max_load_reduction=26994 max_load_reduction_pct=0.823792 moved_to_echo=16335 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4078, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 3933, 18, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 188, 3067, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 0, 0, 3915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 2784, 0, 1006, 5, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 3968, 4377, 4101, 3438, 4920, 3809, 4073] avg=4096.0000 max=4920 max_rank=5 min=3438 min_rank=4 imbalance_ratio=1.201172 max_load_reduction=27848 max_load_reduction_pct=0.849854 moved_to_echo=16240 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 128, 256, 0, 128, 24576, 0, 8064, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 0, 0, 128, 6784, 24576, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 384, 128, 128, 1024, 128, 0, 31360, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 4352, 128, 28544, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 32768, 128, 128, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 32768, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 0, 32640, 256, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 32640, 128, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True + [2026-09-21 08:24:25] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 271.1 | throughput per GPU (TFLOP/s/GPU): 118.9 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.672910E+00 | loss scale: 1.0 | grad norm: 8.531 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[427, 304, 465, 3, 0, 727, 1010, 162, 1, 1440, 9, 5, 3, 101, 66, 154, 1, 970, 546, 1819, 27, 459, 239, 13, 115, 5, 25, 412, 923, 739, 717, 1546, 206, 173, 402, 840, 66, 1876, 227, 1045, 1007, 694, 1529, 21, 1825, 1140, 12, 165, 922, 533, 783, 9, 685, 1029, 1673, 53, 1629, 15, 335, 3, 14, 9, 391, 24] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[377, 381, 315, 3, 0, 504, 1253, 166, 3, 1052, 31, 2, 12, 36, 189, 158, 6, 1261, 719, 1529, 24, 548, 203, 5, 147, 4, 70, 248, 1168, 776, 652, 1075, 241, 191, 536, 645, 74, 1860, 282, 819, 1006, 935, 1540, 27, 1690, 1190, 23, 180, 988, 684, 702, 50, 768, 1517, 1600, 103, 1304, 17, 248, 1, 17, 94, 488, 31] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[297, 432, 337, 3, 2, 530, 1473, 216, 2, 1316, 31, 0, 59, 41, 125, 160, 53, 1030, 759, 2352, 28, 372, 237, 27, 214, 27, 9, 264, 1105, 1396, 524, 1411, 354, 240, 444, 480, 155, 1237, 214, 770, 973, 1270, 1775, 42, 1582, 563, 26, 266, 689, 636, 747, 47, 447, 1456, 1300, 52, 1081, 16, 440, 16, 21, 118, 451, 28] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[359, 610, 275, 2, 0, 468, 1575, 72, 8, 1044, 64, 4, 17, 9, 185, 322, 15, 917, 534, 1666, 35, 63, 313, 20, 249, 11, 21, 379, 1372, 388, 782, 1433, 302, 215, 556, 421, 176, 1588, 374, 565, 960, 868, 2004, 11, 1438, 1095, 8, 101, 640, 1233, 624, 61, 973, 863, 1456, 15, 1537, 9, 363, 38, 9, 130, 912, 11] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[316, 321, 578, 19, 0, 547, 1409, 176, 15, 1321, 40, 0, 20, 13, 293, 263, 7, 823, 605, 1697, 22, 74, 178, 2, 77, 9, 62, 323, 1200, 957, 556, 1497, 583, 285, 523, 636, 71, 2266, 178, 689, 1001, 800, 2094, 30, 1658, 809, 27, 118, 924, 566, 697, 46, 441, 1018, 1346, 76, 1701, 36, 355, 15, 11, 22, 311, 15] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[267, 527, 487, 1, 2, 841, 984, 247, 9, 1465, 17, 0, 26, 155, 460, 327, 4, 1016, 388, 2175, 13, 280, 193, 15, 198, 19, 65, 308, 891, 894, 722, 1373, 335, 350, 539, 610, 85, 1932, 245, 1008, 685, 933, 1333, 31, 1874, 793, 20, 159, 753, 522, 674, 28, 723, 1176, 1321, 64, 1375, 14, 250, 2, 10, 50, 470, 35] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[212, 524, 418, 4, 1, 710, 1208, 67, 2, 1664, 38, 1, 6, 98, 305, 257, 11, 733, 396, 2031, 16, 157, 295, 24, 172, 26, 43, 568, 962, 433, 900, 1670, 318, 325, 570, 582, 228, 1333, 466, 939, 827, 759, 1707, 15, 1845, 905, 18, 54, 650, 704, 586, 38, 730, 962, 1688, 44, 1743, 14, 215, 2, 8, 31, 480, 30] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[294, 524, 343, 1, 0, 589, 1573, 111, 2, 1370, 28, 0, 2, 23, 165, 259, 11, 909, 395, 2231, 18, 194, 256, 15, 236, 39, 72, 255, 1386, 861, 649, 1415, 377, 174, 436, 775, 139, 1708, 242, 587, 957, 1023, 1549, 41, 1362, 833, 8, 246, 773, 863, 495, 33, 662, 1243, 1584, 21, 1639, 9, 246, 13, 8, 37, 436, 23] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[377, 381, 315, 3, 0, 504, 1253, 166, 437, 308, 0, 0, 0, 0, 0, 0, 3, 1052, 31, 2, 12, 36, 189, 158, 1540, 527, 0, 0, 0, 0, 0, 0, 6, 1261, 719, 1529, 24, 548, 203, 5, 94, 8, 0, 0, 0, 0, 0, 0, 147, 4, 70, 248, 1168, 776, 652, 674, 0, 0, 0, 0, 0, 0, 0, 0, 241, 191, 536, 645, 74, 1276, 282, 819, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 935, 0, 27, 1159, 1190, 23, 180, 0, 0, 0, 0, 0, 0, 0, 0, 988, 684, 702, 50, 768, 1509, 0, 103, 0, 0, 0, 0, 0, 0, 0, 0, 1304, 17, 248, 1, 17, 94, 488, 31, 1073, 584, 93, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3744, 3550, 4397, 3739, 4064, 4520, 4804, 3950] avg=4096.0000 max=4804 max_rank=6 min=3550 min_rank=1 imbalance_ratio=1.172852 max_load_reduction=27964 max_load_reduction_pct=0.853394 moved_to_echo=17338 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[427, 304, 465, 3, 0, 727, 1010, 162, 472, 443, 0, 0, 0, 0, 0, 0, 1, 1440, 9, 5, 3, 101, 66, 154, 1529, 551, 0, 0, 0, 0, 0, 0, 1, 970, 546, 1819, 27, 459, 239, 13, 101, 5, 0, 0, 0, 0, 0, 0, 115, 5, 25, 412, 923, 739, 717, 968, 0, 0, 0, 0, 0, 0, 0, 0, 206, 173, 402, 840, 66, 1287, 227, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 694, 0, 21, 1252, 1140, 12, 165, 0, 0, 0, 0, 0, 0, 0, 0, 922, 533, 783, 9, 685, 1024, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 1629, 15, 335, 3, 14, 9, 391, 24, 1122, 589, 135, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4013, 3859, 4180, 3904, 4246, 4291, 4009, 4266] avg=4096.0000 max=4291 max_rank=5 min=3859 min_rank=1 imbalance_ratio=1.047607 max_load_reduction=28477 max_load_reduction_pct=0.869049 moved_to_echo=16812 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[297, 432, 337, 3, 2, 530, 1473, 216, 409, 405, 0, 0, 0, 0, 0, 0, 2, 1316, 31, 0, 59, 41, 125, 160, 1775, 428, 0, 0, 0, 0, 0, 0, 53, 1030, 759, 2352, 28, 372, 237, 27, 88, 8, 0, 0, 0, 0, 0, 0, 214, 27, 9, 264, 1105, 1396, 524, 883, 0, 0, 0, 0, 0, 0, 0, 0, 354, 240, 444, 480, 155, 848, 214, 770, 0, 0, 0, 0, 0, 0, 0, 0, 973, 1270, 0, 42, 1085, 563, 26, 266, 0, 0, 0, 0, 0, 0, 0, 0, 689, 636, 747, 47, 447, 1448, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 1081, 16, 440, 16, 21, 118, 451, 28, 872, 389, 123, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 3937, 4954, 4422, 3505, 4225, 4066, 3555] avg=4096.0000 max=4954 max_rank=2 min=3505 min_rank=4 imbalance_ratio=1.209473 max_load_reduction=27814 max_load_reduction_pct=0.848816 moved_to_echo=15351 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[359, 610, 275, 2, 0, 468, 1575, 72, 372, 411, 0, 0, 0, 0, 0, 0, 8, 1044, 64, 4, 17, 9, 185, 322, 2004, 479, 0, 0, 0, 0, 0, 0, 15, 917, 534, 1666, 35, 63, 313, 20, 80, 4, 0, 0, 0, 0, 0, 0, 249, 11, 21, 379, 1372, 388, 782, 897, 0, 0, 0, 0, 0, 0, 0, 0, 302, 215, 556, 421, 176, 1089, 374, 565, 0, 0, 0, 0, 0, 0, 0, 0, 960, 868, 0, 11, 986, 1095, 8, 101, 0, 0, 0, 0, 0, 0, 0, 0, 640, 1233, 624, 61, 973, 859, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 1537, 9, 363, 38, 9, 130, 912, 11, 977, 499, 125, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4144, 4136, 3647, 4099, 3698, 4029, 4405, 4610] avg=4096.0000 max=4610 max_rank=7 min=3647 min_rank=2 imbalance_ratio=1.125488 max_load_reduction=28158 max_load_reduction_pct=0.859314 moved_to_echo=16742 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[316, 321, 578, 19, 0, 547, 1409, 176, 429, 429, 0, 0, 0, 0, 0, 0, 15, 1321, 40, 0, 20, 13, 293, 263, 2094, 444, 0, 0, 0, 0, 0, 0, 7, 823, 605, 1697, 22, 74, 178, 2, 92, 5, 0, 0, 0, 0, 0, 0, 77, 9, 62, 323, 1200, 957, 556, 938, 0, 0, 0, 0, 0, 0, 0, 0, 583, 285, 523, 636, 71, 1554, 178, 689, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 800, 0, 30, 1137, 809, 27, 118, 0, 0, 0, 0, 0, 0, 0, 0, 924, 566, 697, 46, 441, 1013, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 1701, 36, 355, 15, 11, 22, 311, 15, 902, 712, 130, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4224, 4503, 3505, 4122, 4519, 3922, 3763, 4210] avg=4096.0000 max=4519 max_rank=4 min=3505 min_rank=2 imbalance_ratio=1.103271 max_load_reduction=28249 max_load_reduction_pct=0.862091 moved_to_echo=16414 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[267, 527, 487, 1, 2, 841, 984, 247, 485, 394, 0, 0, 0, 0, 0, 0, 9, 1465, 17, 0, 26, 155, 460, 327, 1333, 436, 0, 0, 0, 0, 0, 0, 4, 1016, 388, 2175, 13, 280, 193, 15, 104, 6, 0, 0, 0, 0, 0, 0, 198, 19, 65, 308, 891, 894, 722, 860, 0, 0, 0, 0, 0, 0, 0, 0, 335, 350, 539, 610, 85, 1325, 245, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 685, 933, 0, 31, 1285, 793, 20, 159, 0, 0, 0, 0, 0, 0, 0, 0, 753, 522, 674, 28, 723, 1170, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 1375, 14, 250, 2, 10, 50, 470, 35, 885, 607, 119, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4235, 4228, 4194, 3957, 4497, 3906, 3934, 3817] avg=4096.0000 max=4497 max_rank=4 min=3817 min_rank=7 imbalance_ratio=1.097900 max_load_reduction=28271 max_load_reduction_pct=0.862762 moved_to_echo=16154 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[294, 524, 343, 1, 0, 589, 1573, 111, 352, 406, 0, 0, 0, 0, 0, 0, 2, 1370, 28, 0, 2, 23, 165, 259, 1549, 522, 0, 0, 0, 0, 0, 0, 11, 909, 395, 2231, 18, 194, 256, 15, 76, 7, 0, 0, 0, 0, 0, 0, 236, 39, 72, 255, 1386, 861, 649, 886, 0, 0, 0, 0, 0, 0, 0, 0, 377, 174, 436, 775, 139, 1171, 242, 587, 0, 0, 0, 0, 0, 0, 0, 0, 957, 1023, 0, 41, 934, 833, 8, 246, 0, 0, 0, 0, 0, 0, 0, 0, 773, 863, 495, 33, 662, 1236, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 1639, 9, 246, 13, 8, 37, 436, 23, 1062, 537, 123, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4193, 3920, 4112, 4384, 3901, 4042, 4083, 4133] avg=4096.0000 max=4384 max_rank=3 min=3901 min_rank=4 imbalance_ratio=1.070312 max_load_reduction=28384 max_load_reduction_pct=0.866211 moved_to_echo=16159 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[212, 524, 418, 4, 1, 710, 1208, 67, 484, 483, 0, 0, 0, 0, 0, 0, 2, 1664, 38, 1, 6, 98, 305, 257, 1707, 557, 0, 0, 0, 0, 0, 0, 11, 733, 396, 2031, 16, 157, 295, 24, 106, 10, 0, 0, 0, 0, 0, 0, 172, 26, 43, 568, 962, 433, 900, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 318, 325, 570, 582, 228, 910, 466, 939, 0, 0, 0, 0, 0, 0, 0, 0, 827, 759, 0, 15, 1255, 905, 18, 54, 0, 0, 0, 0, 0, 0, 0, 0, 650, 704, 586, 38, 730, 952, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 1743, 14, 215, 2, 8, 31, 480, 30, 1131, 423, 150, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4635, 3779, 4141, 4338, 3833, 3704, 4227] avg=4096.0000 max=4635 max_rank=1 min=3704 min_rank=6 imbalance_ratio=1.131592 max_load_reduction=28133 max_load_reduction_pct=0.858551 moved_to_echo=16102 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[6400, 5760, 5376, 384, 5504, 9216, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[2560, 3712, 3328, 128, 128, 4992, 10496, 1280, 3456, 3328, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[2816, 2048, 4096, 4992, 1024, 9472, 2304, 6528, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[7424, 7296, 0, 256, 9216, 7424, 256, 1408, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 7680, 4352, 15616, 256, 2176, 1920, 128, 768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[12032, 256, 2560, 128, 128, 512, 3968, 256, 8064, 4352, 1024, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1408, 256, 384, 2816, 9088, 6528, 5504, 7168, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 10752, 384, 128, 256, 512, 1792, 1920, 13568, 3968, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 1179, 0, 0, 4, 0, 2, 13, 51, 189, 120, 6, 0, 3350, 0, 64, 0, 4018, 5, 0, 69, 681, 15, 1, 69, 0, 0, 15, 0, 3473, 1423, 0, 0, 0, 0, 6, 0, 3494, 2, 4090, 10, 3679, 1, 6, 0, 0, 6, 762, 422, 0, 0, 0, 0, 4, 0, 3358, 14, 1606, 544, 15, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1, 8, 391, 0, 0, 2, 0, 0, 9, 0, 1297, 1160, 160, 0, 223, 0, 0, 0, 4075, 0, 30, 36, 1438, 13, 0, 9, 0, 3, 1, 0, 4071, 919, 4, 0, 0, 0, 0, 22, 2579, 3, 4085, 0, 4010, 0, 6, 0, 0, 0, 526, 6, 176, 1, 0, 0, 0, 0, 4005, 2, 1878, 1136, 474, 0, 8, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[18, 1, 1926, 0, 0, 3, 0, 2, 3, 1, 3288, 1603, 8, 1, 1006, 14, 2, 0, 4090, 8, 0, 0, 667, 10, 0, 156, 0, 3, 22, 130, 2855, 1085, 2, 15, 0, 0, 0, 0, 340, 79, 3967, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 117, 3989, 1, 1283, 607, 1214, 0, 158, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[51, 1, 769, 0, 0, 83, 0, 0, 157, 0, 1579, 466, 63, 0, 22, 0, 3, 0, 4053, 0, 741, 17, 3075, 1, 0, 393, 0, 10, 573, 19, 2201, 731, 5, 0, 0, 0, 7, 0, 4001, 37, 4095, 186, 2229, 0, 0, 0, 116, 3, 83, 1, 16, 1, 0, 0, 1, 1, 1281, 0, 3618, 841, 97, 0, 1141, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1, 4, 129, 1, 5, 4, 0, 0, 133, 0, 2810, 93, 0, 0, 1510, 0, 1, 0, 3926, 3, 77, 2, 1535, 9, 0, 1163, 0, 0, 237, 0, 2405, 1761, 9, 1, 1, 1, 37, 0, 3308, 0, 3963, 1, 3023, 1, 0, 0, 0, 0, 292, 5, 3, 18, 3, 2, 0, 0, 2832, 1, 774, 287, 2094, 4, 299, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[104, 0, 1094, 2, 16, 0, 0, 0, 3, 0, 159, 992, 0, 0, 3310, 0, 1, 4, 3869, 0, 4, 4, 109, 0, 0, 431, 0, 1, 0, 0, 3151, 1570, 0, 0, 13, 0, 6, 139, 1116, 5, 3860, 0, 3975, 0, 5, 0, 90, 0, 1047, 0, 0, 0, 64, 0, 0, 0, 3538, 1, 1173, 2511, 394, 3, 4, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1, 1, 447, 0, 4, 131, 0, 15, 1, 170, 3032, 288, 3, 0, 2817, 0, 0, 0, 3968, 0, 0, 7, 2467, 61, 0, 29, 0, 1, 13, 0, 3985, 651, 0, 0, 11, 0, 61, 0, 1899, 5, 3919, 0, 3869, 0, 1, 0, 2, 2, 6, 0, 3, 5, 0, 0, 0, 0, 1197, 16, 910, 1682, 1065, 0, 23, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[98, 0, 1655, 1, 0, 0, 0, 0, 0, 0, 1378, 148, 2, 0, 1045, 6, 0, 0, 4094, 0, 0, 14, 1794, 0, 0, 673, 0, 0, 31, 0, 2427, 1585, 0, 1, 0, 0, 0, 0, 2536, 2, 4096, 0, 4081, 0, 0, 0, 8, 0, 1075, 1, 3, 0, 2, 0, 0, 17, 2458, 0, 1843, 1388, 303, 0, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[98, 0, 1655, 1, 0, 0, 0, 0, 2094, 599, 77, 0, 0, 0, 0, 0, 0, 0, 1378, 148, 2, 0, 1045, 6, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2527, 0, 0, 14, 1794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 673, 0, 0, 31, 0, 1884, 1585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2536, 2, 968, 536, 0, 0, 0, 0, 0, 0, 431, 0, 4081, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1075, 1, 3, 0, 2, 0, 0, 17, 3588, 0, 0, 0, 0, 0, 0, 0, 364, 0, 1843, 1388, 303, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4524, 2586, 4335, 4173, 4043, 4520, 4686, 3901] avg=4096.0000 max=4686 max_rank=6 min=2586 min_rank=1 imbalance_ratio=1.144043 max_load_reduction=28082 max_load_reduction_pct=0.856995 moved_to_echo=17150 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[18, 1, 1926, 0, 0, 3, 0, 2, 3399, 598, 74, 0, 0, 0, 0, 0, 3, 1, 3288, 1603, 8, 1, 1006, 14, 9, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2525, 8, 0, 0, 667, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 3, 22, 130, 2216, 1085, 0, 0, 0, 0, 0, 0, 0, 0, 2, 15, 0, 0, 0, 0, 340, 79, 967, 630, 0, 0, 0, 0, 0, 0, 418, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 117, 3475, 0, 0, 0, 0, 0, 0, 0, 590, 1, 1283, 607, 1214, 0, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6021, 5933, 3212, 3612, 2033, 4506, 3598, 3853] avg=4096.0000 max=6021 max_rank=0 min=2033 min_rank=4 imbalance_ratio=1.469971 max_load_reduction=26747 max_load_reduction_pct=0.816254 moved_to_echo=13990 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[104, 0, 1094, 2, 16, 0, 0, 0, 3014, 566, 72, 0, 0, 0, 0, 0, 3, 0, 159, 992, 0, 0, 3310, 0, 10, 0, 0, 0, 0, 0, 0, 0, 1, 4, 2388, 0, 4, 4, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 431, 0, 1, 0, 0, 2445, 1570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 6, 139, 1116, 5, 915, 696, 0, 0, 0, 0, 0, 0, 407, 0, 3975, 0, 5, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 0, 0, 0, 64, 0, 0, 0, 3381, 0, 0, 0, 0, 0, 0, 0, 524, 1, 1173, 2511, 394, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4868, 4474, 2510, 4447, 2890, 4477, 4492, 4610] avg=4096.0000 max=4868 max_rank=0 min=2510 min_rank=2 imbalance_ratio=1.188477 max_load_reduction=27900 max_load_reduction_pct=0.851440 moved_to_echo=16469 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 4, 129, 1, 5, 4, 0, 0, 2417, 579, 78, 0, 0, 0, 0, 0, 133, 0, 2810, 93, 0, 0, 1510, 0, 10, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2414, 3, 77, 2, 1535, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1163, 0, 0, 237, 0, 1860, 1761, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 1, 1, 37, 0, 3308, 0, 933, 535, 0, 0, 0, 0, 0, 0, 411, 1, 3023, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 292, 5, 3, 18, 3, 2, 0, 0, 3474, 0, 0, 0, 0, 0, 0, 0, 415, 1, 774, 287, 2094, 4, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3218, 4556, 4041, 5021, 4825, 3436, 3797, 3874] avg=4096.0000 max=5021 max_rank=3 min=3218 min_rank=0 imbalance_ratio=1.225830 max_load_reduction=27747 max_load_reduction_pct=0.846771 moved_to_echo=15932 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[51, 1, 769, 0, 0, 83, 0, 0, 1091, 593, 77, 0, 0, 0, 0, 0, 157, 0, 1579, 466, 63, 0, 22, 0, 7, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2502, 0, 741, 17, 3075, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 393, 0, 10, 573, 19, 1708, 731, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 7, 0, 4001, 37, 958, 486, 0, 0, 0, 0, 0, 0, 431, 186, 2229, 0, 0, 0, 116, 3, 0, 0, 0, 0, 0, 0, 0, 0, 83, 1, 16, 1, 0, 0, 1, 1, 3587, 0, 0, 0, 0, 0, 0, 0, 190, 0, 3618, 841, 97, 0, 1141, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2665, 2294, 6339, 3434, 5494, 2965, 3690, 5887] avg=4096.0000 max=6339 max_rank=2 min=2294 min_rank=1 imbalance_ratio=1.547607 max_load_reduction=26429 max_load_reduction_pct=0.806549 moved_to_echo=18036 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1179, 0, 0, 4, 0, 2, 2861, 588, 77, 0, 0, 0, 0, 0, 13, 51, 189, 120, 6, 0, 3350, 0, 11, 0, 0, 0, 0, 0, 0, 0, 64, 0, 2480, 5, 0, 69, 681, 15, 0, 0, 0, 0, 0, 0, 0, 0, 1, 69, 0, 0, 15, 0, 2695, 1423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 3494, 2, 950, 767, 0, 0, 0, 0, 0, 0, 430, 10, 3679, 1, 6, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 762, 422, 0, 0, 0, 0, 4, 0, 3583, 0, 0, 0, 0, 0, 0, 0, 497, 14, 1606, 544, 15, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4712, 3740, 3314, 4203, 5219, 4132, 4771, 2677] avg=4096.0000 max=5219 max_rank=4 min=2677 min_rank=7 imbalance_ratio=1.274170 max_load_reduction=27549 max_load_reduction_pct=0.840729 moved_to_echo=16799 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 8, 391, 0, 0, 2, 0, 0, 3412, 596, 77, 0, 0, 0, 0, 0, 9, 0, 1297, 1160, 160, 0, 223, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2516, 0, 30, 36, 1438, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 3, 1, 0, 3159, 919, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 22, 2579, 3, 963, 899, 0, 0, 0, 0, 0, 0, 430, 0, 4010, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 6, 176, 1, 0, 0, 0, 0, 3578, 0, 0, 0, 0, 0, 0, 0, 593, 2, 1878, 1136, 474, 0, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4487, 2862, 4033, 4091, 4470, 4446, 4287, 4092] avg=4096.0000 max=4487 max_rank=0 min=2862 min_rank=1 imbalance_ratio=1.095459 max_load_reduction=28281 max_load_reduction_pct=0.863068 moved_to_echo=17295 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 447, 0, 4, 131, 0, 15, 1019, 581, 74, 0, 0, 0, 0, 0, 1, 170, 3032, 288, 3, 0, 2817, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2449, 0, 0, 7, 2467, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 1, 13, 0, 3093, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 61, 0, 1899, 5, 938, 880, 0, 0, 0, 0, 0, 0, 412, 0, 3869, 0, 1, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 3, 5, 0, 0, 0, 0, 3433, 0, 0, 0, 0, 0, 0, 0, 178, 16, 910, 1682, 1065, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2273, 6323, 4984, 3787, 3794, 4286, 3447, 3874] avg=4096.0000 max=6323 max_rank=1 min=2273 min_rank=0 imbalance_ratio=1.543701 max_load_reduction=26445 max_load_reduction_pct=0.807037 moved_to_echo=15401 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 256, 19328, 256, 7680, 5504, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 19840, 128, 896, 256, 11776, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[384, 256, 13824, 4992, 256, 128, 13312, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[384, 128, 7680, 128, 128, 256, 0, 128, 19328, 4736, 640, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[3456, 128, 13184, 9088, 5760, 128, 1664, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 2944, 0, 128, 896, 256, 19072, 9728, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[3840, 512, 256, 128, 128, 128, 128, 256, 28160, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3456, 256, 29056, 128, 128, 0, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3025, 0, 3895, 0, 4, 3, 1, 4090, 0, 0, 0, 4088, 4091, 178, 4, 6, 0, 0, 0, 0, 1, 0, 0, 59, 278, 38, 0, 0, 3, 0, 1171, 0, 7, 0, 0, 1, 0, 0, 2, 0, 7, 0, 4082, 0, 0, 4043, 0, 2, 0, 132, 259, 0, 3199, 0, 4, 1, 71, 1, 13, 9] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 42, 0, 3909, 1, 4089, 0, 0, 0, 0, 4092, 0, 0, 0, 3932, 4090, 202, 11, 146, 0, 0, 0, 0, 0, 0, 0, 8, 0, 1, 0, 0, 1, 4, 1591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3978, 0, 0, 4081, 0, 4, 33, 15, 17, 0, 2432, 23, 0, 2, 0, 0, 62, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 12, 0, 0, 3656, 0, 4094, 3, 0, 2, 35, 4087, 0, 8, 0, 4096, 4036, 1, 60, 326, 0, 0, 1, 0, 0, 0, 0, 644, 33, 129, 0, 0, 0, 20, 8, 3, 0, 4, 0, 0, 0, 8, 0, 0, 0, 0, 3861, 0, 164, 3642, 10, 123, 0, 210, 80, 1, 2284, 0, 0, 0, 1094, 0, 26, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[2, 0, 60, 0, 3926, 0, 4084, 16, 0, 22, 2, 4088, 0, 4, 0, 3600, 4080, 236, 0, 49, 0, 0, 0, 0, 11, 0, 5, 0, 964, 1036, 0, 0, 15, 260, 586, 5, 0, 0, 54, 2, 0, 0, 1, 17, 2, 136, 3263, 1, 26, 3915, 1, 535, 0, 722, 68, 6, 952, 1, 0, 1, 2, 0, 6, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 124, 0, 2687, 0, 4074, 3, 2, 0, 28, 4055, 0, 0, 0, 2854, 4044, 14, 6, 207, 1, 0, 0, 0, 0, 0, 0, 0, 16, 3, 0, 12, 0, 15, 2332, 0, 24, 0, 0, 1, 0, 0, 0, 11, 0, 0, 3533, 0, 43, 4085, 0, 20, 0, 315, 512, 0, 3274, 4, 0, 3, 178, 24, 264, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1, 1, 3894, 0, 3977, 0, 1, 0, 4, 4057, 0, 0, 3, 3157, 4095, 3, 283, 1, 0, 0, 0, 1, 2, 0, 0, 826, 0, 492, 0, 0, 82, 2, 225, 0, 0, 0, 7, 0, 4, 0, 0, 11, 66, 15, 3879, 0, 0, 4067, 0, 0, 90, 53, 3350, 1, 27, 0, 0, 0, 77, 0, 13, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1, 11, 0, 0, 3826, 0, 4072, 0, 0, 4, 0, 3940, 0, 0, 2, 3837, 3953, 0, 0, 1579, 0, 0, 0, 1, 0, 1, 0, 0, 0, 2250, 0, 0, 0, 2, 21, 0, 1, 0, 0, 0, 0, 0, 5, 0, 1, 79, 4032, 0, 0, 4047, 0, 109, 0, 0, 278, 0, 385, 3, 0, 0, 307, 1, 20, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[8, 0, 116, 0, 3357, 0, 4070, 0, 5, 0, 0, 4090, 0, 1, 0, 3958, 3940, 8, 150, 0, 0, 0, 0, 0, 0, 0, 0, 719, 0, 127, 0, 22, 1, 10, 0, 0, 27, 0, 4, 0, 0, 0, 0, 0, 6, 0, 3752, 0, 10, 3789, 25, 665, 0, 8, 908, 0, 2980, 0, 0, 0, 12, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 12, 0, 0, 3656, 0, 518, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 35, 393, 0, 8, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3658, 1, 60, 326, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 644, 33, 129, 0, 0, 2781, 390, 0, 0, 0, 0, 0, 0, 0, 20, 8, 3, 0, 4, 0, 0, 3304, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 3861, 0, 251, 0, 0, 0, 0, 0, 0, 0, 164, 2757, 10, 123, 0, 210, 80, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2284, 0, 0, 0, 1094, 0, 26, 7, 885, 795, 127, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4189, 4534, 4046, 3977, 3339, 4120, 3345, 5218] avg=4096.0000 max=5218 max_rank=7 min=3339 min_rank=4 imbalance_ratio=1.273926 max_load_reduction=27550 max_load_reduction_pct=0.840759 moved_to_echo=16022 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3025, 0, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 1, 393, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 3708, 178, 4, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 59, 278, 38, 0, 0, 2646, 390, 0, 0, 0, 0, 0, 0, 3, 0, 1171, 0, 7, 0, 0, 1, 3307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7, 0, 4082, 0, 254, 0, 0, 0, 0, 0, 0, 0, 0, 3061, 0, 2, 0, 132, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3199, 0, 4, 1, 71, 1, 13, 9, 982, 756, 129, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3518, 4489, 3896, 3412, 4489, 4345, 3454, 5165] avg=4096.0000 max=5165 max_rank=7 min=3412 min_rank=3 imbalance_ratio=1.260986 max_load_reduction=27603 max_load_reduction_pct=0.842377 moved_to_echo=17453 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 124, 0, 2687, 0, 515, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 28, 390, 0, 0, 0, 2854, 0, 0, 0, 0, 0, 0, 0, 0, 3665, 14, 6, 207, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 3, 0, 12, 2768, 387, 0, 0, 0, 0, 0, 0, 0, 15, 2332, 0, 24, 0, 0, 1, 3278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 3533, 0, 252, 0, 0, 0, 0, 0, 0, 0, 43, 3093, 0, 20, 0, 315, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3274, 4, 0, 3, 178, 24, 264, 0, 992, 791, 127, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3329, 3274, 3893, 3186, 5650, 3796, 3983, 5657] avg=4096.0000 max=5657 max_rank=7 min=3186 min_rank=3 imbalance_ratio=1.381104 max_load_reduction=27111 max_load_reduction_pct=0.827362 moved_to_echo=19086 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 60, 0, 3926, 0, 516, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 2, 393, 0, 4, 0, 3600, 0, 0, 0, 0, 0, 0, 0, 0, 3698, 236, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 5, 0, 964, 1036, 0, 0, 2775, 390, 0, 0, 0, 0, 0, 0, 15, 260, 586, 5, 0, 0, 54, 2, 3305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 17, 2, 136, 3263, 1, 254, 0, 0, 0, 0, 0, 0, 0, 26, 2964, 1, 535, 0, 722, 68, 6, 0, 0, 0, 0, 0, 0, 0, 0, 952, 1, 0, 1, 2, 0, 6, 6, 951, 793, 128, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4520, 4021, 3983, 5181, 4227, 3674, 4322, 2840] avg=4096.0000 max=5181 max_rank=3 min=2840 min_rank=7 imbalance_ratio=1.264893 max_load_reduction=27587 max_load_reduction_pct=0.841888 moved_to_echo=15063 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 42, 0, 3909, 1, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 394, 0, 0, 0, 3932, 0, 0, 0, 0, 0, 0, 0, 0, 3707, 202, 11, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 1, 0, 0, 2778, 390, 0, 0, 0, 0, 0, 0, 1, 4, 1591, 0, 0, 0, 0, 0, 3308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3978, 0, 254, 0, 0, 0, 0, 0, 0, 0, 0, 3090, 0, 4, 33, 15, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2432, 23, 0, 2, 0, 0, 62, 0, 991, 794, 129, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4470, 4326, 4066, 3177, 4904, 4233, 3159, 4433] avg=4096.0000 max=4904 max_rank=4 min=3159 min_rank=6 imbalance_ratio=1.197266 max_load_reduction=27864 max_load_reduction_pct=0.850342 moved_to_echo=16729 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 116, 0, 3357, 0, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 393, 0, 1, 0, 3958, 0, 0, 0, 0, 0, 0, 0, 0, 3571, 8, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 719, 0, 127, 0, 22, 2765, 390, 0, 0, 0, 0, 0, 0, 1, 10, 0, 0, 27, 0, 4, 0, 3307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 3752, 0, 245, 0, 0, 0, 0, 0, 0, 0, 10, 2869, 25, 665, 0, 8, 908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2980, 0, 0, 0, 12, 0, 0, 0, 920, 790, 124, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3996, 4357, 3729, 4023, 3349, 4003, 4485, 4826] avg=4096.0000 max=4826 max_rank=7 min=3349 min_rank=4 imbalance_ratio=1.178223 max_load_reduction=27942 max_load_reduction_pct=0.852722 moved_to_echo=16663 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 1, 3894, 0, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 390, 0, 0, 3, 3157, 0, 0, 0, 0, 0, 0, 0, 0, 3711, 3, 283, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 826, 0, 492, 0, 0, 2702, 387, 0, 0, 0, 0, 0, 0, 82, 2, 225, 0, 0, 0, 7, 0, 3280, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 11, 66, 15, 3879, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 3079, 0, 0, 90, 53, 3350, 1, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 77, 0, 13, 1, 988, 772, 129, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4399, 3555, 3999, 4409, 3596, 4230, 6573, 2007] avg=4096.0000 max=6573 max_rank=6 min=2007 min_rank=7 imbalance_ratio=1.604736 max_load_reduction=26195 max_load_reduction_pct=0.799408 moved_to_echo=16406 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 11, 0, 0, 3826, 0, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 369, 0, 0, 2, 3837, 0, 0, 0, 0, 0, 0, 0, 0, 3576, 0, 0, 1579, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2250, 0, 0, 2771, 381, 0, 0, 0, 0, 0, 0, 0, 2, 21, 0, 1, 0, 0, 0, 3190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 1, 79, 4032, 0, 250, 0, 0, 0, 0, 0, 0, 0, 0, 3060, 0, 109, 0, 0, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 385, 3, 0, 0, 307, 1, 20, 0, 987, 792, 127, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4347, 4212, 5156, 5403, 3214, 4367, 3447, 2622] avg=4096.0000 max=5403 max_rank=3 min=2622 min_rank=7 imbalance_ratio=1.319092 max_load_reduction=27365 max_load_reduction_pct=0.835114 moved_to_echo=13650 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 2304, 1408, 4096, 0, 128, 22016, 3200, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[15616, 128, 128, 128, 1792, 128, 512, 128, 7808, 6400, 1024, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 24064, 128, 1536, 128, 1536, 5504, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 384, 6016, 128, 128, 128, 128, 128, 26368, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 384, 128, 28288, 128, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 3200, 0, 128, 128, 29568, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[29312, 768, 640, 2432, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 256, 30464, 128, 2048, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33792 slack_slots=31744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 45, 0, 0, 4010, 0, 0, 0, 4090, 40, 0, 0, 3380, 0, 1259, 0, 0, 3, 0, 1, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 0, 36, 142, 0, 3, 1, 0, 0, 0, 0, 4067, 3879, 3, 0, 0, 7, 0, 0, 4, 0, 1, 0, 4094, 0, 0, 0, 0, 3637, 0, 19, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[19, 0, 0, 0, 0, 4090, 0, 0, 8, 4071, 274, 1, 0, 24, 0, 3068, 4, 0, 1306, 0, 10, 0, 0, 1, 0, 0, 4, 6, 3, 65, 0, 4065, 0, 14, 0, 0, 0, 1, 0, 0, 2, 0, 4051, 3489, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 4094, 0, 0, 0, 0, 4093, 3, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 12, 0, 0, 4095, 2, 2, 1, 4092, 1726, 0, 0, 1942, 0, 588, 0, 0, 2, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4079, 0, 0, 4, 0, 0, 0, 0, 12, 11, 0, 4092, 4092, 0, 0, 0, 0, 0, 6, 0, 0, 2, 0, 3908, 0, 0, 1, 0, 4091, 4, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4092, 0, 0, 1, 4092, 1089, 0, 0, 1725, 0, 203, 0, 0, 896, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 4045, 5, 0, 280, 0, 0, 0, 0, 0, 0, 1, 3938, 4084, 0, 3, 0, 0, 8, 98, 15, 0, 126, 3, 4095, 0, 3, 0, 0, 3950, 0, 0, 0, 9] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[6, 0, 7, 0, 0, 2694, 0, 0, 0, 4095, 1437, 0, 0, 3523, 0, 182, 0, 0, 453, 8, 0, 0, 3, 0, 0, 0, 11, 0, 1, 0, 0, 4094, 0, 3, 106, 0, 0, 1, 0, 0, 20, 0, 4088, 3807, 1, 0, 0, 96, 4, 4, 7, 0, 10, 0, 4031, 0, 0, 0, 2, 4074, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 3922, 0, 0, 0, 4066, 1214, 0, 0, 2831, 0, 264, 0, 1, 166, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 4062, 0, 0, 0, 0, 0, 20, 0, 0, 4, 0, 3905, 0, 0, 0, 0, 4081, 26, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4075, 0, 0, 0, 3755, 841, 0, 0, 3789, 0, 1, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 13, 0, 4096, 0, 0, 221, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 7, 0, 0, 1, 0, 3589, 0, 0, 0, 0, 4096, 48, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 24, 0, 0, 4095, 0, 0, 0, 4078, 185, 0, 0, 3482, 1, 473, 0, 0, 2, 2, 0, 0, 0, 7, 0, 0, 2, 0, 9, 0, 0, 4078, 0, 0, 236, 0, 0, 0, 0, 0, 1, 7, 4056, 4078, 0, 0, 1, 0, 0, 0, 0, 0, 11, 0, 3830, 0, 0, 7, 0, 4095, 0, 0, 8, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[19, 0, 0, 0, 0, 4090, 0, 0, 196, 0, 0, 0, 0, 0, 0, 0, 8, 0, 274, 1, 0, 24, 0, 3068, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1306, 0, 10, 0, 0, 1, 3639, 81, 0, 0, 0, 0, 0, 0, 0, 0, 4, 6, 3, 65, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 1, 0, 0, 3990, 0, 0, 0, 0, 0, 0, 0, 2, 0, 132, 3489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 4094, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4305, 3375, 5041, 4143, 4005, 3623, 4180, 4096] avg=4096.0000 max=5041 max_rank=2 min=3375 min_rank=1 imbalance_ratio=1.230713 max_load_reduction=27727 max_load_reduction_pct=0.846161 moved_to_echo=15904 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 12, 0, 0, 4095, 2, 2, 198, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1726, 0, 0, 1870, 0, 588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 1, 0, 0, 3676, 80, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4079, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 12, 4012, 0, 0, 0, 0, 0, 0, 0, 11, 0, 133, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 2, 0, 3908, 0, 85, 19, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4091, 4, 0, 1, 0, 48, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4309, 4185, 3760, 4085, 4028, 4236, 4020, 4145] avg=4096.0000 max=4309 max_rank=0 min=3760 min_rank=2 imbalance_ratio=1.052002 max_load_reduction=28459 max_load_reduction_pct=0.868500 moved_to_echo=16429 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 45, 0, 0, 4010, 0, 0, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 3254, 0, 1259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 9, 0, 0, 3654, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 8, 0, 0, 0, 0, 0, 0, 0, 0, 36, 142, 0, 3, 1, 0, 0, 4010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 3879, 3, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 4094, 0, 85, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3637, 0, 19, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4252, 4553, 3747, 4046, 4192, 4020, 4218, 3740] avg=4096.0000 max=4553 max_rank=1 min=3740 min_rank=7 imbalance_ratio=1.111572 max_load_reduction=28215 max_load_reduction_pct=0.861053 moved_to_echo=16170 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 7, 0, 0, 2694, 0, 0, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1437, 0, 0, 3391, 0, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 453, 8, 0, 0, 3, 0, 3673, 81, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 1, 0, 0, 4094, 9, 0, 0, 0, 0, 0, 0, 0, 0, 3, 106, 0, 0, 1, 0, 0, 4014, 0, 0, 0, 0, 0, 0, 0, 20, 0, 132, 3807, 1, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 7, 0, 10, 0, 4031, 0, 85, 35, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4074, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2905, 5010, 4218, 4115, 4124, 4056, 4176, 4164] avg=4096.0000 max=5010 max_rank=1 min=2905 min_rank=0 imbalance_ratio=1.223145 max_load_reduction=27758 max_load_reduction_pct=0.847107 moved_to_echo=16520 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4075, 0, 0, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 841, 0, 0, 3647, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 3680, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 13, 0, 4096, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 221, 0, 0, 0, 0, 0, 3680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 133, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 1, 0, 3589, 0, 85, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 48, 0, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4273, 4489, 3809, 4120, 3901, 4217, 3720, 4239] avg=4096.0000 max=4489 max_rank=1 min=3720 min_rank=6 imbalance_ratio=1.095947 max_load_reduction=28279 max_load_reduction_pct=0.863007 moved_to_echo=16077 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4092, 0, 0, 191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1089, 0, 0, 1661, 0, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 896, 1, 0, 0, 0, 0, 3538, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 4045, 4, 0, 0, 0, 0, 0, 0, 0, 5, 0, 280, 0, 0, 0, 0, 0, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 1, 127, 4084, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 98, 15, 0, 126, 3, 4095, 0, 82, 17, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3950, 0, 0, 0, 9, 43, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4283, 2954, 4516, 4055, 4296, 4215, 4444, 4005] avg=4096.0000 max=4516 max_rank=2 min=2954 min_rank=1 imbalance_ratio=1.102539 max_load_reduction=28252 max_load_reduction_pct=0.862183 moved_to_echo=16960 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 3922, 0, 0, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1214, 0, 0, 2725, 0, 264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 166, 22, 0, 0, 0, 0, 3668, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 3985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 4, 0, 3905, 0, 85, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 26, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4203, 3938, 4103, 3989, 4194, 4042, 4178] avg=4096.0000 max=4203 max_rank=1 min=3938 min_rank=2 imbalance_ratio=1.026123 max_load_reduction=28565 max_load_reduction_pct=0.871735 moved_to_echo=16403 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 24, 0, 0, 4095, 0, 0, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 0, 0, 3340, 1, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 7, 3647, 81, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 9, 0, 0, 4078, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236, 0, 0, 0, 0, 0, 3997, 0, 0, 0, 0, 0, 0, 0, 1, 7, 120, 4078, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 3830, 0, 88, 39, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4095, 0, 0, 8, 0, 91, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4320, 3999, 3739, 4101, 4233, 4207, 3968, 4201] avg=4096.0000 max=4320 max_rank=0 min=3739 min_rank=2 imbalance_ratio=1.054688 max_load_reduction=28448 max_load_reduction_pct=0.868164 moved_to_echo=16609 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 2944, 128, 128, 128, 128, 128, 29184, 640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 6912, 128, 0, 19968, 128, 6144, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 0, 0, 31104, 128, 128, 1664, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 256, 128, 128, 256, 128, 31616, 0, 768, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 1024, 0, 128, 128, 0, 128, 31744, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 1152, 31616, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 32128, 128, 128, 128, 128, 640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 0, 128, 128, 128, 128, 128, 32640, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 65, 4, 0, 210, 4062, 0, 0, 0, 2507, 0, 0, 0, 1567, 0, 0, 2, 0, 17, 4012, 0, 0, 0, 0, 0, 32, 0, 0, 22, 0, 1, 4, 1, 1, 0, 0, 0, 0, 2, 4086, 4094, 1, 0, 6, 6, 0, 0, 0, 1, 3345, 0, 4092, 2, 1565, 0, 2663, 1, 0, 0, 1, 396] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 2, 0, 12, 0, 0, 1, 0, 4094, 0, 0, 0, 0, 1, 0, 0, 3234, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 1, 6, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 4086, 4094, 0, 0, 0, 0, 18, 0, 0, 0, 1712, 0, 4085, 0, 3728, 0, 3603, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 19, 0, 4, 0, 17, 4091, 0, 2, 0, 39, 0, 0, 0, 4009, 2, 1, 0, 0, 0, 4079, 10, 1, 0, 0, 0, 0, 0, 0, 86, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4086, 4094, 0, 2, 36, 0, 0, 0, 0, 0, 2866, 0, 4011, 0, 2530, 0, 2737, 0, 0, 12, 0, 30] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 5, 0, 27, 0, 8, 0, 278, 4089, 0, 3, 0, 438, 0, 0, 14, 1990, 0, 0, 0, 0, 0, 4085, 15, 5, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 4089, 0, 0, 1739, 0, 0, 0, 0, 0, 4091, 1, 3840, 0, 2945, 0, 834, 0, 0, 0, 0, 189] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 284, 5, 7, 33, 4093, 0, 0, 0, 226, 0, 0, 0, 4057, 0, 0, 0, 6, 0, 3962, 3, 0, 0, 0, 0, 0, 0, 1, 204, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 4095, 8, 36, 575, 0, 2, 32, 0, 0, 4062, 0, 4046, 0, 203, 0, 2728, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 0, 1, 21, 4096, 0, 3, 0, 39, 0, 0, 0, 3997, 0, 0, 0, 0, 0, 4059, 1, 0, 0, 0, 9, 220, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 4096, 5, 0, 9, 5, 0, 0, 0, 0, 4095, 0, 4053, 0, 510, 0, 3430, 0, 0, 0, 0, 18] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[7, 0, 0, 0, 1, 14, 0, 0, 4087, 0, 0, 0, 904, 0, 0, 4, 3842, 0, 1, 0, 0, 0, 4093, 0, 11, 1, 3, 0, 0, 0, 0, 152, 0, 0, 52, 2, 0, 0, 0, 0, 0, 0, 3860, 4084, 0, 0, 40, 0, 0, 0, 0, 0, 2249, 1, 4080, 3, 3735, 0, 1456, 0, 0, 0, 0, 86] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 98, 4089, 0, 0, 1, 3, 1, 0, 0, 4091, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 4, 0, 0, 0, 52, 0, 0, 5, 0, 0, 0, 0, 0, 0, 7, 4095, 4091, 0, 3, 18, 0, 0, 0, 0, 1, 4077, 0, 4054, 0, 1440, 0, 2549, 0, 0, 1, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 65, 4, 0, 210, 2623, 615, 511, 278, 0, 0, 0, 0, 3551, 0, 0, 0, 2507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1567, 0, 0, 2, 0, 17, 733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 22, 3279, 681, 5, 0, 0, 0, 0, 0, 0, 1, 4, 1, 1, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3808, 0, 1, 0, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3345, 0, 788, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1565, 0, 2048, 1, 0, 0, 1, 396, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4306, 6058, 2319, 4019, 4096, 3823, 4136, 4011] avg=4096.0000 max=6058 max_rank=1 min=2319 min_rank=2 imbalance_ratio=1.479004 max_load_reduction=26710 max_load_reduction_pct=0.815125 moved_to_echo=16066 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 12, 0, 0, 1, 0, 2618, 833, 515, 278, 0, 0, 0, 0, 3579, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3234, 0, 0, 0, 0, 0, 746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 0, 0, 3341, 679, 5, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 1712, 0, 788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3728, 0, 2770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4259, 3580, 3980, 4032, 4093, 3808, 2518, 6498] avg=4096.0000 max=6498 max_rank=7 min=2518 min_rank=6 imbalance_ratio=1.586426 max_load_reduction=26270 max_load_reduction_pct=0.801697 moved_to_echo=16917 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 19, 0, 4, 0, 17, 2571, 632, 515, 278, 0, 0, 0, 0, 3576, 0, 2, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 2, 1, 0, 0, 0, 745, 10, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 86, 3334, 667, 5, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3808, 0, 0, 2, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2866, 0, 773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2530, 0, 2105, 0, 0, 12, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4036, 3617, 4767, 4093, 4093, 3846, 3639, 4677] avg=4096.0000 max=4767 max_rank=2 min=3617 min_rank=1 imbalance_ratio=1.163818 max_load_reduction=28001 max_load_reduction_pct=0.854523 moved_to_echo=16255 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 0, 27, 0, 8, 0, 278, 2461, 192, 514, 278, 0, 0, 0, 0, 3575, 0, 3, 0, 438, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 1990, 0, 0, 0, 0, 0, 746, 15, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 3, 0, 0, 1, 3339, 639, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3801, 0, 0, 0, 1739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2945, 0, 642, 0, 0, 0, 0, 189, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3763, 4030, 2751, 3992, 4084, 5540, 4832, 3776] avg=4096.0000 max=5540 max_rank=5 min=2751 min_rank=2 imbalance_ratio=1.352539 max_load_reduction=27228 max_load_reduction_pct=0.830933 moved_to_echo=18232 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 0, 1, 21, 2598, 793, 515, 279, 0, 0, 0, 0, 3581, 0, 3, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3997, 0, 0, 0, 0, 0, 741, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 220, 0, 0, 1, 3318, 674, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3817, 0, 5, 0, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 0, 2637, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4210, 3623, 4739, 4228, 4091, 3836, 4876, 3165] avg=4096.0000 max=4876 max_rank=6 min=3165 min_rank=7 imbalance_ratio=1.190430 max_load_reduction=27892 max_load_reduction_pct=0.851196 moved_to_echo=15968 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 284, 5, 7, 33, 2593, 630, 515, 279, 0, 0, 0, 0, 3578, 0, 0, 0, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 0, 0, 6, 0, 723, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 204, 3239, 673, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3817, 0, 8, 36, 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 32, 0, 0, 4062, 0, 780, 0, 0, 0, 0, 0, 0, 0, 0, 0, 203, 0, 2098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4348, 3804, 4789, 4123, 4089, 4438, 4876, 2301] avg=4096.0000 max=4876 max_rank=6 min=2301 min_rank=7 imbalance_ratio=1.190430 max_load_reduction=27892 max_load_reduction_pct=0.851196 moved_to_echo=15704 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 98, 2598, 589, 514, 279, 0, 0, 0, 0, 3575, 0, 0, 1, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 52, 3339, 674, 6, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3816, 0, 0, 3, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4077, 0, 782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1440, 0, 1960, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 3580, 4837, 4075, 4090, 3844, 4860, 3403] avg=4096.0000 max=4860 max_rank=6 min=3403 min_rank=7 imbalance_ratio=1.186523 max_load_reduction=27908 max_load_reduction_pct=0.851685 moved_to_echo=16197 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 0, 0, 1, 14, 0, 0, 2619, 340, 519, 267, 0, 0, 0, 0, 3568, 0, 0, 0, 904, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3842, 0, 1, 0, 0, 0, 743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1, 3, 0, 0, 0, 0, 152, 3350, 683, 6, 0, 0, 0, 0, 0, 0, 0, 52, 2, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3593, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2249, 1, 778, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3735, 0, 1116, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3767, 4476, 4586, 4206, 4132, 3633, 3031, 4937] avg=4096.0000 max=4937 max_rank=7 min=3031 min_rank=6 imbalance_ratio=1.205322 max_load_reduction=27831 max_load_reduction_pct=0.849335 moved_to_echo=15733 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 128, 384, 128, 128, 768, 20736, 4736, 4224, 2304, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 384, 0, 128, 640, 26624, 5376, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[16768, 0, 15488, 128, 0, 128, 128, 768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[26880, 128, 128, 128, 128, 128, 6016, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 26624, 128, 6272, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[28672, 0, 128, 128, 4224, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 30336, 0, 128, 128, 2432, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 128, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33792 slack_slots=31744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33792 slack_slots=31744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[5, 0, 4091, 3854, 2, 0, 0, 1, 0, 0, 0, 3962, 1, 4, 4, 2, 0, 0, 0, 3, 0, 0, 170, 3536, 4086, 4089, 0, 0, 7, 0, 5, 0, 0, 0, 0, 0, 21, 0, 4096, 0, 0, 5, 0, 0, 0, 4081, 0, 0, 5, 0, 637, 0, 0, 0, 0, 0, 99, 1, 0, 0, 0, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 2170, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1103, 4083, 4095, 4086, 11, 0, 0, 24, 0, 0, 639, 0, 0, 0, 25, 0, 4085, 0, 0, 0, 0, 0, 0, 4057, 0, 0, 0, 0, 190, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4093, 3780, 0, 118, 2, 0, 1, 3, 2, 2518, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 250, 3947, 3785, 3965, 0, 0, 5, 5, 0, 0, 1628, 0, 0, 5, 0, 45, 4088, 0, 0, 0, 0, 0, 4, 3869, 0, 456, 0, 0, 170, 0, 0, 4, 0, 0, 2, 4, 0, 9, 0, 0, 0, 9] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4086, 3892, 0, 0, 0, 0, 0, 0, 0, 3901, 0, 52, 1, 0, 0, 0, 0, 0, 0, 0, 400, 3522, 3399, 4087, 0, 0, 0, 81, 0, 0, 5, 0, 0, 0, 11, 0, 4083, 6, 0, 0, 0, 0, 28, 4095, 0, 0, 13, 0, 1094, 0, 0, 0, 0, 0, 2, 8, 2, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4092, 4075, 0, 0, 0, 0, 0, 0, 0, 2221, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 691, 4092, 4096, 4094, 2, 0, 0, 4, 0, 0, 37, 0, 0, 0, 0, 0, 3991, 0, 0, 0, 0, 0, 0, 4095, 0, 188, 4, 0, 1051, 0, 0, 0, 0, 0, 0, 0, 1, 8, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4021, 0, 1, 0, 0, 0, 0, 0, 1048, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3344, 3366, 4095, 4081, 0, 0, 0, 0, 0, 1, 574, 0, 0, 0, 2, 0, 4084, 0, 0, 0, 0, 0, 3, 4018, 0, 14, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4086, 4006, 11, 338, 0, 0, 0, 0, 0, 1111, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1195, 3933, 3986, 4080, 14, 0, 0, 65, 0, 0, 1103, 0, 0, 0, 0, 0, 4096, 41, 8, 2, 0, 2, 10, 3970, 0, 2, 0, 0, 677, 0, 1, 0, 0, 0, 1, 0, 0, 25, 0, 0, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 3717, 0, 0, 0, 0, 0, 0, 0, 3791, 0, 2, 0, 0, 0, 131, 0, 17, 0, 0, 567, 3978, 3594, 4095, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 183, 2, 4002, 0, 0, 0, 0, 0, 0, 4096, 0, 12, 0, 0, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 107, 3854, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3962, 1, 4, 4, 2, 649, 514, 257, 31, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 170, 2887, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 178, 0, 0, 7, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 3582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 637, 0, 0, 0, 0, 0, 3568, 0, 0, 0, 0, 0, 0, 0, 99, 1, 0, 0, 0, 0, 1, 0, 3984, 86, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3969, 5424, 3060, 4276, 3603, 4055, 4210, 4171] avg=4096.0000 max=5424 max_rank=1 min=3060 min_rank=2 imbalance_ratio=1.324219 max_load_reduction=27344 max_load_reduction_pct=0.834473 moved_to_echo=16039 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 107, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2170, 0, 0, 0, 0, 749, 513, 256, 31, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1103, 3334, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 179, 11, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 639, 0, 0, 0, 25, 0, 3572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 190, 0, 0, 0, 0, 0, 3565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 3989, 86, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4203, 3720, 4439, 4309, 4236, 4026, 3755, 4080] avg=4096.0000 max=4439 max_rank=2 min=3720 min_rank=1 imbalance_ratio=1.083740 max_load_reduction=28329 max_load_reduction_pct=0.864532 moved_to_echo=16097 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 107, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2221, 0, 0, 0, 0, 751, 501, 257, 31, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 691, 3341, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 179, 2, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 3490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 188, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1051, 0, 0, 0, 0, 0, 3572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8, 0, 0, 0, 0, 3985, 86, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4182, 3761, 4058, 4281, 3527, 4252, 4627, 4080] avg=4096.0000 max=4627 max_rank=6 min=3527 min_rank=4 imbalance_ratio=1.129639 max_load_reduction=28141 max_load_reduction_pct=0.858795 moved_to_echo=16486 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 106, 3892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3901, 0, 52, 1, 0, 646, 512, 256, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 400, 2876, 0, 0, 0, 0, 0, 0, 0, 0, 3399, 179, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 11, 0, 3571, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 1094, 0, 0, 0, 0, 0, 3566, 0, 0, 0, 0, 0, 0, 0, 2, 8, 2, 0, 0, 0, 0, 0, 3980, 86, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3998, 5399, 3276, 3659, 3593, 4092, 4673, 4078] avg=4096.0000 max=5399 max_rank=1 min=3276 min_rank=2 imbalance_ratio=1.318115 max_load_reduction=27369 max_load_reduction_pct=0.835236 moved_to_echo=16436 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 107, 3780, 0, 118, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 2, 2518, 0, 0, 0, 0, 724, 513, 249, 29, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 250, 3223, 0, 0, 0, 0, 0, 0, 0, 0, 3785, 173, 0, 0, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1628, 0, 0, 5, 0, 45, 3575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3840, 0, 456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 170, 0, 0, 4, 0, 0, 3460, 0, 0, 0, 0, 0, 0, 0, 2, 4, 0, 9, 0, 0, 0, 9, 3986, 83, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4007, 4039, 3474, 3968, 5253, 4300, 3634, 4093] avg=4096.0000 max=5253 max_rank=4 min=3474 min_rank=2 imbalance_ratio=1.282471 max_load_reduction=27515 max_load_reduction_pct=0.839691 moved_to_echo=17280 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 107, 3717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3791, 0, 2, 0, 0, 730, 502, 257, 31, 0, 0, 0, 0, 0, 131, 0, 17, 0, 0, 567, 3248, 0, 0, 0, 0, 0, 0, 0, 0, 3594, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 183, 2, 3500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 0, 0, 0, 0, 0, 3573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3989, 86, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3824, 5313, 3963, 3773, 3686, 4077, 4057, 4075] avg=4096.0000 max=5313 max_rank=1 min=3686 min_rank=4 imbalance_ratio=1.297119 max_load_reduction=27455 max_load_reduction_pct=0.837860 moved_to_echo=15895 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 107, 4021, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 0, 0, 0, 0, 618, 512, 256, 30, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3344, 2748, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 178, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 574, 0, 0, 0, 2, 0, 3572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3988, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 3561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 3989, 86, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4129, 2464, 6093, 4274, 4148, 4005, 3568, 4087] avg=4096.0000 max=6093 max_rank=2 min=2464 min_rank=1 imbalance_ratio=1.487549 max_load_reduction=26675 max_load_reduction_pct=0.814056 moved_to_echo=15808 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 101, 4006, 11, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1111, 0, 0, 0, 0, 726, 518, 260, 33, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1195, 3207, 0, 0, 0, 0, 0, 0, 0, 0, 3986, 163, 14, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1103, 0, 0, 0, 0, 0, 3578, 41, 0, 0, 0, 0, 0, 0, 0, 0, 8, 2, 0, 2, 10, 3937, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 677, 0, 1, 0, 0, 0, 3566, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 25, 0, 0, 0, 2, 3985, 91, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4456, 2648, 4405, 4228, 4722, 3961, 4244, 4104] avg=4096.0000 max=4722 max_rank=4 min=2648 min_rank=1 imbalance_ratio=1.152832 max_load_reduction=28046 max_load_reduction_pct=0.855896 moved_to_echo=17031 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 20736, 128, 128, 128, 128, 5632, 4096, 2048, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 256, 128, 128, 0, 0, 7808, 24960, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 4352, 0, 128, 128, 0, 0, 28544, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 0, 0, 128, 256, 128, 28544, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 896, 31488, 128, 512, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[31232, 1408, 128, 0, 128, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 0, 0, 128, 128, 32000, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 128, 32128, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[3607, 13, 305, 0, 10, 4078, 0, 0, 0, 0, 0, 4, 2181, 0, 1, 0, 16, 0, 1, 0, 6, 0, 1, 0, 0, 0, 0, 298, 0, 1895, 1, 40, 1, 0, 0, 1, 4071, 6, 16, 2261, 81, 0, 4087, 0, 2319, 0, 0, 0, 0, 5, 303, 0, 11, 4013, 17, 0, 3092, 0, 5, 12, 0, 0, 10, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[3648, 67, 3067, 0, 215, 4096, 0, 0, 0, 0, 0, 0, 508, 4, 96, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 82, 8, 0, 0, 0, 4096, 0, 7, 4091, 0, 2, 4095, 0, 4054, 0, 0, 0, 0, 0, 108, 0, 0, 4092, 0, 0, 419, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[3203, 0, 577, 0, 376, 4096, 0, 0, 0, 0, 0, 0, 1511, 73, 1, 0, 0, 0, 0, 0, 0, 14, 286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4082, 0, 260, 2917, 0, 9, 4079, 0, 3114, 0, 3, 0, 0, 0, 8, 0, 0, 4095, 16, 0, 4043, 0, 0, 0, 3, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[3995, 0, 2365, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 1868, 10, 8, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 4093, 0, 6, 3819, 6, 0, 4073, 0, 3980, 0, 10, 0, 0, 0, 111, 1, 0, 4091, 0, 0, 167, 8, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[4008, 0, 2180, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4, 91, 0, 13, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2882, 0, 0, 4096, 0, 4090, 0, 74, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 1610, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[4001, 1, 42, 0, 2158, 4067, 0, 0, 0, 0, 0, 0, 3805, 178, 0, 0, 0, 2, 4, 0, 0, 5, 0, 8, 0, 1, 0, 0, 0, 35, 0, 19, 0, 0, 0, 0, 4086, 0, 0, 4083, 0, 2, 4090, 0, 1226, 0, 206, 0, 8, 0, 69, 0, 0, 4091, 0, 0, 581, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[4089, 0, 641, 0, 153, 4093, 0, 0, 0, 0, 0, 0, 2210, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 33, 0, 2, 1, 0, 0, 0, 4093, 0, 0, 3598, 0, 2, 4096, 0, 4091, 0, 1, 0, 0, 0, 2, 0, 0, 4084, 0, 0, 1576, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[4091, 28, 326, 1, 139, 4075, 0, 0, 0, 0, 0, 0, 565, 0, 16, 21, 1, 14, 0, 0, 0, 0, 0, 7, 0, 0, 0, 565, 20, 12, 0, 292, 0, 0, 0, 2, 4085, 0, 0, 2557, 0, 0, 4087, 0, 3569, 3, 74, 2, 2, 0, 53, 0, 0, 4069, 1, 0, 4083, 0, 0, 7, 0, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2367, 13, 305, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2181, 0, 1, 0, 2271, 0, 0, 0, 0, 0, 0, 0, 16, 0, 1, 0, 6, 0, 1, 0, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, 0, 1895, 1, 40, 3355, 267, 44, 0, 0, 0, 0, 0, 1, 0, 0, 1, 779, 6, 16, 2261, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 732, 0, 2319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 303, 0, 11, 3946, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3092, 0, 5, 12, 0, 0, 10, 0, 1240, 754, 67, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2695, 4457, 4058, 5900, 3064, 3132, 4282, 5180] avg=4096.0000 max=5900 max_rank=3 min=2695 min_rank=0 imbalance_ratio=1.440430 max_load_reduction=26868 max_load_reduction_pct=0.819946 moved_to_echo=15658 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2394, 67, 3067, 0, 215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 4, 96, 0, 2285, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 82, 3362, 268, 45, 0, 0, 0, 0, 0, 8, 0, 0, 0, 784, 0, 7, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 733, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 0, 0, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 419, 0, 0, 0, 0, 0, 0, 0, 1254, 759, 68, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5743, 2893, 4052, 3769, 4890, 4789, 4132, 2500] avg=4096.0000 max=5743 max_rank=0 min=2500 min_rank=7 imbalance_ratio=1.402100 max_load_reduction=27025 max_load_reduction_pct=0.824738 moved_to_echo=16311 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2630, 0, 2180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1461, 0, 0, 0, 2285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 91, 0, 13, 0, 0, 3363, 268, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 784, 0, 0, 2882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 733, 0, 4090, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1610, 0, 0, 0, 0, 0, 0, 0, 1378, 759, 68, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4810, 3746, 4054, 3783, 3666, 4897, 3997, 3815] avg=4096.0000 max=4897 max_rank=5 min=3666 min_rank=4 imbalance_ratio=1.195557 max_load_reduction=27871 max_load_reduction_pct=0.850555 moved_to_echo=16375 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2621, 0, 2365, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1868, 10, 8, 0, 2283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3344, 268, 45, 0, 0, 0, 0, 0, 0, 0, 0, 3, 784, 0, 6, 3819, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 729, 0, 3980, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 1, 0, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 167, 8, 0, 0, 0, 0, 0, 0, 1374, 758, 68, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4987, 4169, 4107, 3658, 4612, 4725, 4135, 2375] avg=4096.0000 max=4987 max_rank=0 min=2375 min_rank=7 imbalance_ratio=1.217529 max_load_reduction=27781 max_load_reduction_pct=0.847809 moved_to_echo=15847 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2102, 0, 577, 0, 376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1511, 73, 1, 0, 2277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 286, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3349, 267, 45, 0, 0, 0, 0, 0, 0, 1, 0, 0, 782, 0, 260, 2917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 730, 0, 3114, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4027, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 0, 0, 0, 3, 1, 0, 0, 1101, 756, 68, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3055, 3862, 4351, 3661, 3960, 3856, 4051, 5972] avg=4096.0000 max=5972 max_rank=7 min=3055 min_rank=0 imbalance_ratio=1.458008 max_load_reduction=26796 max_load_reduction_pct=0.817749 moved_to_echo=17839 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2625, 1, 42, 0, 2158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3805, 178, 0, 0, 2279, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 0, 0, 5, 0, 8, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 35, 0, 19, 3358, 268, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 782, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 732, 0, 1226, 0, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 69, 0, 0, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 581, 0, 0, 0, 0, 0, 0, 0, 1376, 757, 68, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4826, 6262, 4043, 3724, 4865, 2166, 4100, 2782] avg=4096.0000 max=6262 max_rank=1 min=2166 min_rank=5 imbalance_ratio=1.528809 max_load_reduction=26506 max_load_reduction_pct=0.808899 moved_to_echo=13913 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2683, 0, 641, 0, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2210, 0, 2, 0, 2283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 33, 0, 2, 3363, 268, 44, 0, 0, 0, 0, 0, 1, 0, 0, 0, 784, 0, 0, 3598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 733, 0, 4091, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1576, 0, 0, 0, 0, 0, 0, 0, 1406, 758, 68, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3477, 4495, 4049, 3711, 4383, 4827, 4018, 3808] avg=4096.0000 max=4827 max_rank=5 min=3477 min_rank=0 imbalance_ratio=1.178467 max_load_reduction=27941 max_load_reduction_pct=0.852692 moved_to_echo=17036 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2681, 28, 326, 1, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 565, 0, 16, 21, 2282, 0, 0, 0, 0, 0, 0, 0, 1, 14, 0, 0, 0, 0, 0, 7, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 565, 20, 12, 0, 292, 3359, 271, 43, 0, 0, 0, 0, 0, 0, 0, 0, 2, 769, 0, 0, 2557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 728, 0, 3569, 3, 74, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 53, 0, 0, 3997, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 7, 0, 0, 1, 0, 1410, 763, 72, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3175, 2884, 4054, 4562, 3328, 4376, 4053, 6336] avg=4096.0000 max=6336 max_rank=7 min=2884 min_rank=1 imbalance_ratio=1.546875 max_load_reduction=26432 max_load_reduction_pct=0.806641 moved_to_echo=18093 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[15616, 128, 128, 128, 128, 128, 128, 0, 10624, 6144, 640, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 14208, 384, 128, 128, 18304, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[20224, 128, 9600, 128, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 6272, 128, 384, 26240, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 128, 1024, 128, 2048, 128, 512, 26880, 2176, 384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 5888, 0, 26496, 128, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 768, 128, 128, 32128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 128, 128, 384, 128, 32384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33792 slack_slots=31744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4083, 4002, 0, 4045, 342, 0, 0, 5, 7, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 2042, 0, 0, 0, 64, 0, 0, 2897, 0, 4085, 1, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 82, 5, 271, 1358, 0, 60, 1, 0, 67, 0, 0, 4040, 0, 0, 1204, 3, 1, 0, 10, 4085] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4092, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 2735, 0, 0, 0, 8, 0, 0, 3665, 0, 4092, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 59, 1590, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 134, 0, 0, 1, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[2, 6, 0, 4058, 4096, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 3558, 0, 0, 0, 0, 0, 0, 2987, 1, 3953, 0, 0, 82, 0, 95, 18, 11, 0, 0, 4, 3, 7, 0, 0, 909, 0, 3, 1, 0, 0, 3, 0, 4014, 0, 0, 730, 4, 1, 0, 72, 4073] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4087, 4096, 0, 4013, 264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 387, 0, 0, 0, 0, 3, 0, 3562, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 88, 100, 0, 3, 0, 0, 1, 0, 0, 4091, 0, 0, 3880, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 8, 2, 4052, 4060, 0, 4071, 147, 0, 0, 41, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 3479, 0, 0, 0, 9, 8, 0, 3557, 0, 4096, 7, 0, 0, 0, 16, 8, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 0, 0, 0, 0, 0, 0, 3892, 0, 0, 1000, 0, 0, 19, 10, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4093, 0, 4095, 317, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 18, 0, 0, 0, 1, 401, 0, 0, 0, 12, 1, 0, 3685, 0, 4096, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 27, 774, 0, 2, 0, 0, 0, 0, 0, 4078, 0, 0, 2950, 0, 0, 0, 5, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4093, 4095, 0, 4002, 261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 650, 0, 0, 0, 30, 0, 0, 3407, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 993, 0, 0, 0, 0, 4, 0, 2, 4096, 0, 0, 2937, 0, 0, 0, 5, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4062, 4086, 0, 4084, 21, 0, 0, 11, 0, 0, 0, 30, 0, 0, 0, 3, 15, 0, 0, 0, 1342, 0, 0, 0, 9, 0, 0, 4086, 0, 4061, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1240, 0, 261, 1, 1, 0, 0, 31, 4076, 0, 0, 1236, 0, 10, 2, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3959, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2735, 1326, 945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 3665, 0, 448, 137, 8, 4, 0, 0, 0, 0, 4084, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 59, 1590, 3144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 0, 0, 1, 0, 2322, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3965, 4109, 5006, 4270, 4086, 4802, 4073, 2457] avg=4096.0000 max=5006 max_rank=2 min=2457 min_rank=7 imbalance_ratio=1.222168 max_load_reduction=27762 max_load_reduction_pct=0.847229 moved_to_echo=15418 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2, 6, 0, 0, 0, 0, 3937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4058, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3558, 1318, 947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2987, 1, 445, 136, 8, 4, 0, 0, 0, 0, 3945, 0, 0, 82, 0, 95, 18, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 7, 0, 0, 909, 3146, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 3, 0, 4010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 730, 4, 1, 0, 72, 2310, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3945, 4064, 5824, 3581, 4151, 4069, 4017, 3117] avg=4096.0000 max=5824 max_rank=2 min=3117 min_rank=7 imbalance_ratio=1.421875 max_load_reduction=26944 max_load_reduction_pct=0.822266 moved_to_echo=15354 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3879, 264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 387, 1326, 946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3562, 0, 448, 134, 8, 4, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 88, 100, 3147, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3880, 0, 0, 0, 0, 2322, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4143, 4090, 2659, 4159, 4088, 3336, 4091, 6202] avg=4096.0000 max=6202 max_rank=7 min=2659 min_rank=2 imbalance_ratio=1.514160 max_load_reduction=26566 max_load_reduction_pct=0.810730 moved_to_echo=17717 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3910, 342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 7, 0, 0, 1, 0, 4083, 2, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 2042, 1322, 926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 2897, 0, 447, 135, 8, 4, 0, 0, 0, 0, 4077, 1, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 5, 271, 1358, 3074, 0, 0, 0, 0, 0, 0, 0, 0, 60, 1, 0, 67, 0, 0, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1204, 3, 1, 0, 10, 2316, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4252, 4098, 4292, 3555, 4083, 4790, 4164, 3534] avg=4096.0000 max=4790 max_rank=5 min=3534 min_rank=7 imbalance_ratio=1.169434 max_load_reduction=27978 max_load_reduction_pct=0.853821 moved_to_echo=16571 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3958, 317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 1, 401, 1326, 945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 0, 3685, 0, 448, 137, 8, 4, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 774, 3144, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2950, 0, 0, 0, 5, 2322, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4275, 4108, 2691, 4295, 4101, 3945, 4076, 5277] avg=4096.0000 max=5277 max_rank=7 min=2691 min_rank=2 imbalance_ratio=1.288330 max_load_reduction=27491 max_load_reduction_pct=0.838959 moved_to_echo=17399 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 2, 0, 0, 0, 3935, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 1, 0, 4052, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3479, 1326, 937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8, 0, 3557, 0, 448, 136, 8, 4, 0, 0, 0, 0, 4088, 7, 0, 0, 0, 16, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 3121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 0, 0, 19, 10, 2322, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4096, 5743, 4170, 4119, 3309, 3888, 3351] avg=4096.0000 max=5743 max_rank=2 min=3309 min_rank=5 imbalance_ratio=1.402100 max_load_reduction=27025 max_load_reduction_pct=0.824738 moved_to_echo=14667 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3869, 261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 650, 1326, 947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 3407, 0, 448, 133, 8, 4, 0, 0, 0, 0, 4088, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 993, 3145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2937, 0, 0, 0, 5, 2322, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4130, 4096, 2923, 4030, 4089, 4138, 4098, 5264] avg=4096.0000 max=5264 max_rank=7 min=2923 min_rank=2 imbalance_ratio=1.285156 max_load_reduction=27504 max_load_reduction_pct=0.839355 moved_to_echo=17589 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3945, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 30, 0, 4062, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3, 15, 0, 0, 0, 1342, 1327, 943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4086, 0, 451, 139, 14, 9, 0, 0, 0, 0, 4047, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1240, 3139, 0, 0, 0, 0, 0, 0, 0, 0, 261, 1, 1, 0, 0, 31, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1236, 0, 10, 2, 0, 2318, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3966, 4107, 3630, 4708, 4051, 4379, 4361, 3566] avg=4096.0000 max=4708 max_rank=3 min=3566 min_rank=7 imbalance_ratio=1.149414 max_load_reduction=28060 max_load_reduction_pct=0.856323 moved_to_echo=16357 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 0, 128, 128, 14720, 10624, 7552, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 14080, 128, 128, 128, 128, 18560, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 128, 128, 128, 128, 512, 7168, 25088, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 0, 0, 256, 128, 0, 27904, 128, 3584, 1152, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 0, 0, 0, 31488, 1408, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 384, 128, 128, 128, 128, 128, 32384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[32512, 128, 0, 128, 128, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 128, 128, 0, 0, 128, 0, 32640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4082, 0, 0, 0, 0, 0, 15, 10, 0, 4096, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 7, 4072, 0, 0, 0, 3, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 3794, 0, 0, 0, 4096, 0, 0, 4082, 0, 0, 0, 1215, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 3226, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4091, 0, 27, 65, 3, 2, 0, 26, 0, 4096, 0, 1, 0, 3868, 0, 0, 3962, 0, 0, 0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 725, 0, 0, 0, 4091, 0, 10, 1, 20, 0, 0, 0, 0, 4092, 4, 12, 0, 0, 3586, 0, 0, 0, 0, 0, 3, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4064, 0, 0, 272, 20, 0, 0, 0, 0, 4066, 0, 0, 0, 3888, 0, 0, 4025, 2, 73, 0, 147, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 9, 10, 7, 1, 4042, 0, 0, 130, 0, 0, 2, 0, 0, 4018, 0, 25, 16, 0, 3817, 0, 0, 0, 0, 0, 24, 14, 6, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 1, 0, 0, 39, 0, 1, 4096, 0, 0, 0, 3983, 0, 0, 3832, 0, 0, 0, 296, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 11, 0, 0, 10] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4093, 0, 0, 29, 0, 0, 30, 0, 0, 4050, 0, 0, 0, 4092, 0, 0, 4087, 0, 0, 0, 1078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 57, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2964, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 6, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4044, 0, 0, 0, 2288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 19, 4095, 0, 0, 0, 3, 0, 0, 0, 0, 4090, 4, 0, 0, 0, 1837, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4072, 0, 0, 3, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 4093, 0, 0, 4088, 0, 6, 0, 39, 0, 0, 0, 0, 0, 0, 1, 0, 0, 18, 4095, 0, 3, 0, 1, 3, 4038, 0, 0, 0, 30, 0, 0, 0, 5, 4066, 0, 0, 0, 1, 4084, 9, 0, 8, 0, 1, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 10, 0, 4096, 0, 0, 0, 25, 4, 0, 0, 0, 0, 0, 0, 3506, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 60, 0, 0, 0, 0, 0, 0, 0, 0, 3994, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 628, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3394, 588, 102, 17, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4040, 4150, 3541, 4156, 4001, 4071, 4708, 4101] avg=4096.0000 max=4708 max_rank=6 min=3541 min_rank=2 imbalance_ratio=1.149414 max_load_reduction=28060 max_load_reduction_pct=0.856323 moved_to_echo=16881 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4022, 0, 0, 272, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 25, 4, 0, 0, 0, 0, 0, 0, 3330, 0, 0, 0, 2, 73, 0, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4025, 59, 0, 0, 0, 0, 0, 0, 0, 0, 3986, 0, 9, 10, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 0, 0, 130, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 619, 0, 25, 16, 0, 3817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 14, 6, 0, 3340, 558, 102, 17, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4314, 4095, 3552, 4086, 4013, 4170, 4477, 4061] avg=4096.0000 max=4477 max_rank=6 min=3552 min_rank=2 imbalance_ratio=1.093018 max_load_reduction=28291 max_load_reduction_pct=0.863373 moved_to_echo=16721 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3794, 0, 0, 0, 25, 5, 0, 0, 0, 0, 0, 0, 3508, 0, 0, 0, 0, 0, 0, 1215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4082, 60, 0, 0, 0, 0, 0, 0, 0, 0, 3994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 625, 0, 0, 0, 0, 3226, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3379, 588, 102, 17, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4052, 3824, 4723, 4146, 3994, 4091, 3852, 4086] avg=4096.0000 max=4723 max_rank=2 min=3824 min_rank=1 imbalance_ratio=1.153076 max_load_reduction=28045 max_load_reduction_pct=0.855865 moved_to_echo=16023 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4052, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 1, 4096, 0, 0, 0, 25, 5, 0, 0, 0, 0, 0, 0, 3411, 0, 0, 0, 0, 0, 0, 296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 3832, 61, 0, 0, 0, 0, 0, 0, 0, 0, 3983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 629, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 10, 3404, 572, 102, 17, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4053, 4166, 3707, 3951, 3983, 4074, 4718, 4116] avg=4096.0000 max=4718 max_rank=6 min=3707 min_rank=2 imbalance_ratio=1.151855 max_load_reduction=28050 max_load_reduction_pct=0.856018 moved_to_echo=16891 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4051, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 4050, 0, 0, 0, 25, 5, 0, 0, 0, 0, 0, 0, 3505, 0, 0, 0, 0, 0, 0, 1078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 61, 0, 0, 0, 0, 0, 0, 0, 0, 3994, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 630, 0, 0, 0, 0, 2964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3405, 587, 102, 17, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4110, 4583, 4148, 4051, 4091, 3594, 4111] avg=4096.0000 max=4583 max_rank=2 min=3594 min_rank=6 imbalance_ratio=1.118896 max_load_reduction=28185 max_load_reduction_pct=0.860138 moved_to_echo=15847 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4049, 0, 27, 65, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 26, 0, 4096, 0, 1, 0, 25, 5, 0, 0, 0, 0, 0, 0, 3313, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3962, 60, 0, 0, 0, 0, 0, 0, 0, 0, 3975, 0, 725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 10, 1, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 630, 4, 12, 0, 0, 3586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3402, 555, 101, 17, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4144, 4155, 3319, 4023, 4700, 4117, 4232, 4078] avg=4096.0000 max=4700 max_rank=4 min=3319 min_rank=2 imbalance_ratio=1.147461 max_load_reduction=28068 max_load_reduction_pct=0.856567 moved_to_echo=17127 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4054, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 25, 5, 0, 0, 0, 0, 0, 0, 3508, 0, 0, 0, 0, 0, 0, 2288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 60, 0, 0, 0, 0, 0, 0, 0, 0, 3992, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 630, 4, 0, 0, 0, 1837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3400, 588, 102, 17, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4126, 5796, 4104, 4011, 4093, 2471, 4107] avg=4096.0000 max=5796 max_rank=2 min=2471 min_rank=6 imbalance_ratio=1.415039 max_load_reduction=26972 max_load_reduction_pct=0.823120 moved_to_echo=14682 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4022, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 32, 7, 0, 0, 0, 0, 0, 0, 3502, 0, 0, 0, 0, 6, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4088, 65, 0, 0, 0, 0, 0, 0, 0, 18, 3990, 0, 3, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4031, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 617, 0, 0, 0, 1, 4084, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 1, 0, 0, 0, 1, 3384, 591, 105, 18, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4025, 4142, 3547, 4154, 4015, 4061, 4716, 4108] avg=4096.0000 max=4716 max_rank=6 min=3547 min_rank=2 imbalance_ratio=1.151367 max_load_reduction=28052 max_load_reduction_pct=0.856079 moved_to_echo=16900 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 5120, 128, 128, 128, 128, 27776, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 128, 128, 128, 128, 128, 27136, 4736, 896, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[27648, 0, 0, 0, 128, 128, 0, 5120, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 32000, 0, 896, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 0, 0, 0, 128, 0, 32256, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 32384, 0, 128, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 32512, 0, 128, 0, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32640, 0, 128, 256, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33792 slack_slots=31744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 1, 4096, 0, 1, 4095, 7, 0, 0, 4096, 1252, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 844, 0, 8, 0, 0, 0, 0, 0, 2002, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[7, 1, 0, 0, 0, 6, 0, 0, 55, 0, 4078, 0, 0, 4087, 0, 0, 4050, 191, 5, 0, 4056, 562, 0, 0, 4059, 34, 0, 4086, 7, 0, 0, 0, 4088, 5, 0, 0, 11, 0, 0, 0, 0, 7, 3, 0, 0, 0, 2, 0, 207, 0, 310, 0, 8, 0, 0, 0, 2832, 0, 0, 3, 2, 1, 5, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 4086, 0, 0, 4086, 592, 0, 37, 4083, 0, 4, 0, 4079, 0, 0, 3623, 0, 0, 3, 0, 4093, 0, 0, 22, 0, 0, 0, 0, 1, 4, 0, 0, 0, 0, 0, 0, 3852, 0, 1, 0, 0, 0, 0, 22, 73, 0, 0, 0, 0, 0, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3839, 0, 0, 4096, 0, 0, 4096, 40, 0, 0, 4096, 4, 0, 0, 4093, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 997, 0, 0, 0, 0, 0, 0, 1952, 1, 0, 0, 0, 0, 0, 0, 1363, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 11, 0, 0, 4076, 0, 0, 4093, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 14, 0, 0, 0, 22, 0, 0, 0, 4082, 0, 0, 4096, 0, 0, 4083, 658, 0, 0, 4095, 9, 0, 0, 4095, 0, 0, 3076, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 10, 0, 17, 13, 0, 0, 0, 3688, 0, 0, 0, 0, 0, 0, 35, 675, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4020, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 3572, 0, 0, 0, 0, 0, 0, 0, 592, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5, 0, 20, 2, 0, 1, 4022, 0, 0, 4079, 7, 0, 4072, 10, 0, 0, 4096, 233, 0, 0, 4093, 0, 0, 1538, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 2, 1, 0, 0, 0, 2562, 1, 0, 0, 0, 0, 0, 16, 3823, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[7, 1, 0, 0, 0, 6, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 55, 0, 4078, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3614, 191, 5, 0, 0, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 34, 0, 4086, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 5, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3, 0, 0, 0, 2, 0, 3939, 4, 0, 0, 0, 0, 0, 0, 207, 0, 310, 0, 8, 0, 0, 0, 1504, 436, 1, 0, 0, 0, 0, 0, 2832, 0, 0, 3, 2, 1, 5, 0, 2046, 108, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 4173, 4372, 4636, 4103, 3955, 2466, 4997] avg=4096.0000 max=4997 max_rank=7 min=2466 min_rank=6 imbalance_ratio=1.219971 max_load_reduction=27771 max_load_reduction_pct=0.847504 moved_to_echo=15521 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 1, 41, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3654, 7, 0, 0, 0, 1252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 3947, 4, 0, 0, 0, 0, 0, 0, 844, 0, 8, 0, 0, 0, 0, 0, 1517, 441, 1, 0, 0, 0, 0, 0, 2002, 0, 0, 0, 0, 0, 0, 0, 2065, 108, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4113, 4913, 4610, 4094, 3956, 2811, 4175] avg=4096.0000 max=4913 max_rank=2 min=2811 min_rank=6 imbalance_ratio=1.199463 max_load_reduction=27855 max_load_reduction_pct=0.850067 moved_to_echo=15036 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3839, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3655, 40, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 997, 0, 0, 0, 0, 0, 0, 3947, 3, 0, 0, 0, 0, 0, 0, 1952, 1, 0, 0, 0, 0, 0, 0, 1516, 441, 1, 0, 0, 0, 0, 0, 1363, 0, 0, 0, 0, 0, 0, 0, 2063, 108, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 3880, 3699, 4609, 4095, 4947, 3911, 3534] avg=4096.0000 max=4947 max_rank=5 min=3534 min_rank=7 imbalance_ratio=1.207764 max_load_reduction=27821 max_load_reduction_pct=0.849030 moved_to_echo=16487 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3655, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 3947, 4, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 1510, 441, 1, 0, 0, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 2055, 108, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4137, 3666, 4604, 4092, 3967, 1997, 6213] avg=4096.0000 max=6213 max_rank=7 min=1997 min_rank=6 imbalance_ratio=1.516846 max_load_reduction=26555 max_load_reduction_pct=0.810394 moved_to_echo=16269 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3646, 592, 0, 37, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 0, 0, 3623, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 0, 0, 0, 0, 3938, 4, 0, 0, 0, 0, 0, 0, 3852, 0, 1, 0, 0, 0, 0, 22, 1511, 440, 1, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 0, 2, 0, 2056, 107, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4143, 4279, 4138, 4114, 3947, 5827, 2238] avg=4096.0000 max=5827 max_rank=6 min=2238 min_rank=7 imbalance_ratio=1.422607 max_load_reduction=26941 max_load_reduction_pct=0.822174 moved_to_echo=16126 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 14, 0, 0, 0, 22, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3643, 658, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 0, 0, 3076, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 10, 0, 17, 13, 0, 0, 0, 3947, 4, 0, 0, 0, 0, 0, 0, 3688, 0, 0, 0, 0, 0, 0, 35, 1517, 440, 1, 0, 0, 0, 0, 0, 675, 0, 0, 0, 0, 0, 0, 0, 2064, 108, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4127, 4123, 4310, 3592, 4095, 3993, 5681, 2847] avg=4096.0000 max=5681 max_rank=6 min=2847 min_rank=7 imbalance_ratio=1.386963 max_load_reduction=27087 max_load_reduction_pct=0.826630 moved_to_echo=16616 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3655, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 0, 0, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 3947, 4, 0, 0, 0, 0, 0, 0, 3572, 0, 0, 0, 0, 0, 0, 0, 1517, 441, 1, 0, 0, 0, 0, 0, 592, 0, 0, 0, 0, 0, 0, 0, 2065, 108, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4137, 3656, 4534, 4095, 3958, 5531, 2765] avg=4096.0000 max=5531 max_rank=6 min=2765 min_rank=7 imbalance_ratio=1.350342 max_load_reduction=27237 max_load_reduction_pct=0.831207 moved_to_echo=16349 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5, 0, 20, 2, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4022, 0, 0, 32, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3630, 10, 0, 0, 0, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 0, 0, 1538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 2, 1, 0, 0, 0, 3937, 3, 0, 0, 0, 0, 0, 0, 2562, 1, 0, 0, 0, 0, 0, 16, 1520, 442, 3, 0, 0, 0, 0, 0, 3823, 0, 0, 0, 0, 0, 0, 0, 2066, 110, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4062, 3873, 2045, 4080, 4045, 4544, 5999] avg=4096.0000 max=5999 max_rank=7 min=2045 min_rank=3 imbalance_ratio=1.464600 max_load_reduction=26769 max_load_reduction_pct=0.816925 moved_to_echo=18668 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[16768, 128, 384, 0, 128, 0, 0, 128, 12160, 3584, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[15488, 0, 0, 128, 128, 128, 128, 0, 16512, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 128, 0, 28672, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[29184, 1536, 128, 128, 0, 2176, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 1152, 128, 128, 128, 0, 128, 0, 31616, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 32384, 0, 128, 384, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 17, 0, 2, 25, 4074, 0, 0, 0, 0, 0, 0, 3, 1132, 4, 41, 0, 4024, 4077, 0, 3856, 0, 483, 0, 0, 0, 3986, 0, 5, 1, 0, 4, 0, 0, 0, 2769, 0, 12, 5, 6, 0, 0, 7, 0, 0, 0, 313, 5, 5, 0, 4084, 0, 2, 0, 24, 23, 10, 0, 0, 3769] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 68, 0, 0, 210, 4096, 0, 0, 0, 0, 0, 0, 0, 1520, 0, 68, 0, 4096, 4096, 0, 4025, 13, 3450, 0, 0, 0, 4082, 2, 0, 0, 0, 0, 0, 0, 0, 2755, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 134, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 29] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 6, 0, 0, 120, 4096, 15, 0, 0, 0, 0, 0, 0, 146, 0, 1, 0, 4096, 4095, 0, 4088, 8, 443, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 3563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3935] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 0, 9, 0, 0, 13, 4082, 23, 0, 0, 0, 0, 0, 0, 3769, 0, 0, 0, 4096, 4096, 0, 4095, 17, 627, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 3181, 0, 0, 0, 0, 64, 0, 0, 0, 1, 0, 5, 0, 0, 0, 4096, 0, 0, 3, 0, 14, 0, 0, 0, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 34, 9, 2, 8, 181, 0, 0, 438, 4087, 266, 0, 0, 0, 6, 0, 0, 654, 0, 0, 0, 4077, 4075, 0, 4040, 88, 311, 0, 0, 0, 3961, 0, 0, 0, 9, 0, 0, 0, 0, 4049, 1, 0, 0, 0, 10, 4, 0, 0, 0, 0, 576, 0, 0, 0, 4085, 0, 0, 0, 0, 9, 0, 0, 0, 1788] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 11, 0, 873, 0, 0, 594, 4095, 0, 0, 0, 1, 0, 0, 0, 1644, 0, 3, 0, 4096, 4088, 0, 3996, 114, 484, 0, 0, 0, 4014, 0, 0, 0, 0, 0, 0, 0, 0, 617, 0, 0, 0, 0, 1691, 0, 0, 0, 0, 0, 104, 0, 0, 0, 4079, 0, 0, 0, 0, 2, 0, 0, 0, 2262] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 864, 0, 0, 465, 4095, 1, 0, 0, 0, 0, 0, 0, 1476, 0, 0, 0, 4096, 4087, 0, 4090, 0, 18, 0, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 2661, 0, 0, 0, 0, 126, 0, 0, 0, 152, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2483] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 30, 0, 0, 8, 92, 0, 0, 25, 4079, 24, 0, 0, 0, 0, 0, 0, 1046, 3, 0, 8, 4070, 4080, 3, 4089, 236, 855, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 2578, 0, 0, 0, 0, 84, 0, 0, 0, 1, 0, 150, 0, 0, 0, 4096, 0, 0, 5, 0, 0, 1, 0, 0, 3110] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 68, 0, 0, 3609, 245, 0, 0, 0, 0, 0, 0, 210, 3826, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1520, 0, 68, 0, 2667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3195, 13, 3450, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2755, 830, 270, 175, 43, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 3851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 0, 0, 0, 3921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 29, 1386, 473, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3922, 4036, 4255, 6660, 4073, 3878, 4055, 1889] avg=4096.0000 max=6660 max_rank=3 min=1889 min_rank=7 imbalance_ratio=1.625977 max_load_reduction=26108 max_load_reduction_pct=0.796753 moved_to_echo=13895 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 17, 0, 2, 3524, 245, 0, 0, 0, 0, 0, 0, 25, 3805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1132, 4, 41, 0, 2620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3061, 0, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 4, 0, 0, 0, 2769, 795, 269, 175, 42, 0, 0, 0, 0, 0, 12, 5, 6, 0, 0, 7, 0, 3832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 313, 5, 5, 0, 3909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 24, 23, 10, 0, 0, 3769, 1362, 462, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3788, 3830, 3800, 3544, 4060, 3862, 4232, 5652] avg=4096.0000 max=5652 max_rank=7 min=3544 min_rank=3 imbalance_ratio=1.379883 max_load_reduction=27116 max_load_reduction_pct=0.827515 moved_to_echo=17806 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 0, 9, 0, 0, 3604, 246, 0, 0, 0, 0, 0, 0, 13, 3813, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3769, 0, 0, 0, 2667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3251, 17, 627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3181, 844, 269, 175, 43, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 3850, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 0, 3921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 14, 0, 0, 0, 498, 1386, 472, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3862, 3849, 6436, 3895, 4512, 3914, 3927, 2373] avg=4096.0000 max=6436 max_rank=2 min=2373 min_rank=7 imbalance_ratio=1.571289 max_load_reduction=26332 max_load_reduction_pct=0.803589 moved_to_echo=14726 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 6, 0, 0, 3590, 246, 0, 0, 0, 0, 0, 0, 120, 3826, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 0, 1, 0, 2667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3245, 8, 443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3563, 843, 270, 175, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3935, 1386, 470, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3842, 3961, 2814, 3696, 4894, 3849, 3921, 5791] avg=4096.0000 max=5791 max_rank=7 min=2814 min_rank=2 imbalance_ratio=1.413818 max_load_reduction=26977 max_load_reduction_pct=0.823273 moved_to_echo=18455 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 11, 0, 873, 0, 0, 3549, 245, 0, 0, 0, 0, 0, 0, 594, 3825, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1644, 0, 3, 0, 2667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3172, 114, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 617, 824, 270, 175, 43, 0, 0, 0, 0, 0, 0, 0, 0, 1691, 0, 0, 0, 3843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 0, 3904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2262, 1386, 465, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4678, 4420, 4314, 3770, 1929, 5534, 4008, 4115] avg=4096.0000 max=5534 max_rank=5 min=1929 min_rank=4 imbalance_ratio=1.351074 max_load_reduction=27234 max_load_reduction_pct=0.831116 moved_to_echo=15586 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 34, 9, 2, 8, 181, 0, 0, 3503, 245, 0, 0, 0, 0, 0, 0, 438, 3817, 266, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 654, 0, 0, 0, 2654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3207, 88, 311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4049, 833, 270, 175, 43, 0, 0, 0, 0, 1, 0, 0, 0, 10, 4, 0, 0, 3830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 576, 0, 0, 0, 3910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 1788, 1380, 458, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3982, 4527, 3308, 3606, 5379, 3845, 4486, 3635] avg=4096.0000 max=5379 max_rank=4 min=3308 min_rank=2 imbalance_ratio=1.313232 max_load_reduction=27389 max_load_reduction_pct=0.835846 moved_to_echo=17345 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 864, 0, 0, 3588, 246, 0, 0, 0, 0, 0, 0, 465, 3825, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1476, 0, 0, 0, 2667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3247, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2661, 843, 270, 175, 43, 0, 0, 0, 0, 0, 0, 0, 0, 126, 0, 0, 0, 3841, 0, 0, 0, 0, 0, 0, 0, 152, 0, 0, 0, 0, 0, 3921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2483, 1386, 470, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4698, 4291, 4143, 3265, 3992, 3967, 4073, 4339] avg=4096.0000 max=4698 max_rank=0 min=3265 min_rank=3 imbalance_ratio=1.146973 max_load_reduction=28070 max_load_reduction_pct=0.856628 moved_to_echo=16371 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 30, 0, 0, 8, 92, 0, 0, 3621, 245, 0, 0, 0, 0, 0, 0, 25, 3805, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 3, 0, 8, 2638, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3241, 236, 855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2578, 848, 274, 181, 48, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 3835, 0, 0, 0, 0, 0, 0, 0, 1, 0, 150, 0, 0, 0, 3915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 1, 0, 0, 3110, 1384, 474, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3996, 3854, 3698, 4332, 3929, 3919, 4066, 4974] avg=4096.0000 max=4974 max_rank=7 min=3698 min_rank=2 imbalance_ratio=1.214355 max_load_reduction=27794 max_load_reduction_pct=0.848206 moved_to_echo=16888 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 11392, 128, 128, 128, 21248, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 0, 0, 17920, 11136, 3840, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 0, 0, 0, 22272, 6784, 2176, 1408, 384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[25728, 512, 6784, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 128, 2176, 0, 128, 28672, 2048, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1920, 30592, 384, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 2048, 128, 128, 0, 30848, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 0, 1408, 128, 128, 0, 31360, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 3147, 0, 21, 0, 0, 0, 0, 4096, 320, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 629, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 4096, 4056, 0, 0, 4096, 0, 3952, 0, 3, 0, 0, 0, 0, 4096, 144, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 11, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 4078, 13, 4087, 4022, 0, 0, 4080, 2, 2891, 9, 0, 0, 0, 0, 13, 4069, 487, 4048, 0, 0, 0, 0, 579, 0, 0, 7, 2, 20, 185, 6, 0, 5, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4076, 12, 24, 9, 7, 16, 0, 8, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4089, 0, 4096, 3980, 0, 0, 4096, 0, 606, 0, 95, 0, 0, 0, 0, 4096, 1540, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 1710, 0, 67, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[9, 0, 0, 0, 4089, 0, 4095, 4018, 0, 0, 4096, 0, 1278, 0, 1, 0, 0, 0, 0, 4088, 1, 3862, 0, 0, 0, 0, 4, 0, 0, 0, 2792, 0, 257, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 75, 5, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 4088, 0, 4091, 4041, 0, 0, 4096, 0, 1783, 0, 102, 0, 0, 42, 0, 4052, 245, 4087, 0, 0, 0, 0, 5, 0, 0, 0, 2055, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 1, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 5, 0, 3, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 2700, 0, 3, 0, 0, 0, 0, 4095, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 1395, 0, 6, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[61, 0, 0, 0, 4096, 6, 4095, 4018, 0, 0, 4088, 0, 2236, 0, 5, 0, 0, 1, 0, 4093, 225, 4018, 0, 0, 0, 1, 1073, 0, 0, 0, 583, 29, 15, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 44, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1744, 0, 3147, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 320, 3748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 629, 0, 2352, 321, 65, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 3987, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4140, 4912, 4068, 3367, 4004, 4095, 4087, 4095] avg=4096.0000 max=4912 max_rank=1 min=3367 min_rank=3 imbalance_ratio=1.199219 max_load_reduction=27856 max_load_reduction_pct=0.850098 moved_to_echo=16281 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 45, 13, 0, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1737, 2, 2891, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 487, 3723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 579, 0, 0, 7, 2, 20, 2343, 319, 64, 0, 0, 0, 0, 0, 185, 6, 0, 5, 0, 0, 0, 3, 3969, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 12, 24, 6, 0, 0, 0, 0, 0, 0, 0, 9, 7, 16, 0, 8, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7, 4087, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4639, 4223, 3334, 4177, 4118, 4100, 4095] avg=4096.0000 max=4639 max_rank=1 min=3334 min_rank=3 imbalance_ratio=1.132568 max_load_reduction=28129 max_load_reduction_pct=0.858429 moved_to_echo=16490 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 44, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1744, 0, 3952, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 3761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2352, 322, 65, 0, 0, 0, 0, 0, 11, 3, 0, 0, 0, 0, 0, 0, 3987, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 4086, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 5699, 3905, 2741, 4011, 4102, 4114, 4096] avg=4096.0000 max=5699 max_rank=1 min=2741 min_rank=3 imbalance_ratio=1.391357 max_load_reduction=27069 max_load_reduction_pct=0.826080 moved_to_echo=16323 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 44, 0, 0, 3980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1744, 0, 606, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1540, 3766, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1710, 0, 2352, 322, 65, 0, 0, 0, 0, 0, 67, 203, 0, 0, 0, 0, 0, 0, 3980, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4024, 2445, 5306, 4449, 4261, 4102, 4087, 4094] avg=4096.0000 max=5306 max_rank=2 min=2445 min_rank=1 imbalance_ratio=1.295410 max_load_reduction=27462 max_load_reduction_pct=0.838074 moved_to_echo=16544 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[9, 0, 0, 0, 44, 0, 0, 4018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1744, 0, 1278, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 2792, 0, 2352, 304, 65, 0, 0, 0, 0, 0, 257, 9, 0, 0, 0, 0, 0, 0, 3980, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 5, 4078, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4071, 3023, 3553, 5517, 4256, 4095, 4159, 4094] avg=4096.0000 max=5517 max_rank=3 min=3023 min_rank=1 imbalance_ratio=1.346924 max_load_reduction=27251 max_load_reduction_pct=0.831635 moved_to_echo=16604 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 44, 0, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1744, 0, 1783, 0, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 245, 3759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 2055, 0, 2352, 322, 65, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3979, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 6, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 3629, 4046, 4799, 3989, 4069, 4050, 4099] avg=4096.0000 max=4799 max_rank=3 min=3629 min_rank=1 imbalance_ratio=1.171631 max_load_reduction=27969 max_load_reduction_pct=0.853546 moved_to_echo=16207 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 44, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1744, 0, 2700, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1395, 0, 2352, 321, 65, 0, 0, 0, 0, 0, 6, 7, 0, 0, 0, 0, 0, 0, 3987, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4085, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4140, 4447, 3752, 4133, 4010, 4104, 4087, 4095] avg=4096.0000 max=4447 max_rank=1 min=3752 min_rank=2 imbalance_ratio=1.085693 max_load_reduction=28321 max_load_reduction_pct=0.864288 moved_to_echo=16296 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[61, 0, 0, 0, 39, 6, 0, 4018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1733, 0, 2236, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 225, 3689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1073, 0, 0, 0, 583, 29, 2355, 320, 67, 0, 0, 0, 0, 0, 15, 44, 0, 0, 0, 0, 0, 0, 3990, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 1, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 2, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4124, 3974, 3915, 4428, 4060, 4083, 4084, 4100] avg=4096.0000 max=4428 max_rank=3 min=3915 min_rank=2 imbalance_ratio=1.081055 max_load_reduction=28340 max_load_reduction_pct=0.864868 moved_to_echo=16327 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 1664, 0, 0, 128, 9216, 128, 18816, 2560, 640, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 13952, 128, 18688, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 0, 3072, 29824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[640, 384, 0, 128, 0, 0, 0, 128, 31872, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 128, 128, 0, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 384, 128, 0, 32384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 128, 128, 128, 128, 32640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[6, 0, 0, 3881, 0, 0, 0, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 61, 0, 4096, 0, 0, 0, 0, 0, 3967, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 1021, 3366, 0, 0, 0, 0, 2, 0, 0, 0, 1, 4010, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 71, 0, 3235, 0, 18, 0, 0, 11, 4, 0, 0, 0, 0, 0, 1, 0, 2721, 15, 4011, 0, 0, 62, 0, 8, 4027, 0, 6, 1, 0, 0, 0, 2, 0, 3, 1, 1, 5, 4058, 0, 13, 0, 0, 0, 0, 4004, 24, 227, 892, 0, 0, 0, 0, 0, 0, 4, 0, 1987, 3269, 0, 0, 0, 4087, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 3991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1425, 0, 4096, 0, 0, 0, 0, 0, 4079, 0, 0, 5, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4092, 0, 2027, 788, 0, 0, 0, 0, 0, 0, 0, 0, 557, 3951, 0, 0, 0, 3662, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4021, 0, 0, 0, 0, 0, 185, 0, 0, 0, 0, 0, 0, 0, 2479, 0, 4096, 0, 0, 0, 0, 0, 3707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 1746, 0, 0, 0, 0, 0, 0, 0, 0, 236, 4031, 0, 1, 0, 4067, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[14, 0, 0, 3985, 0, 0, 0, 0, 0, 488, 0, 0, 0, 0, 0, 0, 0, 2377, 0, 4096, 0, 0, 0, 0, 0, 2643, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 4096, 0, 158, 3710, 0, 1, 0, 0, 0, 0, 0, 0, 600, 3021, 0, 2, 0, 3479, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 1542, 0, 0, 0, 0, 0, 625, 0, 0, 0, 1, 0, 0, 0, 2224, 0, 4089, 0, 0, 0, 0, 2, 3885, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 6, 0, 24, 4096, 0, 2570, 1407, 0, 0, 0, 0, 15, 0, 0, 0, 62, 4081, 0, 0, 0, 4043, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 3120, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1470, 0, 4096, 0, 0, 0, 0, 0, 3606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4, 0, 4094, 0, 1631, 2193, 0, 0, 0, 0, 0, 0, 0, 0, 775, 3725, 0, 0, 0, 3958, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[21, 10, 0, 2475, 0, 0, 15, 0, 0, 118, 0, 0, 0, 0, 0, 0, 0, 1180, 0, 4095, 0, 0, 0, 0, 0, 3493, 0, 0, 0, 0, 7, 0, 0, 8, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 58, 0, 4080, 0, 33, 2683, 0, 0, 0, 0, 0, 0, 0, 3, 3336, 3179, 1, 0, 0, 3877, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 71, 0, 3235, 0, 18, 0, 0, 692, 79, 0, 0, 0, 0, 0, 0, 11, 4, 0, 0, 0, 0, 0, 1, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 2721, 15, 2303, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4027, 0, 6, 1, 0, 0, 0, 370, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 1, 1, 5, 4058, 0, 3, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 3063, 24, 227, 0, 0, 0, 0, 0, 0, 0, 0, 892, 0, 0, 0, 0, 0, 0, 4, 1708, 249, 0, 0, 0, 0, 0, 0, 0, 1987, 2817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4103, 5101, 4412, 4073, 3327, 2853, 4804] avg=4096.0000 max=5101 max_rank=2 min=2853 min_rank=6 imbalance_ratio=1.245361 max_load_reduction=27667 max_load_reduction_pct=0.844330 moved_to_echo=15057 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3991, 0, 0, 0, 0, 708, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3661, 0, 0, 0, 0, 0, 0, 0, 0, 1425, 0, 2351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 5, 0, 0, 0, 448, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4093, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3129, 0, 2027, 0, 0, 0, 0, 0, 0, 0, 0, 788, 0, 0, 0, 0, 0, 0, 0, 1745, 255, 1, 0, 0, 0, 0, 0, 0, 557, 3404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4795, 3661, 3776, 4532, 4098, 5156, 2789, 3961] avg=4096.0000 max=5156 max_rank=5 min=2789 min_rank=6 imbalance_ratio=1.258789 max_load_reduction=27612 max_load_reduction_pct=0.842651 moved_to_echo=16004 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4021, 0, 0, 0, 0, 708, 98, 0, 0, 0, 0, 0, 0, 0, 185, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 2479, 0, 2351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3707, 0, 0, 0, 0, 0, 0, 457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3133, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 1746, 0, 0, 0, 0, 0, 0, 0, 1745, 255, 0, 0, 0, 0, 0, 0, 0, 236, 3473, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4827, 4252, 4830, 4164, 4099, 3140, 3746, 3710] avg=4096.0000 max=4830 max_rank=2 min=3140 min_rank=5 imbalance_ratio=1.179199 max_load_reduction=27938 max_load_reduction_pct=0.852600 moved_to_echo=14695 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1542, 0, 0, 0, 0, 708, 99, 0, 0, 0, 0, 0, 0, 0, 625, 0, 0, 0, 1, 0, 0, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 2224, 0, 2347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3885, 0, 0, 0, 6, 0, 0, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 24, 3133, 0, 2570, 0, 0, 0, 0, 0, 0, 0, 0, 1407, 0, 0, 0, 0, 15, 0, 0, 1742, 255, 0, 0, 0, 0, 0, 0, 0, 62, 3516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2349, 4669, 4571, 4355, 4094, 5733, 3419, 3578] avg=4096.0000 max=5733 max_rank=5 min=2349 min_rank=0 imbalance_ratio=1.399658 max_load_reduction=27035 max_load_reduction_pct=0.825043 moved_to_echo=16824 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[14, 0, 0, 3985, 0, 0, 0, 0, 708, 73, 0, 0, 0, 0, 0, 0, 0, 488, 0, 0, 0, 0, 0, 0, 3478, 0, 0, 0, 0, 0, 0, 0, 0, 2377, 0, 2351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2643, 0, 0, 0, 0, 7, 0, 342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3133, 0, 158, 0, 0, 0, 0, 0, 0, 0, 0, 3710, 0, 1, 0, 0, 0, 0, 0, 1745, 255, 1, 0, 0, 0, 0, 0, 0, 600, 2604, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4780, 3966, 4728, 2992, 4093, 3291, 5712, 3206] avg=4096.0000 max=5712 max_rank=6 min=2992 min_rank=3 imbalance_ratio=1.394531 max_load_reduction=27056 max_load_reduction_pct=0.825684 moved_to_echo=16302 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 0, 3881, 0, 0, 0, 0, 708, 97, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 2351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3967, 0, 0, 0, 0, 0, 0, 454, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4089, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3133, 0, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 3366, 0, 0, 0, 0, 2, 0, 0, 1745, 255, 0, 0, 0, 0, 0, 0, 0, 1, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4692, 4163, 2412, 4421, 4101, 4154, 5368, 3457] avg=4096.0000 max=5368 max_rank=6 min=2412 min_rank=2 imbalance_ratio=1.310547 max_load_reduction=27400 max_load_reduction_pct=0.836182 moved_to_echo=17080 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3120, 0, 0, 0, 0, 708, 90, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3958, 0, 0, 0, 0, 0, 0, 0, 0, 1470, 0, 2351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3606, 0, 0, 0, 0, 0, 0, 422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3131, 0, 1631, 0, 0, 0, 0, 0, 0, 0, 0, 2193, 0, 0, 0, 0, 0, 0, 0, 1745, 255, 0, 0, 0, 0, 0, 0, 0, 775, 3210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3918, 3959, 3821, 4028, 4098, 4766, 4193, 3985] avg=4096.0000 max=4766 max_rank=5 min=3821 min_rank=2 imbalance_ratio=1.163574 max_load_reduction=28002 max_load_reduction_pct=0.854553 moved_to_echo=17042 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[21, 10, 0, 2475, 0, 0, 15, 0, 711, 80, 0, 0, 0, 0, 0, 0, 0, 118, 0, 0, 0, 0, 0, 0, 3877, 0, 0, 0, 0, 0, 0, 0, 0, 1180, 0, 2349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3493, 0, 0, 0, 0, 7, 0, 364, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 4096, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 3110, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 2683, 0, 0, 0, 0, 0, 0, 0, 1746, 259, 0, 0, 0, 0, 0, 0, 3, 3336, 2727, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3312, 3995, 3529, 3864, 4112, 3201, 4688, 6067] avg=4096.0000 max=6067 max_rank=7 min=3201 min_rank=5 imbalance_ratio=1.481201 max_load_reduction=26701 max_load_reduction_pct=0.814850 moved_to_echo=18068 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[16896, 0, 128, 0, 0, 128, 0, 128, 13952, 2048, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 13952, 128, 18816, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 128, 25088, 128, 7680, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 7680, 25216, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 0, 26368, 0, 128, 128, 0, 5760, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 29440, 0, 128, 128, 128, 128, 0, 3328, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 128, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 1536, 0, 0, 0, 128, 0, 128, 31360, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4079, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 38, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 4096, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 4011, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4065, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 0, 0, 0, 0, 4096, 0, 0, 0, 4086, 0, 0, 4096, 2, 0, 0, 0, 0, 28, 0, 164, 0, 0, 0, 0, 0, 0, 0, 3910, 0, 0, 4096, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[4, 5, 0, 6, 0, 16, 4086, 3977, 0, 0, 0, 0, 215, 11, 8, 0, 0, 0, 0, 3, 0, 8, 0, 29, 5, 6, 4052, 0, 0, 0, 15, 8, 12, 4060, 0, 35, 0, 4082, 10, 2, 4081, 1, 0, 0, 0, 0, 52, 0, 0, 0, 0, 1, 0, 0, 0, 16, 3945, 0, 0, 4017, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4030, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 4091, 2, 0, 282, 0, 0, 0, 4087, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3883, 0, 0, 4086, 0, 6, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3825, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 0, 0, 0, 4096, 0, 0, 0, 4090, 0, 0, 4096, 0, 0, 45, 0, 0, 0, 0, 210, 0, 0, 0, 0, 0, 0, 56, 4065, 0, 0, 4079, 0, 0, 0, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4086, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 43, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 4092, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3904, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 188, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 5, 4095, 0, 0, 4096, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 12, 43, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 4088, 1, 0, 39, 5, 0, 0, 6, 0, 0, 0, 0, 0, 1, 0, 0, 4065, 0, 0, 4021, 0, 0, 0, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4, 5, 0, 6, 0, 16, 4086, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 215, 11, 8, 0, 3908, 50, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8, 0, 29, 4078, 0, 0, 0, 0, 0, 0, 0, 5, 6, 3978, 0, 0, 0, 15, 8, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4059, 0, 35, 0, 0, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 1, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 16, 3879, 74, 22, 1, 0, 0, 0, 0, 3945, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4186, 4196, 4118, 4012, 4118, 4112, 3993, 4033] avg=4096.0000 max=4196 max_rank=1 min=3993 min_rank=6 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16256 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4079, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 51, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4021, 0, 1, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 3956, 75, 22, 1, 0, 0, 0, 0, 4011, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4151, 4079, 4091, 4060, 4095, 4092, 4100, 4100] avg=4096.0000 max=4151 max_rank=0 min=4060 min_rank=3 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16387 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4065, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4024, 51, 4, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4021, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 2, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 0, 0, 0, 0, 0, 0, 0, 3956, 75, 22, 1, 0, 0, 0, 0, 3910, 0, 0, 89, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4137, 4082, 4104, 4028, 4095, 4104, 4218, 4000] avg=4096.0000 max=4218 max_rank=6 min=4000 min_rank=7 imbalance_ratio=1.029785 max_load_reduction=28550 max_load_reduction_pct=0.871277 moved_to_echo=16417 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4086, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4024, 51, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4018, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3952, 75, 22, 1, 0, 0, 0, 0, 4069, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4158, 4080, 4092, 4061, 4095, 4074, 4050, 4158] avg=4096.0000 max=4158 max_rank=0 min=4050 min_rank=6 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16377 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4030, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4024, 51, 5, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4016, 2, 0, 282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 1, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3946, 75, 22, 1, 0, 0, 0, 0, 3883, 0, 0, 89, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4083, 4095, 4300, 4086, 4080, 4044, 3978] avg=4096.0000 max=4300 max_rank=3 min=3978 min_rank=7 imbalance_ratio=1.049805 max_load_reduction=28468 max_load_reduction_pct=0.868774 moved_to_echo=16188 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3825, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 51, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4021, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 210, 0, 0, 0, 0, 0, 0, 56, 3939, 75, 22, 1, 0, 0, 0, 0, 4065, 0, 0, 89, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3897, 4080, 4085, 4028, 4095, 4119, 4303, 4161] avg=4096.0000 max=4303 max_rank=6 min=3897 min_rank=0 imbalance_ratio=1.050537 max_load_reduction=28465 max_load_reduction_pct=0.868683 moved_to_echo=16678 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3904, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 51, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4021, 0, 0, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 5, 3956, 75, 22, 1, 0, 0, 0, 0, 4095, 0, 0, 89, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3976, 4080, 4091, 4209, 4095, 4071, 4061, 4185] avg=4096.0000 max=4209 max_rank=3 min=3976 min_rank=0 imbalance_ratio=1.027588 max_load_reduction=28559 max_load_reduction_pct=0.871552 moved_to_echo=16412 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4031, 52, 5, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4015, 0, 12, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 1, 0, 39, 5, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3887, 81, 23, 7, 0, 0, 0, 0, 4065, 0, 0, 82, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4161, 4088, 4092, 4070, 4089, 4116, 3999, 4153] avg=4096.0000 max=4161 max_rank=0 min=3999 min_rank=6 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16357 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[384, 0, 128, 128, 0, 128, 0, 128, 31488, 640, 256, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32128, 0, 0, 768, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 0, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 256, 128, 128, 0, 32128, 512, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32640, 128, 0, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 0, 128, 32256, 640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32768, 0, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 32128, 128, 128, 640, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 1263, 4096, 0, 35, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2827, 4095, 0, 0, 0, 0, 4084, 0, 0, 31, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4048, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 0, 0, 0, 0, 1626, 4093, 0, 0, 0, 3983, 3, 0, 0, 0, 0, 4096, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1217, 4091, 3, 0, 0, 10, 4093, 0, 0, 0, 0, 0, 275, 0, 4085, 0, 0, 0, 0, 0, 0, 3, 0, 1291, 0, 0, 0, 0, 0, 0, 3873, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 3941, 4096, 0, 0, 0, 3900, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 4096, 0, 0, 0, 0, 4096, 0, 0, 18, 0, 0, 3, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 721, 0, 0, 0, 0, 0, 0, 3625, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 6, 10, 20, 14, 0, 0, 8, 715, 4067, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 4085, 0, 10, 0, 0, 0, 0, 0, 0, 0, 10, 1, 707, 4079, 0, 0, 10, 0, 4087, 0, 0, 6, 0, 11, 0, 0, 4079, 0, 0, 0, 13, 3, 0, 0, 0, 2660, 0, 8, 0, 0, 6, 0, 4079, 1, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 2162, 4096, 0, 1, 0, 3385, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2908, 4096, 0, 0, 0, 1, 4074, 0, 0, 18, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 932, 0, 0, 0, 0, 0, 0, 2901, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 1265, 4096, 0, 5, 0, 3838, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2465, 4096, 0, 0, 0, 0, 4091, 0, 0, 5, 0, 0, 292, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 327, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4, 0, 0, 1530, 4094, 0, 0, 0, 3384, 0, 0, 12, 0, 0, 4093, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1572, 4086, 0, 0, 0, 0, 4056, 0, 0, 33, 0, 0, 107, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 2227, 0, 0, 0, 0, 0, 0, 3486, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 708, 4096, 0, 6, 0, 3116, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3039, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 12, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1377, 0, 0, 0, 0, 0, 0, 4031, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 10, 20, 14, 0, 0, 8, 4063, 0, 0, 0, 0, 0, 0, 0, 715, 0, 0, 0, 0, 2666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1, 707, 1182, 1154, 0, 0, 0, 0, 0, 0, 10, 0, 0, 10, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 11, 0, 0, 3984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 3, 0, 0, 0, 2660, 0, 2887, 4, 0, 0, 0, 0, 0, 0, 8, 0, 0, 6, 0, 4079, 1, 6, 247, 95, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 3381, 4092, 3054, 4107, 4001, 5567, 4445] avg=4096.0000 max=5567 max_rank=6 min=3054 min_rank=3 imbalance_ratio=1.359131 max_load_reduction=27201 max_load_reduction_pct=0.830109 moved_to_echo=18120 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 1263, 0, 0, 35, 0, 2683, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2827, 1187, 1161, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 4000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2899, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 0, 248, 96, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 3981, 4093, 5175, 4093, 4031, 2908, 4395] avg=4096.0000 max=5175 max_rank=3 min=2908 min_rank=6 imbalance_ratio=1.263428 max_load_reduction=27593 max_load_reduction_pct=0.842072 moved_to_echo=15427 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 3941, 0, 0, 0, 0, 2557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 1187, 1106, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 3, 0, 4000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 721, 0, 2899, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3625, 0, 0, 237, 95, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 6498, 4093, 2374, 4106, 4021, 3624, 3960] avg=4096.0000 max=6498 max_rank=1 min=2374 min_rank=3 imbalance_ratio=1.586426 max_load_reduction=26270 max_load_reduction_pct=0.801697 moved_to_echo=15711 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 1265, 0, 0, 5, 0, 2516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2465, 1187, 1089, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 292, 0, 4000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 327, 0, 2900, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 233, 96, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 3786, 4093, 4741, 4100, 4297, 3231, 4428] avg=4096.0000 max=4741 max_rank=3 min=3231 min_rank=6 imbalance_ratio=1.157471 max_load_reduction=28027 max_load_reduction_pct=0.855316 moved_to_echo=16056 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 2162, 0, 0, 1, 0, 2220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2908, 1187, 960, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 1, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 4000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 932, 0, 2899, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2901, 0, 0, 205, 96, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4383, 4093, 5056, 4085, 4018, 3836, 3205] avg=4096.0000 max=5056 max_rank=3 min=3205 min_rank=7 imbalance_ratio=1.234375 max_load_reduction=27712 max_load_reduction_pct=0.845703 moved_to_echo=15144 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 1626, 0, 0, 0, 0, 2611, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1217, 1186, 1130, 0, 0, 0, 0, 0, 0, 9, 3, 0, 0, 10, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 275, 0, 3990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1291, 0, 2896, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3873, 0, 0, 242, 95, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4240, 4115, 3533, 4115, 4265, 4194, 4213] avg=4096.0000 max=4265 max_rank=5 min=3533 min_rank=3 imbalance_ratio=1.041260 max_load_reduction=28503 max_load_reduction_pct=0.869843 moved_to_echo=16787 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 708, 0, 0, 6, 0, 2043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3039, 1187, 884, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1377, 0, 2899, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4031, 0, 0, 189, 95, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 2757, 4093, 5110, 4106, 4012, 4280, 4318] avg=4096.0000 max=5110 max_rank=3 min=2757 min_rank=1 imbalance_ratio=1.247559 max_load_reduction=27658 max_load_reduction_pct=0.844055 moved_to_echo=16716 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 1530, 0, 0, 0, 0, 2212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1572, 1189, 962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 107, 0, 3983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2227, 0, 2897, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3486, 0, 0, 210, 99, 9, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 3742, 4096, 3725, 4056, 4123, 5128, 3804] avg=4096.0000 max=5128 max_rank=6 min=3725 min_rank=3 imbalance_ratio=1.251953 max_load_reduction=27640 max_load_reduction_pct=0.843506 moved_to_echo=17111 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 0, 128, 128, 14848, 9600, 8448, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[13312, 0, 0, 128, 0, 19584, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 128, 0, 9600, 0, 23296, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 0, 0, 128, 0, 30208, 128, 128, 1920, 768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 128, 768, 0, 32000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 128, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 224, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 9, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 3864, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 67, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 375, 0, 0, 2000, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4051, 0, 0, 0, 1697, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1313, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 2804, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 777, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 2, 3, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 3268, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 7, 0, 0, 4096, 0, 0, 480, 0, 0, 0, 0, 0, 0, 1, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 447, 117, 0, 4087, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 4088, 34, 0, 0, 3085, 4085, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 4076, 8, 0, 0, 0, 0, 13, 4082, 0, 8, 237, 0, 0, 0, 1, 0, 0, 0, 4069, 0, 7, 1, 0, 5, 0, 7, 0, 0, 0, 0, 0, 8, 0, 0, 0, 14, 0, 26, 0, 0, 0, 241, 13, 9, 4077, 0, 0, 23, 9, 0, 4044, 68, 0, 22, 0, 0, 3872, 0, 5, 0, 3742, 4081, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2885, 0, 0, 2, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 230, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 988, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[10, 4096, 0, 0, 0, 2, 0, 0, 4077, 0, 0, 51, 0, 0, 41, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4000, 4085, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3338, 0, 0, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 8, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 2899, 757, 380, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 3715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1164, 0, 0, 0, 3864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 3562, 4107, 4099, 4066, 3724, 4088, 5028] avg=4096.0000 max=5028 max_rank=7 min=3562 min_rank=1 imbalance_ratio=1.227539 max_load_reduction=27740 max_load_reduction_pct=0.846558 moved_to_echo=16906 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4074, 8, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 3328, 0, 8, 237, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 7, 1, 0, 8, 2, 1, 0, 0, 0, 0, 0, 5, 0, 7, 0, 0, 0, 0, 0, 4081, 2, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 14, 0, 26, 0, 2761, 754, 378, 0, 0, 0, 0, 0, 0, 0, 241, 13, 9, 3698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 9, 0, 4036, 68, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1109, 0, 5, 0, 3742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 3574, 4088, 4095, 3941, 3961, 4158, 4856] avg=4096.0000 max=4856 max_rank=7 min=3574 min_rank=1 imbalance_ratio=1.185547 max_load_reduction=27912 max_load_reduction_pct=0.851807 moved_to_echo=16916 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3339, 0, 0, 1313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 8, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2909, 757, 380, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1168, 0, 0, 0, 2804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4652, 4096, 4099, 4048, 3719, 4088, 3972] avg=4096.0000 max=4652 max_rank=1 min=3719 min_rank=5 imbalance_ratio=1.135742 max_load_reduction=28116 max_load_reduction_pct=0.858032 moved_to_echo=15827 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3339, 0, 0, 777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 8, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 2918, 757, 380, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 3715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1171, 0, 0, 0, 3268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4116, 4107, 4099, 4105, 3720, 4088, 4439] avg=4096.0000 max=4439 max_rank=7 min=3720 min_rank=5 imbalance_ratio=1.083740 max_load_reduction=28329 max_load_reduction_pct=0.864532 moved_to_echo=16352 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3339, 0, 0, 67, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 8, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 375, 2888, 757, 380, 0, 0, 0, 0, 0, 0, 0, 2000, 0, 0, 3715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1160, 0, 0, 0, 1697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 3407, 4107, 4099, 4400, 5715, 4088, 2857] avg=4096.0000 max=5715 max_rank=5 min=2857 min_rank=7 imbalance_ratio=1.395264 max_load_reduction=27053 max_load_reduction_pct=0.825592 moved_to_echo=17060 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3339, 0, 0, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4033, 0, 0, 0, 0, 8, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 2915, 757, 379, 0, 0, 0, 0, 0, 0, 0, 447, 117, 0, 3707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1170, 34, 0, 0, 3085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 3819, 4045, 4088, 4066, 4271, 4089, 4289] avg=4096.0000 max=4289 max_rank=7 min=3819 min_rank=1 imbalance_ratio=1.047119 max_load_reduction=28479 max_load_reduction_pct=0.869110 moved_to_echo=16715 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3339, 0, 0, 2885, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 8, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2920, 757, 380, 0, 0, 0, 0, 0, 0, 0, 0, 230, 0, 3715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1173, 0, 0, 0, 988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 6226, 4098, 4099, 4057, 3945, 4088, 2161] avg=4096.0000 max=6226 max_rank=1 min=2161 min_rank=7 imbalance_ratio=1.520020 max_load_reduction=26542 max_load_reduction_pct=0.809998 moved_to_echo=14251 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[10, 4089, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3320, 0, 0, 51, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 15, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 2923, 757, 382, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1166, 0, 0, 0, 4000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 3412, 4120, 4090, 4085, 3713, 4081, 5166] avg=4096.0000 max=5166 max_rank=7 min=3412 min_rank=1 imbalance_ratio=1.261230 max_load_reduction=27602 max_load_reduction_pct=0.842346 moved_to_echo=17045 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 128, 0, 128, 512, 23168, 6144, 3072, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 9344, 128, 128, 0, 23552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[26752, 0, 128, 6144, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 2816, 384, 128, 29696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32768, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 0, 0, 0, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 32768, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 0, 128, 32768, 0, 128, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True + [2026-09-21 08:24:25] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 264.2 | throughput per GPU (TFLOP/s/GPU): 121.9 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 9.133475E+00 | loss scale: 1.0 | grad norm: 15.155 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[177, 151, 1054, 2, 0, 697, 1143, 462, 0, 946, 13, 0, 46, 12, 361, 54, 26, 893, 558, 2201, 40, 271, 276, 1, 43, 8, 55, 237, 898, 885, 796, 2088, 541, 298, 408, 1124, 138, 1495, 209, 746, 1621, 462, 1885, 95, 828, 663, 20, 72, 1011, 376, 615, 55, 510, 1591, 1494, 40, 960, 40, 607, 11, 6, 26, 421, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[191, 129, 527, 5, 1, 689, 908, 139, 5, 755, 24, 2, 14, 44, 297, 101, 6, 1080, 1118, 1910, 16, 298, 269, 2, 77, 2, 20, 212, 1078, 806, 1063, 1082, 808, 383, 608, 1128, 38, 1515, 563, 880, 1092, 340, 1568, 27, 857, 831, 19, 88, 1022, 592, 731, 32, 675, 1527, 2295, 66, 1301, 23, 256, 14, 20, 23, 551, 25] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[287, 270, 715, 1, 4, 802, 728, 161, 0, 1336, 26, 5, 18, 16, 176, 157, 20, 1205, 919, 1949, 23, 255, 239, 16, 225, 9, 74, 274, 1324, 595, 813, 1489, 560, 177, 464, 633, 190, 1493, 223, 510, 1628, 339, 1587, 100, 1021, 836, 9, 140, 922, 465, 603, 66, 748, 1631, 1976, 72, 1464, 25, 186, 4, 20, 19, 496, 30] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[129, 259, 687, 0, 0, 521, 1083, 408, 3, 1059, 21, 0, 5, 20, 399, 77, 16, 872, 1103, 2201, 31, 51, 225, 1, 78, 17, 51, 205, 1077, 1007, 1204, 1616, 909, 387, 333, 749, 197, 1217, 203, 948, 950, 664, 1815, 22, 1034, 692, 11, 35, 729, 972, 582, 118, 659, 900, 1483, 13, 1138, 11, 350, 7, 7, 33, 1164, 10] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[241, 169, 786, 2, 1, 449, 1083, 286, 1, 934, 25, 0, 11, 11, 138, 156, 1, 922, 1201, 1951, 28, 195, 262, 4, 64, 11, 118, 252, 963, 1017, 1040, 1589, 534, 313, 590, 899, 98, 1367, 299, 572, 1406, 646, 2193, 34, 872, 806, 15, 161, 863, 436, 736, 4, 570, 1522, 1765, 27, 1036, 19, 301, 8, 4, 15, 729, 17] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[240, 185, 815, 1, 2, 691, 850, 231, 1, 547, 30, 0, 15, 37, 553, 152, 2, 888, 660, 1968, 22, 170, 137, 2, 73, 27, 49, 236, 1075, 702, 1090, 1453, 424, 365, 555, 1371, 77, 1685, 571, 884, 1209, 566, 1863, 41, 1031, 810, 36, 36, 770, 506, 620, 23, 803, 1212, 1947, 22, 1234, 11, 252, 5, 26, 11, 879, 19] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[100, 190, 655, 4, 0, 758, 1217, 189, 0, 1403, 63, 0, 10, 6, 430, 111, 9, 1075, 770, 2353, 27, 53, 294, 1, 47, 23, 58, 202, 1027, 1046, 950, 1717, 380, 270, 401, 664, 160, 1445, 239, 771, 1089, 435, 2444, 36, 894, 977, 9, 32, 599, 483, 784, 63, 617, 1402, 1642, 58, 1175, 20, 352, 13, 9, 19, 487, 11] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[130, 553, 600, 0, 0, 672, 1209, 375, 0, 863, 30, 2, 36, 41, 165, 119, 2, 893, 778, 1839, 30, 298, 275, 11, 93, 56, 93, 409, 1500, 611, 823, 1473, 507, 435, 406, 1415, 102, 1688, 834, 933, 840, 644, 1918, 68, 1051, 795, 14, 192, 588, 690, 648, 84, 441, 828, 1565, 88, 917, 35, 203, 10, 10, 11, 806, 23] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[177, 151, 1054, 2, 0, 697, 1143, 462, 879, 328, 6, 0, 0, 0, 0, 0, 0, 946, 13, 0, 46, 12, 361, 54, 1494, 829, 0, 0, 0, 0, 0, 0, 26, 893, 558, 1873, 40, 271, 276, 1, 0, 0, 0, 0, 0, 0, 0, 0, 43, 8, 55, 237, 898, 885, 796, 999, 0, 0, 0, 0, 0, 0, 0, 0, 541, 298, 408, 1124, 138, 420, 209, 746, 0, 0, 0, 0, 0, 0, 0, 0, 1621, 462, 436, 95, 828, 663, 20, 72, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 376, 615, 55, 510, 1585, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 960, 40, 607, 11, 6, 26, 421, 6, 1075, 620, 210, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4899, 3755, 3938, 3921, 3884, 4197, 4192, 3982] avg=4096.0000 max=4899 max_rank=0 min=3755 min_rank=1 imbalance_ratio=1.196045 max_load_reduction=27869 max_load_reduction_pct=0.850494 moved_to_echo=16255 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[287, 270, 715, 1, 4, 802, 728, 161, 627, 290, 6, 0, 0, 0, 0, 0, 0, 1336, 26, 5, 18, 16, 176, 157, 1976, 698, 0, 0, 0, 0, 0, 0, 20, 1205, 919, 1659, 23, 255, 239, 16, 0, 0, 0, 0, 0, 0, 0, 0, 225, 9, 74, 274, 1324, 595, 813, 713, 0, 0, 0, 0, 0, 0, 0, 0, 560, 177, 464, 633, 190, 420, 223, 510, 0, 0, 0, 0, 0, 0, 0, 0, 1628, 339, 367, 100, 1021, 836, 9, 140, 0, 0, 0, 0, 0, 0, 0, 0, 922, 465, 603, 66, 748, 1625, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 1464, 25, 186, 4, 20, 19, 496, 30, 1073, 522, 149, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3891, 4408, 4336, 4027, 3177, 4440, 4501, 3988] avg=4096.0000 max=4501 max_rank=6 min=3177 min_rank=4 imbalance_ratio=1.098877 max_load_reduction=28267 max_load_reduction_pct=0.862640 moved_to_echo=16106 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[129, 259, 687, 0, 0, 521, 1083, 408, 680, 328, 3, 0, 0, 0, 0, 0, 3, 1059, 21, 0, 5, 20, 399, 77, 1483, 798, 0, 0, 0, 0, 0, 0, 16, 872, 1103, 1873, 31, 51, 225, 1, 0, 0, 0, 0, 0, 0, 0, 0, 78, 17, 51, 205, 1077, 1007, 1204, 774, 0, 0, 0, 0, 0, 0, 0, 0, 909, 387, 333, 749, 197, 342, 203, 948, 0, 0, 0, 0, 0, 0, 0, 0, 950, 664, 420, 22, 1034, 692, 11, 35, 0, 0, 0, 0, 0, 0, 0, 0, 729, 972, 582, 118, 659, 897, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 1138, 11, 350, 7, 7, 33, 1164, 10, 875, 597, 162, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 3865, 4172, 4413, 4068, 3828, 3970, 4354] avg=4096.0000 max=4413 max_rank=3 min=3828 min_rank=5 imbalance_ratio=1.077393 max_load_reduction=28355 max_load_reduction_pct=0.865326 moved_to_echo=16220 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[191, 129, 527, 5, 1, 689, 908, 139, 455, 285, 6, 0, 0, 0, 0, 0, 5, 755, 24, 2, 14, 44, 297, 101, 2295, 689, 0, 0, 0, 0, 0, 0, 6, 1080, 1118, 1625, 16, 298, 269, 2, 0, 0, 0, 0, 0, 0, 0, 0, 77, 2, 20, 212, 1078, 806, 1063, 519, 0, 0, 0, 0, 0, 0, 0, 0, 808, 383, 608, 1128, 38, 426, 563, 880, 0, 0, 0, 0, 0, 0, 0, 0, 1092, 340, 363, 27, 857, 831, 19, 88, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 592, 731, 32, 675, 1521, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 1301, 23, 256, 14, 20, 23, 551, 25, 1089, 516, 108, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3335, 4226, 4414, 3777, 4834, 3617, 4639, 3926] avg=4096.0000 max=4834 max_rank=4 min=3335 min_rank=0 imbalance_ratio=1.180176 max_load_reduction=27934 max_load_reduction_pct=0.852478 moved_to_echo=17016 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[241, 169, 786, 2, 1, 449, 1083, 286, 669, 291, 6, 0, 0, 0, 0, 0, 1, 934, 25, 0, 11, 11, 138, 156, 1765, 964, 0, 0, 0, 0, 0, 0, 1, 922, 1201, 1660, 28, 195, 262, 4, 0, 0, 0, 0, 0, 0, 0, 0, 64, 11, 118, 252, 963, 1017, 1040, 761, 0, 0, 0, 0, 0, 0, 0, 0, 534, 313, 590, 899, 98, 384, 299, 572, 0, 0, 0, 0, 0, 0, 0, 0, 1406, 646, 508, 34, 872, 806, 15, 161, 0, 0, 0, 0, 0, 0, 0, 0, 863, 436, 736, 4, 570, 1516, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 19, 301, 8, 4, 15, 729, 17, 983, 721, 159, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3983, 4005, 4273, 4226, 3689, 4448, 4152, 3992] avg=4096.0000 max=4448 max_rank=5 min=3689 min_rank=4 imbalance_ratio=1.085938 max_load_reduction=28320 max_load_reduction_pct=0.864258 moved_to_echo=16281 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[240, 185, 815, 1, 2, 691, 850, 231, 612, 293, 4, 0, 0, 0, 0, 0, 1, 547, 30, 0, 15, 37, 553, 152, 1947, 819, 0, 0, 0, 0, 0, 0, 2, 888, 660, 1675, 22, 170, 137, 2, 0, 0, 0, 0, 0, 0, 0, 0, 73, 27, 49, 236, 1075, 702, 1090, 695, 0, 0, 0, 0, 0, 0, 0, 0, 424, 365, 555, 1371, 77, 474, 571, 884, 0, 0, 0, 0, 0, 0, 0, 0, 1209, 566, 431, 41, 1031, 810, 36, 36, 0, 0, 0, 0, 0, 0, 0, 0, 770, 506, 620, 23, 803, 1208, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 1234, 11, 252, 5, 26, 11, 879, 19, 1211, 613, 146, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3924, 4101, 3556, 3947, 4721, 4160, 3952, 4407] avg=4096.0000 max=4721 max_rank=4 min=3556 min_rank=2 imbalance_ratio=1.152588 max_load_reduction=28047 max_load_reduction_pct=0.855927 moved_to_echo=17240 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[100, 190, 655, 4, 0, 758, 1217, 189, 723, 351, 5, 0, 0, 0, 0, 0, 0, 1403, 63, 0, 10, 6, 430, 111, 1642, 1075, 0, 0, 0, 0, 0, 0, 9, 1075, 770, 2002, 27, 53, 294, 1, 0, 0, 0, 0, 0, 0, 0, 0, 47, 23, 58, 202, 1027, 1046, 950, 822, 0, 0, 0, 0, 0, 0, 0, 0, 380, 270, 401, 664, 160, 406, 239, 771, 0, 0, 0, 0, 0, 0, 0, 0, 1089, 435, 565, 36, 894, 977, 9, 32, 0, 0, 0, 0, 0, 0, 0, 0, 599, 483, 784, 63, 617, 1397, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 1175, 20, 352, 13, 9, 19, 487, 11, 1039, 804, 172, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4192, 4740, 4231, 4175, 3291, 4037, 4001, 4101] avg=4096.0000 max=4740 max_rank=1 min=3291 min_rank=4 imbalance_ratio=1.157227 max_load_reduction=28028 max_load_reduction_pct=0.855347 moved_to_echo=15430 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[130, 553, 600, 0, 0, 672, 1209, 375, 623, 278, 6, 0, 0, 0, 0, 0, 0, 863, 30, 2, 36, 41, 165, 119, 1565, 847, 0, 0, 0, 0, 0, 0, 2, 893, 778, 1561, 30, 298, 275, 11, 0, 0, 0, 0, 0, 0, 0, 0, 93, 56, 93, 409, 1500, 611, 823, 697, 0, 0, 0, 0, 0, 0, 0, 0, 507, 435, 406, 1415, 102, 472, 834, 933, 0, 0, 0, 0, 0, 0, 0, 0, 840, 644, 437, 68, 1051, 795, 14, 192, 0, 0, 0, 0, 0, 0, 0, 0, 588, 690, 648, 84, 441, 822, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 917, 35, 203, 10, 10, 11, 806, 23, 1216, 634, 153, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4446, 3668, 3848, 4282, 5104, 4041, 3361, 4018] avg=4096.0000 max=5104 max_rank=4 min=3361 min_rank=6 imbalance_ratio=1.246094 max_load_reduction=27664 max_load_reduction_pct=0.844238 moved_to_echo=16524 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4736, 2688, 3840, 8064, 1024, 3456, 3200, 6272, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[1536, 1920, 5888, 128, 128, 5376, 8320, 2304, 5376, 2560, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[9344, 256, 2560, 128, 128, 256, 5632, 256, 8576, 5120, 1280, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 7936, 7168, 13952, 256, 1664, 2048, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[9856, 4096, 3584, 512, 7680, 6528, 256, 768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[6528, 4608, 5376, 512, 5120, 10624, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[768, 256, 640, 2048, 8960, 6784, 7808, 6016, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 7936, 256, 128, 256, 256, 2560, 1024, 14208, 6784, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[2, 0, 4, 0, 0, 2287, 0, 67, 4, 0, 3942, 1, 4, 0, 0, 0, 0, 0, 3997, 0, 15, 46, 3996, 0, 0, 174, 0, 0, 0, 0, 2538, 1, 0, 0, 0, 0, 1, 3074, 452, 0, 4011, 0, 11, 0, 0, 0, 36, 0, 8, 11, 124, 272, 1, 0, 0, 3, 71, 0, 337, 3376, 3551, 0, 351, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[2, 1, 49, 0, 0, 990, 1, 72, 7, 3, 4077, 0, 9, 0, 7, 0, 1, 0, 4020, 1, 81, 1, 4057, 1, 0, 1801, 0, 0, 51, 0, 3261, 0, 0, 0, 0, 0, 259, 3892, 345, 3, 4086, 8, 92, 1, 0, 0, 0, 0, 0, 0, 31, 0, 121, 0, 0, 1, 13, 0, 4, 2413, 2574, 0, 431, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[13, 0, 7, 0, 0, 164, 0, 24, 0, 0, 4009, 5, 0, 0, 148, 0, 0, 0, 3982, 1, 11, 10, 3793, 1, 0, 65, 3, 0, 0, 1, 3819, 0, 0, 0, 0, 0, 397, 2595, 1150, 16, 3850, 0, 203, 0, 0, 0, 2, 0, 17, 0, 1221, 77, 0, 0, 0, 7, 116, 1, 4, 3732, 3211, 0, 113, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[10, 0, 1, 0, 0, 2257, 0, 9, 1, 0, 3890, 0, 0, 0, 2, 0, 0, 0, 3238, 0, 686, 26, 4073, 0, 0, 13, 0, 1, 4, 28, 3603, 37, 0, 0, 0, 0, 10, 2683, 230, 1, 4012, 0, 66, 0, 0, 0, 1, 0, 108, 1, 120, 1, 0, 0, 0, 1, 3, 0, 877, 3429, 2617, 0, 729, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[14, 0, 0, 0, 0, 854, 0, 0, 0, 0, 3861, 46, 0, 23, 3, 0, 0, 4, 3824, 37, 94, 137, 2810, 0, 0, 1684, 0, 13, 0, 56, 3346, 99, 1, 0, 0, 0, 35, 1467, 1285, 0, 3697, 0, 138, 6, 9, 0, 1, 0, 38, 7, 576, 1, 4, 0, 0, 2, 112, 5, 1242, 3512, 2491, 0, 1229, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 0, 1161, 0, 0, 0, 0, 3960, 0, 0, 0, 42, 0, 0, 0, 4054, 20, 0, 111, 3142, 4, 0, 313, 0, 0, 6, 4, 2956, 0, 0, 0, 0, 0, 172, 3367, 324, 0, 4092, 0, 1304, 0, 0, 0, 0, 0, 0, 0, 0, 20, 29, 0, 8, 0, 158, 1, 344, 3990, 2377, 0, 806, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2054, 0, 2, 0, 9, 4044, 39, 0, 0, 10, 0, 0, 0, 3847, 40, 203, 133, 3997, 0, 0, 1113, 0, 0, 0, 12, 2856, 3, 0, 0, 0, 0, 3, 2969, 1036, 0, 4081, 0, 48, 0, 0, 0, 1, 0, 94, 0, 48, 1, 0, 0, 68, 0, 21, 6, 88, 2819, 678, 0, 2445, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[65, 0, 14, 0, 0, 2013, 0, 65, 0, 0, 3652, 23, 0, 2, 100, 0, 0, 0, 3616, 37, 133, 0, 991, 0, 0, 21, 0, 0, 1, 1, 3499, 7, 0, 0, 3, 0, 10, 3617, 676, 0, 4008, 0, 137, 0, 0, 0, 0, 20, 1, 6, 44, 2, 4, 1, 12, 0, 6, 0, 773, 3725, 2703, 1, 2779, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[13, 0, 7, 0, 0, 164, 0, 24, 2677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 5, 0, 0, 148, 0, 80, 40, 0, 0, 0, 0, 0, 0, 0, 0, 533, 1, 11, 10, 3793, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 3, 0, 0, 1, 3819, 0, 98, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 397, 2595, 1150, 16, 350, 0, 0, 0, 0, 0, 0, 0, 3712, 0, 203, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 1221, 77, 0, 0, 0, 7, 3732, 349, 0, 0, 0, 0, 0, 0, 116, 1, 4, 0, 3131, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2885, 4282, 4349, 4059, 4508, 3917, 5403, 3365] avg=4096.0000 max=5403 max_rank=6 min=2885 min_rank=0 imbalance_ratio=1.319092 max_load_reduction=27365 max_load_reduction_pct=0.835114 moved_to_echo=17193 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[14, 0, 0, 0, 0, 854, 0, 0, 2575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3861, 46, 0, 23, 3, 0, 67, 44, 0, 0, 0, 0, 0, 0, 0, 4, 496, 37, 94, 137, 2810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1684, 0, 13, 0, 56, 3346, 99, 98, 75, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 35, 1467, 1285, 0, 340, 0, 0, 0, 0, 0, 0, 0, 3555, 0, 138, 6, 9, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 7, 576, 1, 4, 0, 0, 2, 3512, 338, 0, 0, 0, 0, 0, 0, 112, 5, 1242, 0, 2424, 0, 1229, 5, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3443, 4044, 3578, 5371, 3128, 3709, 4478, 5017] avg=4096.0000 max=5371 max_rank=3 min=3128 min_rank=4 imbalance_ratio=1.311279 max_load_reduction=27397 max_load_reduction_pct=0.836090 moved_to_echo=16332 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 1, 0, 0, 2257, 0, 9, 2177, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3890, 0, 0, 0, 2, 0, 65, 42, 0, 0, 0, 0, 0, 0, 0, 0, 434, 0, 686, 26, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 1, 4, 28, 3603, 37, 102, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 2683, 230, 1, 285, 0, 0, 0, 0, 0, 0, 0, 3868, 0, 66, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 1, 120, 1, 0, 0, 0, 1, 3429, 283, 0, 0, 0, 0, 0, 0, 3, 0, 877, 0, 2552, 0, 729, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4454, 4000, 5219, 3847, 3209, 3935, 3943, 4161] avg=4096.0000 max=5219 max_rank=2 min=3209 min_rank=4 imbalance_ratio=1.274170 max_load_reduction=27549 max_load_reduction_pct=0.840729 moved_to_echo=15248 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[65, 0, 14, 0, 0, 2013, 0, 65, 2431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3652, 23, 0, 2, 100, 0, 67, 42, 0, 0, 0, 0, 0, 0, 0, 0, 484, 37, 133, 0, 991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 1, 1, 3499, 7, 102, 66, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 10, 3617, 676, 0, 318, 0, 0, 0, 0, 0, 0, 0, 3864, 0, 137, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 44, 2, 4, 1, 12, 0, 3725, 317, 0, 0, 0, 0, 0, 0, 6, 0, 773, 0, 2636, 1, 2779, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4588, 3886, 1645, 3697, 4624, 4021, 4112, 6195] avg=4096.0000 max=6195 max_rank=7 min=1645 min_rank=2 imbalance_ratio=1.512451 max_load_reduction=26573 max_load_reduction_pct=0.810944 moved_to_echo=18952 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2054, 0, 2, 2587, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4044, 39, 0, 0, 10, 0, 16, 43, 0, 0, 0, 0, 0, 0, 0, 0, 514, 40, 203, 133, 3997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1113, 0, 0, 0, 12, 2856, 3, 104, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2969, 1036, 0, 339, 0, 0, 0, 0, 0, 0, 0, 3934, 0, 48, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 48, 1, 0, 0, 68, 0, 2819, 337, 0, 0, 0, 0, 0, 0, 21, 6, 88, 0, 662, 0, 2445, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4643, 4161, 4887, 4158, 4347, 3983, 3367, 3222] avg=4096.0000 max=4887 max_rank=2 min=3222 min_rank=7 imbalance_ratio=1.193115 max_load_reduction=27881 max_load_reduction_pct=0.850861 moved_to_echo=14919 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 1, 49, 0, 0, 990, 1, 72, 2703, 0, 0, 0, 0, 0, 0, 0, 7, 3, 4077, 0, 9, 0, 7, 0, 64, 43, 0, 0, 0, 0, 0, 0, 1, 0, 537, 1, 81, 1, 4057, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1801, 0, 0, 51, 0, 3261, 0, 104, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 259, 3892, 345, 3, 354, 0, 0, 0, 0, 0, 0, 0, 3939, 8, 92, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 121, 0, 0, 1, 2413, 352, 0, 0, 0, 0, 0, 0, 13, 0, 4, 0, 2510, 0, 431, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3818, 4210, 4679, 5291, 4853, 4040, 2918, 2959] avg=4096.0000 max=5291 max_rank=3 min=2918 min_rank=6 imbalance_ratio=1.291748 max_load_reduction=27477 max_load_reduction_pct=0.838531 moved_to_echo=14770 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 4, 0, 0, 2287, 0, 67, 2687, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3942, 1, 4, 0, 0, 0, 88, 42, 0, 0, 0, 0, 0, 0, 0, 0, 535, 0, 15, 46, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174, 0, 0, 0, 0, 2538, 1, 102, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3074, 452, 0, 352, 0, 0, 0, 0, 0, 0, 0, 3867, 0, 11, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 11, 124, 272, 1, 0, 0, 3, 3376, 350, 0, 0, 0, 0, 0, 0, 71, 0, 337, 0, 3463, 0, 351, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5047, 4081, 4592, 2888, 3879, 3914, 4145, 4222] avg=4096.0000 max=5047 max_rank=0 min=2888 min_rank=3 imbalance_ratio=1.232178 max_load_reduction=27721 max_load_reduction_pct=0.845978 moved_to_echo=16160 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 0, 1161, 0, 0, 2726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3960, 0, 0, 0, 42, 0, 59, 43, 0, 0, 0, 0, 0, 0, 0, 0, 542, 20, 0, 111, 3142, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 313, 0, 0, 6, 4, 2956, 0, 104, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 3367, 324, 0, 357, 0, 0, 0, 0, 0, 0, 0, 3945, 0, 1304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 29, 0, 8, 0, 3990, 355, 0, 0, 0, 0, 0, 0, 158, 1, 344, 0, 2318, 0, 806, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3890, 4104, 3819, 3457, 4220, 5249, 4402, 3627] avg=4096.0000 max=5249 max_rank=5 min=3457 min_rank=3 imbalance_ratio=1.281494 max_load_reduction=27519 max_load_reduction_pct=0.839813 moved_to_echo=17498 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 0, 0, 11904, 128, 256, 20608, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[512, 128, 3712, 0, 19712, 128, 8960, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 0, 896, 23680, 5504, 128, 2816, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 4096, 256, 1280, 512, 26880, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[384, 128, 2176, 384, 256, 128, 128, 128, 27008, 2688, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 5248, 128, 128, 128, 128, 25984, 256, 896, 640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[30720, 128, 2048, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 31488, 128, 128, 128, 384, 0, 512, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4023, 0, 3151, 0, 6, 0, 0, 4018, 0, 4045, 0, 303, 860, 3, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 3919, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3952, 13, 0, 3389, 958, 0, 0, 2, 0, 0, 4078, 1, 17, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4085, 0, 4039, 0, 116, 0, 0, 4050, 0, 4086, 0, 2071, 535, 25, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 1135, 45, 0, 0, 10, 1, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 33, 3847, 1, 0, 3459, 1008, 0, 0, 2, 0, 0, 4036, 35, 76, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 3539, 0, 4090, 0, 95, 0, 0, 4085, 1, 3965, 1, 1986, 593, 0, 0, 1, 0, 10, 0, 0, 0, 0, 0, 7, 0, 0, 6, 0, 6, 1052, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 21, 0, 0, 1, 0, 9, 3674, 210, 0, 4009, 1360, 0, 0, 0, 0, 0, 4040, 0, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[2, 0, 47, 0, 4001, 0, 3944, 0, 2, 0, 0, 4089, 0, 4087, 0, 1588, 3786, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 3112, 5, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 2, 3944, 0, 15, 1639, 59, 0, 2, 7, 0, 1, 2389, 2, 22, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4092, 0, 4067, 0, 209, 0, 1, 4090, 0, 3937, 0, 244, 3599, 1, 71, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 18, 0, 0, 0, 1958, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 224, 155, 4092, 0, 0, 2473, 884, 0, 0, 0, 0, 0, 2634, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 3940, 1, 3341, 0, 145, 0, 0, 4091, 0, 4037, 0, 222, 1602, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 1, 345, 0, 1, 7, 0, 1872, 12, 0, 2, 0, 0, 3, 4, 0, 1, 0, 951, 1, 0, 0, 1, 0, 3929, 0, 0, 4079, 8, 0, 0, 0, 6, 0, 4088, 2, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4078, 0, 4092, 0, 0, 0, 3, 4080, 0, 4089, 0, 431, 1509, 0, 7, 54, 10, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 2217, 0, 0, 0, 0, 0, 0, 1, 0, 0, 18, 27, 2, 0, 0, 18, 5, 3920, 4, 0, 3263, 760, 0, 27, 0, 0, 0, 4067, 72, 9, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 0, 1, 4053, 0, 4095, 0, 4, 0, 5, 4086, 2, 3926, 0, 1001, 2474, 216, 5, 0, 0, 0, 0, 0, 1, 0, 0, 1, 67, 0, 0, 0, 3, 2721, 10, 0, 2, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 1, 4058, 157, 0, 2933, 73, 0, 0, 3, 0, 1, 2800, 0, 41, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 240, 0, 3151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 3060, 0, 303, 0, 0, 0, 0, 0, 0, 0, 0, 860, 3, 1, 0, 2, 0, 0, 0, 2179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 3974, 0, 0, 0, 0, 0, 0, 0, 0, 3919, 1, 0, 0, 0, 0, 0, 1399, 444, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3783, 176, 44, 0, 0, 0, 0, 0, 2, 0, 198, 13, 0, 3389, 958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4078, 1, 17, 0, 541, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3391, 3369, 3045, 3998, 5763, 4003, 4560, 4639] avg=4096.0000 max=5763 max_rank=4 min=3045 min_rank=2 imbalance_ratio=1.406982 max_load_reduction=27005 max_load_reduction_pct=0.824127 moved_to_echo=18965 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 243, 0, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 0, 0, 0, 0, 3091, 0, 2071, 0, 0, 0, 0, 0, 0, 0, 0, 535, 25, 1, 0, 3, 0, 0, 0, 2121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 4007, 0, 0, 0, 0, 0, 0, 0, 0, 1135, 45, 0, 0, 10, 1, 0, 1362, 449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 3842, 171, 43, 0, 0, 0, 0, 0, 0, 33, 193, 1, 0, 3459, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4036, 35, 76, 0, 546, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4283, 5278, 2685, 4044, 3002, 4087, 4694, 4695] avg=4096.0000 max=5278 max_rank=1 min=2685 min_rank=2 imbalance_ratio=1.288574 max_load_reduction=27490 max_load_reduction_pct=0.838928 moved_to_echo=16478 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 244, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 209, 0, 1, 0, 0, 2979, 0, 244, 0, 0, 0, 0, 0, 0, 0, 0, 3599, 1, 71, 1, 0, 0, 0, 0, 2256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 18, 0, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 1958, 0, 0, 0, 11, 0, 0, 1449, 432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 3848, 182, 44, 0, 0, 0, 0, 0, 224, 155, 205, 0, 0, 2473, 884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2634, 0, 0, 1, 526, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4312, 3433, 5928, 4065, 3850, 4078, 3941, 3161] avg=4096.0000 max=5928 max_rank=2 min=3161 min_rank=7 imbalance_ratio=1.447266 max_load_reduction=26840 max_load_reduction_pct=0.819092 moved_to_echo=15030 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 47, 0, 238, 0, 3944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3092, 0, 1588, 0, 0, 0, 0, 0, 0, 0, 0, 3786, 0, 0, 0, 3, 0, 0, 0, 2174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 3112, 5, 0, 0, 1, 0, 0, 1397, 449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3763, 176, 45, 0, 0, 0, 0, 0, 3, 2, 197, 0, 15, 1639, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7, 0, 1, 2389, 2, 22, 0, 546, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4231, 4682, 5963, 4057, 4964, 3987, 1915, 2969] avg=4096.0000 max=5963 max_rank=2 min=1915 min_rank=6 imbalance_ratio=1.455811 max_load_reduction=26805 max_load_reduction_pct=0.818024 moved_to_echo=13835 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 235, 1, 3341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 0, 0, 0, 3055, 0, 222, 0, 0, 0, 0, 0, 0, 0, 0, 1602, 0, 74, 0, 0, 0, 0, 0, 2166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 345, 0, 1, 7, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 1872, 12, 0, 2, 0, 0, 3, 1391, 443, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 951, 1, 0, 0, 3705, 175, 44, 0, 0, 0, 0, 0, 1, 0, 197, 0, 0, 4079, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4088, 2, 0, 0, 539, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3579, 3422, 3842, 4401, 3723, 4881, 4285, 4635] avg=4096.0000 max=4881 max_rank=5 min=3422 min_rank=1 imbalance_ratio=1.191650 max_load_reduction=27887 max_load_reduction_pct=0.851044 moved_to_echo=17524 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 243, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3094, 0, 431, 0, 0, 0, 0, 0, 0, 0, 0, 1509, 0, 7, 54, 10, 0, 0, 0, 2161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4036, 0, 0, 0, 0, 0, 0, 0, 2, 2217, 0, 0, 0, 0, 0, 0, 1388, 449, 0, 0, 0, 0, 0, 0, 1, 0, 0, 18, 27, 2, 0, 0, 3835, 175, 44, 0, 0, 0, 0, 0, 18, 5, 196, 4, 0, 3263, 760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 4067, 72, 9, 1, 546, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4335, 3528, 3741, 4038, 4056, 4102, 4246, 4722] avg=4096.0000 max=4722 max_rank=7 min=3528 min_rank=1 imbalance_ratio=1.152832 max_load_reduction=28046 max_load_reduction_pct=0.855896 moved_to_echo=17126 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 1, 241, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 5, 0, 2, 2970, 0, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 2474, 216, 5, 0, 0, 0, 0, 0, 2237, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 67, 0, 0, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 3, 2721, 10, 0, 2, 0, 0, 0, 1437, 431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 3812, 181, 44, 0, 0, 0, 0, 0, 0, 1, 203, 157, 0, 2933, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 2800, 0, 41, 0, 525, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4338, 3982, 4932, 4111, 4604, 4064, 3367, 3370] avg=4096.0000 max=4932 max_rank=2 min=3367 min_rank=6 imbalance_ratio=1.204102 max_load_reduction=27836 max_load_reduction_pct=0.849487 moved_to_echo=15405 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 207, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 1, 2991, 1, 1986, 0, 0, 0, 0, 0, 0, 0, 0, 593, 0, 0, 1, 0, 10, 0, 0, 2028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 6, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 6, 1052, 0, 0, 0, 0, 0, 2, 1306, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 1, 3332, 168, 44, 0, 0, 0, 0, 0, 0, 9, 172, 210, 0, 4009, 1360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4040, 0, 3, 0, 534, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4299, 5074, 2632, 4054, 2806, 3566, 5760, 4577] avg=4096.0000 max=5760 max_rank=6 min=2632 min_rank=2 imbalance_ratio=1.406250 max_load_reduction=27008 max_load_reduction_pct=0.824219 moved_to_echo=16709 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[14976, 256, 256, 128, 128, 128, 0, 0, 17408, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 18048, 128, 0, 128, 128, 128, 128, 11136, 3584, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 256, 1664, 512, 128, 25344, 5120, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[640, 0, 128, 0, 128, 24448, 128, 7936, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 28160, 128, 256, 128, 4352, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 1920, 128, 30848, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 0, 128, 512, 128, 128, 128, 32256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 0, 128, 128, 1152, 128, 128, 128, 29952, 1408, 384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 3613, 387, 0, 0, 3026, 0, 0, 0, 0, 4096, 1866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4088, 0, 0, 3898, 0, 0, 2, 0, 0, 1, 0, 3625, 0, 0, 0, 0, 0, 0, 154, 1, 0, 33, 0, 3866, 0, 0, 0, 0, 2, 7, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4087, 0, 0, 0, 1673, 51, 0, 0, 2371, 0, 42, 0, 0, 4096, 2315, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 1992, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 2, 0, 3478, 0, 0, 739, 0, 3687, 0, 0, 0, 0, 0, 40, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4089, 0, 0, 0, 157, 91, 0, 0, 4055, 0, 1, 0, 0, 4096, 3538, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 2421, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 5, 0, 1983, 0, 0, 243, 0, 3826, 0, 0, 0, 0, 0, 64, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 4091, 0, 4, 0, 1640, 15, 0, 0, 1686, 0, 0, 5, 0, 4096, 4065, 0, 0, 0, 2, 0, 0, 0, 0, 1, 6, 0, 4094, 0, 0, 3696, 0, 1, 0, 0, 1, 0, 0, 4092, 24, 0, 0, 0, 2, 2, 58, 0, 0, 1174, 3, 3986, 0, 0, 0, 0, 0, 20, 2, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1, 0, 4, 1, 0, 4095, 0, 0, 0, 3478, 556, 0, 0, 3777, 0, 0, 0, 0, 4084, 3992, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4093, 0, 0, 168, 0, 0, 2, 0, 0, 0, 0, 4053, 2, 0, 0, 0, 0, 0, 855, 31, 0, 22, 1, 3529, 0, 0, 0, 1, 2, 15, 4, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 4094, 0, 2, 0, 2153, 1436, 0, 0, 890, 0, 11, 0, 0, 4084, 2992, 0, 0, 0, 0, 0, 0, 6, 0, 0, 5, 0, 4070, 0, 0, 978, 0, 0, 1, 0, 0, 4, 0, 4095, 0, 0, 0, 0, 0, 0, 3967, 0, 0, 127, 0, 3845, 0, 0, 7, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 1, 0, 1, 0, 4094, 0, 49, 0, 2895, 1, 0, 0, 441, 1, 3, 0, 0, 4055, 1735, 0, 4, 0, 0, 1, 0, 1, 0, 0, 1, 0, 4059, 0, 9, 3573, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 11, 0, 3072, 0, 0, 979, 0, 3691, 0, 0, 0, 0, 1, 3, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4, 4096, 0, 0, 0, 3457, 42, 1, 0, 2218, 0, 55, 0, 0, 3983, 3602, 0, 10, 0, 0, 0, 0, 0, 16, 4, 15, 0, 4074, 0, 0, 2902, 2, 0, 0, 0, 0, 0, 1, 3944, 0, 0, 0, 0, 63, 0, 118, 17, 3, 56, 2, 4065, 1, 5, 0, 10, 0, 0, 2, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 51, 0, 0, 2371, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1087, 2315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1992, 0, 0, 0, 0, 0, 606, 751, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 2, 47, 1, 0, 0, 0, 0, 0, 0, 0, 3478, 0, 0, 739, 0, 1893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 3009, 1043, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 3484, 3402, 4100, 3349, 4149, 6110, 4092] avg=4096.0000 max=6110 max_rank=6 min=3349 min_rank=4 imbalance_ratio=1.491699 max_load_reduction=26658 max_load_reduction_pct=0.813538 moved_to_echo=17700 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 91, 0, 0, 4055, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1087, 3538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2421, 0, 0, 0, 0, 0, 56, 779, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 5, 4, 1, 0, 0, 0, 0, 0, 0, 0, 1983, 0, 0, 243, 0, 1964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 3009, 1083, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4244, 4625, 4100, 3256, 4113, 4190, 4156] avg=4096.0000 max=4625 max_rank=2 min=3256 min_rank=4 imbalance_ratio=1.129150 max_load_reduction=28143 max_load_reduction_pct=0.858856 moved_to_echo=15715 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2201, 387, 0, 0, 3026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1087, 1866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4088, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3898, 0, 0, 2, 0, 0, 1309, 787, 0, 0, 0, 0, 0, 0, 1, 0, 3625, 0, 0, 0, 0, 0, 103, 1, 0, 0, 0, 0, 0, 0, 0, 154, 1, 0, 33, 0, 1985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7, 0, 0, 0, 3009, 1094, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 5614, 2953, 4099, 5996, 3730, 2173, 4112] avg=4096.0000 max=5996 max_rank=4 min=2173 min_rank=6 imbalance_ratio=1.463867 max_load_reduction=26772 max_load_reduction_pct=0.817017 moved_to_echo=16011 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 4086, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 15, 0, 0, 1686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 1087, 4065, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 4094, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3696, 0, 1, 0, 0, 1, 594, 811, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 24, 0, 0, 0, 2, 46, 1, 0, 0, 0, 0, 0, 0, 2, 58, 0, 0, 1174, 3, 2047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 2, 0, 0, 3009, 1128, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 2701, 5159, 4105, 5103, 4165, 3284, 4159] avg=4096.0000 max=5159 max_rank=2 min=2701 min_rank=1 imbalance_ratio=1.259521 max_load_reduction=27609 max_load_reduction_pct=0.842560 moved_to_echo=16711 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 4, 1, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2119, 556, 0, 0, 3777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1083, 3992, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4093, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 0, 0, 2, 0, 0, 1260, 718, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 2, 0, 0, 0, 0, 99, 1, 0, 0, 0, 0, 0, 0, 0, 855, 31, 0, 22, 1, 1812, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 15, 4, 0, 0, 3001, 999, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 6452, 5076, 4098, 2148, 4155, 2721, 4022] avg=4096.0000 max=6452 max_rank=1 min=2148 min_rank=4 imbalance_ratio=1.575195 max_load_reduction=26316 max_load_reduction_pct=0.803101 moved_to_echo=13046 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 4089, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1312, 1436, 0, 0, 890, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1083, 2992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 5, 0, 4070, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 978, 0, 0, 1, 0, 0, 780, 783, 0, 0, 0, 0, 0, 0, 4, 0, 4095, 0, 0, 0, 0, 0, 61, 1, 0, 0, 0, 0, 0, 0, 0, 3967, 0, 0, 127, 0, 1974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 3001, 1088, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 3649, 4075, 4085, 2542, 4161, 6068, 4096] avg=4096.0000 max=6068 max_rank=6 min=2542 min_rank=4 imbalance_ratio=1.481445 max_load_reduction=26700 max_load_reduction_pct=0.814819 moved_to_echo=16867 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2098, 42, 1, 0, 2218, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 3602, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4, 15, 0, 4074, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2902, 2, 0, 0, 0, 0, 1256, 832, 0, 0, 0, 0, 0, 0, 0, 1, 3944, 0, 0, 0, 0, 63, 103, 3, 0, 0, 0, 0, 0, 0, 0, 118, 17, 3, 56, 2, 2079, 1, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 10, 0, 0, 2, 0, 0, 2932, 1154, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4414, 4663, 4115, 4992, 4114, 2276, 4103] avg=4096.0000 max=4992 max_rank=4 min=2276 min_rank=6 imbalance_ratio=1.218750 max_load_reduction=27776 max_load_reduction_pct=0.847656 moved_to_echo=15485 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 1, 0, 4089, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1764, 1, 0, 0, 441, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1076, 1735, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 4059, 4, 0, 0, 0, 0, 0, 0, 0, 0, 9, 3573, 0, 0, 0, 0, 0, 1049, 751, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 11, 82, 1, 0, 0, 0, 0, 0, 0, 0, 3072, 0, 0, 979, 0, 1895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 2979, 1045, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4140, 2210, 2815, 4066, 5382, 4181, 5946, 4028] avg=4096.0000 max=5946 max_rank=6 min=2210 min_rank=1 imbalance_ratio=1.451660 max_load_reduction=26822 max_load_reduction_pct=0.818542 moved_to_echo=19537 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 256, 128, 0, 0, 24064, 8704, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 13696, 128, 128, 3456, 128, 15744, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 19712, 128, 128, 128, 0, 128, 6912, 6272, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 11648, 2688, 128, 0, 18560, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 8704, 24192, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 32128, 128, 0, 0, 0, 128, 640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 128, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 171, 0, 0, 4059, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 47, 0, 0, 0, 0, 2, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4095, 3971, 0, 0, 0, 0, 0, 4093, 0, 3880, 0, 151, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 175, 0, 0, 4074, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4090, 1820, 0, 0, 0, 0, 0, 4054, 0, 4093, 0, 2167, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 3, 0, 4096, 4095, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 4022, 4057, 0, 0, 0, 0, 0, 4093, 12, 3895, 0, 28, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 15, 0, 0, 4044, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4082, 4096, 4, 4096, 3823, 0, 0, 0, 22, 0, 4096, 1, 4061, 0, 253, 0, 0, 0, 0, 0, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 0, 0, 3990, 4090, 8, 0, 0, 114, 0, 2, 0, 0, 0, 0, 1, 1, 0, 0, 2, 0, 3, 1, 0, 1, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 4084, 2916, 0, 0, 0, 0, 0, 4089, 0, 4082, 0, 1185, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 13, 0, 1, 11, 0, 0, 3990, 4092, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 13, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4084, 4071, 0, 3861, 2564, 3, 0, 0, 0, 0, 4076, 10, 4095, 0, 1739, 0, 1, 0, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 849, 0, 3, 4096, 4091, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 241, 0, 1, 0, 0, 2, 0, 0, 232, 1, 3, 1, 0, 0, 0, 0, 0, 0, 0, 4080, 4094, 0, 4095, 1181, 0, 0, 0, 0, 0, 4090, 52, 3981, 0, 1616, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 7, 2, 326, 6, 0, 4063, 4084, 5, 0, 30, 6, 0, 0, 0, 0, 0, 0, 2, 3, 0, 4, 0, 2, 0, 0, 15, 0, 0, 0, 8, 0, 0, 1, 1, 0, 0, 0, 3, 0, 35, 4076, 4058, 0, 4071, 1189, 0, 2, 0, 1, 0, 3862, 63, 4045, 2, 2794, 1, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 171, 0, 0, 3905, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 47, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 7, 4005, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1415, 3971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 3880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 0, 0, 0, 0, 0, 0, 0, 2680, 154, 27, 19, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4077, 4104, 4051, 4102, 5386, 3941, 3031] avg=4096.0000 max=5386 max_rank=5 min=3031 min_rank=7 imbalance_ratio=1.314941 max_load_reduction=27382 max_load_reduction_pct=0.835632 moved_to_echo=16460 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 175, 0, 0, 3919, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3967, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1414, 1820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2167, 0, 0, 0, 0, 0, 0, 0, 2676, 155, 27, 19, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4077, 4059, 4003, 4102, 3234, 4153, 5044] avg=4096.0000 max=5044 max_rank=7 min=3234 min_rank=5 imbalance_ratio=1.231445 max_load_reduction=27724 max_load_reduction_pct=0.846069 moved_to_echo=16533 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 3, 0, 3940, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, 4005, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1390, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 12, 3895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 2632, 156, 27, 19, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3946, 4078, 4056, 4308, 4102, 5448, 3968, 2862] avg=4096.0000 max=5448 max_rank=5 min=2862 min_rank=7 imbalance_ratio=1.330078 max_load_reduction=27320 max_load_reduction_pct=0.833740 moved_to_echo=16380 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 0, 0, 3838, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 8, 0, 0, 114, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 2, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 1, 0, 0, 0, 4002, 36, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1412, 2916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1185, 0, 0, 0, 0, 0, 0, 0, 2672, 152, 27, 19, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3844, 4195, 4055, 4043, 4106, 4328, 4142, 4055] avg=4096.0000 max=4328 max_rank=5 min=3844 min_rank=0 imbalance_ratio=1.056641 max_load_reduction=28440 max_load_reduction_pct=0.867920 moved_to_echo=16631 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 15, 0, 0, 3890, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 73, 4008, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 6, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 1416, 3823, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 61, 1, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 253, 0, 0, 0, 0, 0, 0, 2, 2680, 154, 27, 19, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3905, 4071, 4060, 4120, 4088, 5244, 4145, 3135] avg=4096.0000 max=5244 max_rank=5 min=3135 min_rank=7 imbalance_ratio=1.280273 max_load_reduction=27524 max_load_reduction_pct=0.839966 moved_to_echo=16612 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 849, 0, 3, 3940, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 241, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 232, 4003, 36, 0, 0, 0, 0, 0, 0, 1, 3, 1, 0, 0, 0, 0, 0, 4080, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1415, 1181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 52, 3981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1616, 0, 0, 0, 0, 0, 0, 0, 2680, 156, 27, 19, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4792, 4097, 4327, 4274, 4091, 2596, 4093, 4498] avg=4096.0000 max=4792 max_rank=0 min=2596 min_rank=5 imbalance_ratio=1.169922 max_load_reduction=27976 max_load_reduction_pct=0.853760 moved_to_echo=15278 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 13, 0, 1, 11, 0, 0, 3838, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 32, 0, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 88, 3989, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 6, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1334, 2564, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 10, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1739, 0, 1, 0, 0, 0, 0, 1, 2527, 152, 27, 19, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3863, 4073, 4081, 4125, 4090, 3905, 4165, 4466] avg=4096.0000 max=4466 max_rank=7 min=3863 min_rank=0 imbalance_ratio=1.090332 max_load_reduction=28302 max_load_reduction_pct=0.863708 moved_to_echo=16626 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 7, 2, 326, 6, 0, 3905, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 5, 0, 30, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 4, 0, 4017, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 15, 0, 0, 0, 8, 3784, 35, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 3, 4076, 6, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 1403, 1189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 48, 63, 4045, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2794, 1, 0, 0, 1, 0, 0, 0, 2668, 158, 30, 25, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4246, 4100, 4026, 3844, 4087, 2627, 4161, 5677] avg=4096.0000 max=5677 max_rank=7 min=2627 min_rank=5 imbalance_ratio=1.385986 max_load_reduction=27091 max_load_reduction_pct=0.826752 moved_to_echo=16552 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[9984, 128, 128, 0, 128, 0, 0, 128, 21248, 1280, 256, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 0, 0, 128, 11264, 21632, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 128, 0, 128, 384, 32384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 512, 256, 32256, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32640, 128, 128, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 1664, 128, 128, 31232, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 0, 0, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 0, 0, 768, 31872, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 621, 0, 8, 0, 0, 0, 0, 0, 4094, 0, 1, 0, 0, 0, 2308, 2188, 0, 0, 0, 368, 0, 4093, 4062, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 4, 0, 7, 0, 0, 4096, 0, 0, 0, 0, 0, 2695, 0, 0, 28, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 153, 0, 2, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 1498, 2738, 0, 0, 0, 0, 0, 4096, 3984, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 3, 2, 0, 2, 0, 0, 4095, 0, 0, 0, 0, 0, 3909, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 23, 341, 0, 53, 0, 9, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 1738, 1388, 38, 0, 0, 303, 0, 3244, 4086, 0, 0, 0, 6, 0, 0, 3957, 0, 0, 0, 2, 3, 3954, 0, 0, 0, 1, 0, 1782, 297, 0, 0, 11, 1, 4083, 0, 0, 1, 2, 1, 3359, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 358, 0, 0, 0, 1, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 1572, 816, 0, 0, 0, 483, 0, 3999, 4080, 0, 0, 0, 0, 0, 2, 4086, 0, 1, 0, 0, 3, 4096, 0, 2, 0, 0, 0, 714, 0, 4, 175, 0, 0, 4092, 0, 2, 0, 0, 0, 4096, 0, 0, 93, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 347, 0, 238, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 625, 1871, 1, 0, 0, 509, 0, 4065, 3536, 0, 0, 0, 1, 0, 0, 4092, 0, 0, 0, 0, 4, 3918, 0, 0, 1, 0, 0, 453, 19, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 3960, 0, 0, 942, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 53, 0, 34, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 50, 3897, 1, 0, 0, 0, 25, 4096, 4070, 0, 1, 0, 0, 0, 0, 4067, 0, 1, 0, 2, 9, 3982, 0, 0, 0, 0, 0, 2347, 4, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 1944, 0, 0, 6, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 316, 0, 1, 0, 0, 0, 0, 4, 4093, 0, 1, 0, 0, 0, 3035, 698, 21, 0, 0, 1, 0, 4061, 3969, 9, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 0, 474, 35, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 3814, 1, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 3, 0, 29, 0, 1918, 1, 0, 0, 2, 0, 4087, 6, 35, 0, 0, 0, 1493, 71, 14, 0, 0, 11, 169, 4037, 3988, 0, 0, 3, 0, 0, 0, 4013, 0, 8, 0, 0, 0, 4080, 0, 0, 44, 0, 0, 826, 0, 0, 11, 41, 2, 4035, 0, 0, 0, 4, 1, 3811, 0, 0, 6, 2, 0, 17, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 153, 0, 2, 0, 0, 3561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1498, 2738, 0, 0, 0, 0, 0, 606, 0, 0, 0, 0, 0, 0, 0, 4096, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 2, 2729, 477, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3909, 0, 0, 0, 0, 0, 0, 0, 512, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3716, 4096, 4842, 4230, 4152, 3213, 4098, 4421] avg=4096.0000 max=4842 max_rank=2 min=3213 min_rank=5 imbalance_ratio=1.182129 max_load_reduction=27926 max_load_reduction_pct=0.852234 moved_to_echo=15884 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 621, 0, 8, 0, 0, 3561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2308, 2188, 0, 0, 0, 368, 0, 618, 0, 0, 0, 0, 0, 0, 0, 4093, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 7, 2782, 477, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 2695, 0, 0, 28, 0, 0, 0, 0, 522, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4190, 4095, 5482, 4230, 4154, 3273, 4099, 3245] avg=4096.0000 max=5482 max_rank=2 min=3245 min_rank=7 imbalance_ratio=1.338379 max_load_reduction=27286 max_load_reduction_pct=0.832703 moved_to_echo=14771 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 23, 341, 0, 53, 0, 9, 3440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1738, 1388, 38, 0, 0, 303, 0, 622, 0, 0, 0, 0, 0, 0, 0, 3244, 137, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 2, 3, 3954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1782, 297, 0, 0, 2799, 461, 0, 0, 0, 0, 0, 0, 11, 1, 4083, 0, 0, 1, 2, 1, 3, 0, 0, 0, 0, 0, 0, 0, 3359, 0, 0, 0, 0, 0, 0, 0, 525, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3866, 4085, 4089, 3387, 4015, 5340, 4102, 3884] avg=4096.0000 max=5340 max_rank=5 min=3387 min_rank=3 imbalance_ratio=1.303711 max_load_reduction=27428 max_load_reduction_pct=0.837036 moved_to_echo=17341 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 358, 0, 0, 0, 1, 3552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1572, 816, 0, 0, 0, 483, 0, 621, 0, 0, 0, 0, 0, 0, 0, 3999, 137, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 1, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 714, 0, 4, 175, 2795, 476, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 2, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 93, 0, 0, 0, 1, 524, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3911, 4092, 3492, 4138, 4158, 4166, 4097, 4714] avg=4096.0000 max=4714 max_rank=7 min=3492 min_rank=2 imbalance_ratio=1.150879 max_load_reduction=28054 max_load_reduction_pct=0.856140 moved_to_echo=17135 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 347, 0, 238, 0, 0, 3558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 625, 1871, 1, 0, 0, 509, 0, 538, 0, 0, 0, 0, 0, 0, 0, 4065, 119, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 4, 3918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 453, 19, 0, 1, 2422, 476, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3960, 0, 0, 942, 0, 0, 0, 0, 454, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4143, 4089, 3544, 4185, 3980, 3372, 4099, 5356] avg=4096.0000 max=5356 max_rank=7 min=3372 min_rank=5 imbalance_ratio=1.307617 max_load_reduction=27412 max_load_reduction_pct=0.836548 moved_to_echo=16807 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 53, 0, 34, 0, 0, 3536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 3897, 1, 0, 0, 0, 25, 619, 0, 0, 0, 0, 0, 0, 0, 4096, 137, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 1, 0, 2, 9, 3982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2347, 4, 0, 0, 2788, 474, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1944, 0, 0, 6, 0, 0, 0, 0, 523, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3623, 4082, 4592, 4234, 4051, 5613, 4100, 2473] avg=4096.0000 max=5613 max_rank=5 min=2473 min_rank=7 imbalance_ratio=1.370361 max_load_reduction=27155 max_load_reduction_pct=0.828705 moved_to_echo=16237 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 316, 0, 1, 0, 0, 3558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4093, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3035, 698, 21, 0, 0, 1, 0, 604, 0, 0, 0, 0, 0, 0, 0, 4061, 133, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 35, 0, 0, 2719, 477, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3814, 1, 0, 0, 0, 0, 0, 0, 510, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3875, 4098, 4359, 4203, 4120, 3705, 4083, 4325] avg=4096.0000 max=4359 max_rank=2 min=3705 min_rank=5 imbalance_ratio=1.064209 max_load_reduction=28409 max_load_reduction_pct=0.866974 moved_to_echo=16233 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 29, 0, 1918, 1, 0, 3493, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4087, 6, 35, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1493, 71, 14, 0, 0, 11, 169, 610, 0, 0, 0, 0, 0, 0, 0, 4037, 121, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 8, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 826, 0, 0, 11, 2735, 470, 0, 0, 0, 0, 0, 0, 41, 2, 4035, 0, 0, 0, 4, 1, 7, 0, 0, 0, 0, 0, 0, 0, 3811, 0, 0, 6, 2, 0, 17, 0, 514, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5444, 4131, 2368, 4161, 4138, 4086, 4090, 4350] avg=4096.0000 max=5444 max_rank=0 min=2368 min_rank=2 imbalance_ratio=1.329102 max_load_reduction=27324 max_load_reduction_pct=0.833862 moved_to_echo=16664 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 12416, 13696, 128, 0, 0, 1792, 256, 4864, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[27648, 128, 0, 1152, 128, 0, 128, 128, 4096, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 6656, 384, 128, 256, 21888, 3840, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 2304, 0, 2304, 128, 128, 28288, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[512, 0, 128, 0, 128, 128, 32384, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[31744, 1152, 128, 128, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 32768, 0, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 32768, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1320, 0, 3, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 4096, 0, 0, 0, 2595, 0, 0, 0, 0, 4096, 0, 1390, 0, 0, 0, 612, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 3646, 2740, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[3308, 0, 2, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3994, 4096, 0, 0, 0, 2282, 0, 0, 0, 0, 4096, 0, 669, 0, 0, 0, 47, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3179, 2699, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[2946, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 3740, 4094, 0, 3, 0, 965, 0, 0, 0, 0, 4096, 0, 2728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3310, 2319, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[3372, 0, 11, 0, 4075, 4093, 0, 0, 0, 0, 0, 0, 90, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4021, 4089, 0, 0, 3, 239, 0, 0, 0, 0, 4096, 0, 919, 0, 0, 0, 1193, 0, 0, 0, 1, 2, 0, 0, 1, 3, 0, 4085, 2473, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[2915, 0, 63, 0, 4089, 4096, 0, 0, 0, 0, 0, 0, 134, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3711, 4096, 0, 0, 0, 1393, 0, 1, 0, 0, 4096, 0, 690, 6, 0, 0, 24, 0, 0, 0, 1, 20, 0, 0, 2, 0, 0, 3978, 3431, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[15, 2, 0, 1, 4089, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 2489, 4093, 0, 0, 2, 2992, 0, 9, 0, 0, 4091, 1, 3229, 0, 0, 0, 336, 0, 0, 0, 0, 0, 3, 2, 0, 0, 3, 3853, 3447, 0, 0, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[3297, 0, 234, 0, 4008, 4096, 0, 0, 0, 0, 0, 0, 1, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 4090, 1, 35, 0, 3655, 0, 1, 0, 0, 4095, 0, 508, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 0, 0, 0, 2960, 1673, 0, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[3434, 0, 0, 0, 4080, 4087, 0, 0, 5, 0, 0, 19, 5, 0, 0, 11, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2676, 4038, 0, 0, 0, 2779, 28, 0, 0, 0, 4082, 0, 2516, 0, 24, 0, 49, 6, 1, 0, 2, 0, 0, 0, 58, 0, 0, 3151, 1662, 1, 0, 0, 6, 0, 0, 36, 4, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3308, 0, 2, 0, 1488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 0, 0, 0, 2607, 1251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2198, 0, 0, 0, 0, 2282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2511, 0, 669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 4, 0, 0, 0, 1796, 1585, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1384, 2699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4798, 4058, 4095, 4480, 3180, 3976, 4083, 4098] avg=4096.0000 max=4798 max_rank=0 min=3180 min_rank=4 imbalance_ratio=1.171387 max_load_reduction=27970 max_load_reduction_pct=0.853577 moved_to_echo=15337 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1320, 0, 3, 0, 1488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 2607, 1435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2230, 0, 0, 0, 0, 2595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2511, 0, 1390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 612, 0, 5, 0, 0, 0, 1823, 1585, 624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1587, 2740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2811, 4062, 4095, 4825, 3901, 4649, 4327, 4098] avg=4096.0000 max=4825 max_rank=3 min=2811 min_rank=0 imbalance_ratio=1.177979 max_load_reduction=27943 max_load_reduction_pct=0.852753 moved_to_echo=16975 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2946, 0, 0, 0, 1488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 359, 0, 0, 0, 2607, 1303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2058, 0, 0, 3, 0, 965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2511, 0, 2728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1682, 1585, 567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1440, 2319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4434, 4269, 4111, 3026, 5239, 3834, 3759, 4096] avg=4096.0000 max=5239 max_rank=4 min=3026 min_rank=3 imbalance_ratio=1.279053 max_load_reduction=27529 max_load_reduction_pct=0.840118 moved_to_echo=16928 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3297, 0, 234, 0, 1456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 33, 0, 0, 2551, 1165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2242, 0, 1, 35, 0, 3655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2511, 0, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1832, 1584, 507, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1288, 1673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4090, 1, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4987, 3750, 4095, 5933, 3020, 3927, 2963, 4093] avg=4096.0000 max=5933 max_rank=3 min=2963 min_rank=6 imbalance_ratio=1.448486 max_load_reduction=26835 max_load_reduction_pct=0.818939 moved_to_echo=14003 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[15, 2, 0, 1, 1486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 10, 2602, 1517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1370, 0, 0, 0, 2, 2992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 2508, 1, 3229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 336, 0, 0, 0, 0, 0, 1119, 1583, 660, 0, 0, 0, 0, 0, 3, 2, 0, 0, 3, 1676, 3447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4093, 1, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1504, 4131, 4094, 4367, 5747, 3698, 5131, 4096] avg=4096.0000 max=5747 max_rank=4 min=1504 min_rank=0 imbalance_ratio=1.403076 max_load_reduction=27021 max_load_reduction_pct=0.824615 moved_to_echo=18672 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3372, 0, 11, 0, 1481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 2, 0, 0, 2593, 1608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2213, 0, 0, 0, 3, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2511, 0, 919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1193, 0, 0, 0, 1, 2, 1808, 1585, 699, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 1778, 2473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 1, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4864, 4293, 4092, 2455, 3430, 5288, 4255, 4091] avg=4096.0000 max=5288 max_rank=5 min=2455 min_rank=3 imbalance_ratio=1.291016 max_load_reduction=27480 max_load_reduction_pct=0.838623 moved_to_echo=17064 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2915, 0, 63, 0, 1486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 20, 0, 0, 2602, 1566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2042, 0, 0, 0, 0, 1393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2511, 0, 690, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 1, 20, 1669, 1585, 681, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1731, 3431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4464, 4322, 4097, 3435, 3208, 3980, 5164, 4098] avg=4096.0000 max=5164 max_rank=6 min=3208 min_rank=4 imbalance_ratio=1.260742 max_load_reduction=27604 max_load_reduction_pct=0.842407 moved_to_echo=16450 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3434, 0, 0, 0, 1472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 19, 5, 0, 0, 11, 2600, 1243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1468, 0, 0, 0, 0, 2779, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 2499, 0, 2516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 49, 6, 1, 0, 2, 0, 1208, 1583, 543, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 1365, 1662, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 36, 4, 6, 4038, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4906, 3883, 4089, 4247, 5043, 3416, 3086, 4098] avg=4096.0000 max=5043 max_rank=4 min=3086 min_rank=6 imbalance_ratio=1.231201 max_load_reduction=27725 max_load_reduction_pct=0.846100 moved_to_echo=15643 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 0, 2304, 128, 128, 0, 128, 128, 13056, 12800, 4864, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 0, 128, 896, 128, 0, 128, 20864, 11136, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 15872, 0, 128, 128, 128, 17024, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[20608, 128, 384, 128, 11904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 0, 20096, 128, 12672, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 12288, 20480, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 0, 128, 128, 128, 32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4092, 0, 0, 780, 0, 0, 0, 4076, 0, 0, 0, 66, 0, 0, 0, 0, 0, 6, 1, 0, 4096, 0, 0, 0, 4, 35, 0, 0, 0, 0, 1, 0, 1418, 2874, 197, 0, 0, 0, 0, 0, 0, 0, 0, 316, 274, 0, 0, 0, 0, 0, 0, 0, 2650, 0, 0, 4096, 0, 0, 0, 3700, 4086] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4062, 0, 0, 1984, 0, 0, 0, 3255, 0, 0, 1, 1, 1, 0, 0, 0, 0, 26, 0, 0, 4088, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 1204, 3686, 0, 7, 5, 0, 6, 0, 0, 0, 0, 471, 497, 0, 0, 0, 0, 0, 0, 0, 1390, 0, 1, 4095, 0, 4, 0, 3866, 4080] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 1, 0, 205, 0, 0, 0, 3974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2346, 1423, 0, 0, 0, 0, 0, 0, 0, 1, 0, 27, 2920, 0, 0, 0, 0, 0, 0, 0, 1438, 0, 0, 4096, 0, 0, 0, 4048, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 41, 0, 1600, 0, 0, 0, 3925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 551, 2261, 38, 0, 0, 0, 0, 23, 0, 0, 0, 518, 212, 0, 0, 0, 0, 0, 0, 0, 3383, 0, 0, 4096, 0, 0, 0, 3832, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 5, 0, 1513, 16, 0, 0, 4010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 701, 1429, 47, 0, 0, 0, 0, 0, 0, 0, 0, 44, 3805, 0, 0, 0, 0, 45, 0, 0, 847, 0, 0, 4096, 0, 0, 0, 3939, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 2, 0, 475, 0, 0, 0, 3665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1289, 3518, 211, 0, 0, 0, 0, 0, 0, 0, 0, 27, 472, 0, 0, 0, 0, 0, 0, 0, 2638, 0, 0, 4096, 0, 0, 0, 4087, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[2, 0, 3, 4044, 51, 2, 1889, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 2, 0, 13, 0, 0, 0, 0, 4094, 1, 1, 0, 0, 0, 0, 32, 0, 0, 0, 2, 1040, 1885, 33, 0, 0, 0, 0, 0, 0, 17, 0, 0, 727, 0, 4, 0, 2, 0, 0, 0, 2593, 0, 0, 4092, 2, 0, 0, 4062, 4085] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 5, 0, 4064, 18, 0, 508, 0, 0, 3, 202, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 10, 0, 4088, 0, 0, 0, 0, 0, 0, 12, 5, 0, 0, 0, 287, 3025, 0, 29, 0, 0, 0, 1165, 6, 0, 8, 738, 3542, 12, 0, 0, 0, 0, 0, 0, 2792, 1, 5, 4070, 0, 0, 0, 4086, 4085] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2970, 2, 0, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3665, 0, 0, 0, 0, 0, 689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1289, 3518, 211, 0, 0, 406, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 472, 2109, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2638, 1844, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 143, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3447, 4354, 4093, 4084, 5427, 2621, 4512, 4230] avg=4096.0000 max=5427 max_rank=4 min=2621 min_rank=5 imbalance_ratio=1.324951 max_load_reduction=27341 max_load_reduction_pct=0.834381 moved_to_echo=16790 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2970, 1, 0, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3974, 0, 0, 0, 0, 0, 689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2346, 1423, 0, 0, 0, 406, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 27, 2920, 2109, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1438, 1844, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 143, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3176, 4663, 4093, 4085, 4178, 5070, 3312, 4191] avg=4096.0000 max=5070 max_rank=5 min=3176 min_rank=0 imbalance_ratio=1.237793 max_load_reduction=27698 max_load_reduction_pct=0.845276 moved_to_echo=16751 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2970, 41, 0, 1600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3925, 0, 0, 0, 0, 0, 689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 2261, 38, 0, 0, 405, 3, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 518, 212, 2109, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3383, 1844, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3832, 143, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4611, 4614, 4093, 4085, 3258, 2875, 5257, 3975] avg=4096.0000 max=5257 max_rank=6 min=2875 min_rank=5 imbalance_ratio=1.283447 max_load_reduction=27511 max_load_reduction_pct=0.839569 moved_to_echo=15365 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2947, 0, 0, 1984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3255, 0, 0, 1, 1, 1, 684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1204, 3686, 0, 7, 5, 402, 2, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 471, 497, 2101, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1390, 1837, 29, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 0, 3866, 142, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4931, 3942, 4112, 4121, 5306, 3087, 3256, 4013] avg=4096.0000 max=5306 max_rank=4 min=3087 min_rank=5 imbalance_ratio=1.295410 max_load_reduction=27462 max_load_reduction_pct=0.838074 moved_to_echo=15662 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2968, 0, 0, 780, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 66, 0, 689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 35, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1418, 2874, 197, 0, 0, 405, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 316, 274, 2104, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2650, 1840, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3700, 142, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3748, 4831, 4100, 4122, 4898, 2707, 4520, 3842] avg=4096.0000 max=4898 max_rank=4 min=2707 min_rank=5 imbalance_ratio=1.195801 max_load_reduction=27870 max_load_reduction_pct=0.850525 moved_to_echo=15967 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 3, 2933, 51, 2, 1889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 681, 0, 0, 0, 0, 0, 0, 0, 2, 0, 13, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 32, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1040, 1885, 33, 0, 0, 401, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 727, 2103, 13, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2, 0, 0, 0, 2593, 1839, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4062, 143, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4880, 4771, 4106, 4113, 3364, 2860, 4467, 4207] avg=4096.0000 max=4880 max_rank=0 min=2860 min_rank=5 imbalance_ratio=1.191406 max_load_reduction=27888 max_load_reduction_pct=0.851074 moved_to_echo=14898 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2971, 5, 0, 1513, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4010, 0, 0, 0, 0, 0, 689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 701, 1429, 47, 0, 0, 406, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 3805, 2108, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 847, 1844, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3939, 143, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4505, 4699, 4078, 4083, 2585, 5970, 2766, 4082] avg=4096.0000 max=5970 max_rank=5 min=2585 min_rank=4 imbalance_ratio=1.457520 max_load_reduction=26798 max_load_reduction_pct=0.817810 moved_to_echo=15403 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 0, 2939, 18, 0, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 202, 0, 0, 0, 0, 2, 687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 5, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 287, 3025, 0, 29, 0, 406, 5, 0, 0, 0, 0, 0, 0, 0, 0, 1165, 6, 0, 8, 738, 3542, 2107, 12, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 2792, 1842, 32, 0, 0, 0, 0, 0, 0, 1, 5, 0, 0, 0, 0, 4086, 136, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3470, 894, 4093, 4075, 3752, 7578, 4678, 4228] avg=4096.0000 max=7578 max_rank=5 min=894 min_rank=1 imbalance_ratio=1.850098 max_load_reduction=25190 max_load_reduction_pct=0.768738 moved_to_echo=20236 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 1280, 128, 128, 128, 2176, 12544, 16896, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 128, 0, 0, 17792, 14848, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 23680, 128, 128, 8960, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 8960, 20224, 640, 128, 128, 3328, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 27264, 0, 0, 128, 128, 128, 5504, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 0, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 128, 0, 31744, 1152, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 128, 0, 128, 128, 32640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33792 slack_slots=31744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4074, 0, 0, 4096, 0, 0, 0, 0, 0, 1271, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0, 0, 1, 0, 0, 1672, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 2422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4064, 0, 0, 4075, 7, 32, 39, 0, 0, 20, 0, 0, 2, 20, 0, 0, 4095, 0, 0, 0, 4095, 0, 0, 4057, 2, 5, 0, 1, 0, 0, 9, 4045, 0, 4085, 0, 0, 0, 16, 1, 0, 78, 0, 0, 0, 0, 0, 4008, 0, 0, 4, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 3982, 0, 1, 4092, 0, 0, 0, 0, 0, 3616, 0, 0, 0, 165, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4089, 0, 0, 0, 0, 0, 2, 0, 17, 0, 0, 0, 0, 417, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 20, 3, 4094, 4, 1, 4092, 0, 8, 0, 0, 1, 1111, 0, 0, 0, 0, 1, 0, 4086, 0, 0, 0, 4093, 0, 1, 4093, 0, 0, 4, 0, 1, 0, 0, 4070, 0, 4091, 3, 0, 0, 0, 0, 0, 74, 8, 0, 0, 0, 0, 2905, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 3843, 0, 0, 4096, 0, 0, 0, 0, 0, 904, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3773, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 3746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4056, 0, 0, 4070, 1, 0, 5, 0, 0, 384, 0, 0, 0, 56, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 4085, 0, 4092, 0, 0, 14, 8, 0, 0, 0, 0, 0, 0, 0, 0, 3705, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 3910, 0, 0, 4080, 0, 0, 0, 0, 7, 17, 0, 14, 0, 49, 2, 0, 4083, 0, 0, 0, 4045, 0, 6, 4088, 0, 38, 13, 0, 4, 0, 0, 4052, 41, 4033, 0, 7, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 4, 0, 0, 0, 1, 8, 0, 0, 0, 14, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1672, 0, 0, 0, 2964, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4073, 7, 1, 0, 0, 0, 0, 0, 0, 2422, 0, 0, 0, 0, 0, 0, 1047, 30, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4174, 4637, 4068, 4093, 4114, 4085, 3502, 4095] avg=4096.0000 max=4637 max_rank=1 min=3502 min_rank=6 imbalance_ratio=1.132080 max_load_reduction=28131 max_load_reduction_pct=0.858490 moved_to_echo=15796 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4074, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1271, 0, 0, 0, 2964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4074, 7, 1, 0, 0, 0, 0, 0, 0, 2848, 0, 0, 0, 0, 0, 0, 1047, 30, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4152, 4235, 4066, 4093, 4116, 4083, 3928, 4095] avg=4096.0000 max=4235 max_rank=1 min=3928 min_rank=6 imbalance_ratio=1.033936 max_load_reduction=28533 max_load_reduction_pct=0.870758 moved_to_echo=16222 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4064, 0, 0, 78, 7, 0, 0, 0, 0, 0, 0, 0, 0, 32, 39, 0, 0, 20, 0, 0, 2, 2949, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 2, 5, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 21, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 1, 0, 78, 0, 0, 0, 0, 4024, 7, 1, 0, 0, 0, 0, 0, 0, 4008, 0, 0, 4, 0, 0, 0, 1041, 30, 3, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4149, 3042, 4085, 4062, 4115, 4127, 5086, 4102] avg=4096.0000 max=5086 max_rank=6 min=3042 min_rank=1 imbalance_ratio=1.241699 max_load_reduction=27682 max_load_reduction_pct=0.844788 moved_to_echo=17430 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 20, 3, 4094, 4, 1, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 1, 1111, 0, 0, 0, 2961, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4090, 0, 0, 4, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 4091, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 8, 0, 0, 0, 4048, 7, 1, 0, 0, 0, 0, 0, 0, 2905, 0, 0, 0, 1, 0, 0, 1046, 30, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4200, 4081, 4064, 4096, 4116, 4138, 3985, 4088] avg=4096.0000 max=4200 max_rank=0 min=3985 min_rank=6 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16327 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3843, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 904, 0, 0, 0, 2964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 3773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 4072, 7, 1, 0, 0, 0, 0, 0, 0, 3746, 0, 0, 0, 0, 0, 0, 1047, 30, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3921, 3868, 4066, 4093, 3795, 4104, 4826, 4095] avg=4096.0000 max=4826 max_rank=6 min=3795 min_rank=4 imbalance_ratio=1.178223 max_load_reduction=27942 max_load_reduction_pct=0.852722 moved_to_echo=16820 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3982, 0, 1, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3616, 0, 0, 0, 2961, 0, 0, 0, 0, 0, 0, 0, 165, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 17, 0, 0, 0, 4074, 7, 1, 0, 0, 0, 0, 0, 0, 417, 0, 0, 0, 0, 0, 3, 1046, 30, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4061, 6577, 4231, 4093, 4111, 4101, 1499, 4095] avg=4096.0000 max=6577 max_rank=1 min=1499 min_rank=6 imbalance_ratio=1.605713 max_load_reduction=26191 max_load_reduction_pct=0.799286 moved_to_echo=13806 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4056, 0, 0, 78, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 384, 0, 0, 0, 2945, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 4092, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 4063, 7, 1, 0, 0, 0, 0, 0, 0, 3705, 0, 0, 0, 0, 2, 0, 1040, 30, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4135, 3334, 4122, 4096, 4128, 4079, 4780, 4094] avg=4096.0000 max=4780 max_rank=6 min=3334 min_rank=1 imbalance_ratio=1.166992 max_load_reduction=27988 max_load_reduction_pct=0.854126 moved_to_echo=17081 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3910, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 17, 0, 14, 0, 2956, 0, 0, 0, 0, 0, 0, 0, 49, 2, 0, 0, 0, 0, 0, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4081, 0, 38, 13, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 41, 4033, 0, 7, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 13, 2, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 4, 0, 0, 1045, 30, 7, 0, 0, 0, 0, 0, 0, 1, 8, 0, 0, 0, 14, 0, 4081, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3976, 2994, 4066, 4142, 4273, 4051, 5162, 4104] avg=4096.0000 max=5162 max_rank=6 min=2994 min_rank=1 imbalance_ratio=1.260254 max_load_reduction=27606 max_load_reduction_pct=0.842468 moved_to_echo=17590 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 24192, 0, 0, 128, 128, 128, 128, 8448, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 9088, 0, 128, 128, 23680, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[384, 128, 0, 0, 0, 0, 0, 32512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 256, 128, 32384, 128, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 0, 0, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 256, 128, 0, 0, 0, 32512, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 32128, 128, 128, 640, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 32768, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 792, 0, 0, 0, 0, 4096, 0, 0, 394, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 4096, 0, 0, 0, 0, 2938, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4093, 6, 0, 182, 0, 0, 0, 0, 4096, 0, 0, 350, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4050, 0, 0, 0, 0, 3582, 0, 27, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 125, 0, 0, 0, 1, 4091, 0, 0, 87, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4001, 0, 4094, 0, 0, 0, 0, 3979, 0, 0, 0, 0, 0, 0, 9, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 4, 0, 0, 0, 0, 0, 37, 2, 0, 4093, 0, 0, 4092, 0, 2, 418, 0, 0, 0, 0, 4091, 0, 4, 568, 0, 1, 0, 0, 4047, 5, 0, 0, 0, 0, 0, 2, 0, 4003, 0, 4086, 0, 0, 0, 0, 3236, 0, 0, 1, 2, 0, 0, 1, 4061, 0, 0, 0, 1, 2, 2, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4080, 0, 0, 82, 27, 0, 0, 0, 4092, 0, 0, 34, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 4062, 1, 0, 0, 0, 4000, 0, 3, 0, 0, 0, 0, 13, 4091, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 102, 0, 0, 0, 0, 4096, 0, 0, 85, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 4084, 0, 0, 0, 0, 3975, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 6, 0, 49, 13, 0, 0, 0, 4096, 0, 0, 627, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 4074, 0, 0, 0, 0, 3436, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 4044, 0, 3, 4079, 0, 7, 26, 0, 0, 0, 0, 4068, 50, 4, 5, 4, 0, 0, 0, 3974, 18, 0, 0, 0, 0, 0, 0, 110, 4083, 0, 4082, 0, 8, 0, 0, 4079, 1, 6, 23, 9, 0, 0, 0, 4017, 0, 6, 0, 0, 13, 9, 12] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4073, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3872, 0, 0, 792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3825, 0, 0, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2938, 0, 0, 0, 0, 0, 0, 0, 233, 197, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 7, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4095, 4664, 4219, 4112, 4095, 3368, 4104] avg=4096.0000 max=4664 max_rank=2 min=3368 min_rank=6 imbalance_ratio=1.138672 max_load_reduction=28104 max_load_reduction_pct=0.857666 moved_to_echo=15679 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4027, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3870, 6, 0, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3825, 0, 0, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3582, 0, 27, 0, 0, 0, 0, 2, 233, 196, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 7, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4065, 4095, 4058, 4175, 4116, 4115, 4040, 4104] avg=4096.0000 max=4175 max_rank=3 min=4040 min_rank=6 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4061, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3872, 0, 0, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3825, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3975, 0, 0, 0, 0, 0, 0, 2, 233, 197, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 7, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4095, 3974, 3910, 4109, 4070, 4407, 4104] avg=4096.0000 max=4407 max_rank=6 min=3910 min_rank=3 imbalance_ratio=1.075928 max_load_reduction=28361 max_load_reduction_pct=0.865509 moved_to_echo=16690 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 4, 0, 0, 4039, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3857, 0, 0, 82, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3821, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 23, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4000, 0, 3, 0, 0, 0, 0, 13, 233, 196, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 7, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4094, 3966, 3855, 4116, 4111, 4445, 4099] avg=4096.0000 max=4445 max_rank=6 min=3855 min_rank=3 imbalance_ratio=1.085205 max_load_reduction=28323 max_load_reduction_pct=0.864349 moved_to_echo=16771 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4071, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3872, 0, 0, 125, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3820, 0, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 4001, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3979, 0, 0, 0, 0, 0, 0, 9, 233, 197, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 7, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4095, 3998, 3907, 4113, 4024, 4418, 4104] avg=4096.0000 max=4418 max_rank=6 min=3907 min_rank=3 imbalance_ratio=1.078613 max_load_reduction=28350 max_load_reduction_pct=0.865173 moved_to_echo=16659 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 4, 0, 0, 0, 4063, 38, 0, 0, 0, 0, 0, 0, 0, 0, 37, 2, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3869, 0, 2, 418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3820, 0, 4, 568, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 5, 0, 0, 0, 0, 0, 2, 20, 0, 0, 0, 0, 0, 0, 0, 0, 4003, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3236, 0, 0, 1, 2, 0, 0, 1, 233, 196, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 0, 1, 2, 2, 3, 7, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4131, 4289, 4393, 4074, 4026, 3669, 4077] avg=4096.0000 max=4393 max_rank=3 min=3669 min_rank=6 imbalance_ratio=1.072510 max_load_reduction=28375 max_load_reduction_pct=0.865936 moved_to_echo=15846 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 4051, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3871, 6, 0, 49, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3825, 0, 0, 627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3436, 0, 5, 0, 0, 0, 0, 0, 233, 196, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 7, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4095, 3939, 4452, 4111, 4107, 3870, 4104] avg=4096.0000 max=4452 max_rank=3 min=3870 min_rank=6 imbalance_ratio=1.086914 max_load_reduction=28316 max_load_reduction_pct=0.864136 moved_to_echo=16192 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4063, 40, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 4037, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3847, 0, 7, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3794, 50, 4, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3974, 18, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 110, 4083, 0, 19, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 1, 6, 23, 9, 0, 0, 0, 234, 199, 0, 0, 0, 0, 0, 0, 4017, 0, 6, 0, 0, 13, 9, 12, 8, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4068, 3880, 3857, 4017, 4220, 4551, 4072] avg=4096.0000 max=4551 max_rank=6 min=3857 min_rank=3 imbalance_ratio=1.111084 max_load_reduction=28217 max_load_reduction_pct=0.861115 moved_to_echo=16860 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[30592, 128, 128, 2176, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32512, 0, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[29312, 128, 128, 128, 128, 0, 0, 128, 1920, 1664, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[30976, 128, 128, 1792, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 128, 128, 128, 0, 32512, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[32768, 0, 128, 0, 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32640, 128, 0, 0, 0, 0, 0, 128, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 0, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 4070, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 343, 3, 0, 0, 3753, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 26, 0, 0, 4080, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 3, 0, 0, 0, 4080, 0, 4075, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 166, 0, 0, 0, 3925, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 4082, 4096, 4096, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 4087, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 3993, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 63, 1, 0, 0, 4082, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4096, 4096, 4048, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 4095, 0, 0, 0, 4095, 0, 0, 0, 0, 10, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 4093, 0, 0, 5, 4086, 0, 0, 4, 10, 0, 3, 3, 1, 0, 0, 10, 0, 26, 0, 4096, 4066, 4090, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 4088, 0, 4085, 0, 0, 0, 4083, 0, 0, 0, 6, 3, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 9, 0, 0, 0, 4057, 0, 18, 51, 4058, 0, 0, 0, 0, 0, 11, 0, 0, 0, 4, 11, 0, 0, 0, 3953, 4083, 4056, 0, 8, 0, 0, 19, 1, 0, 0, 55, 0, 5, 1, 0, 0, 8, 12, 11, 40, 0, 0, 2, 4045, 0, 4078, 0, 0, 0, 4051, 0, 0, 3, 43, 0, 33, 10, 0, 32] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 5, 5, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4064, 16, 0, 0, 0, 0, 0, 0, 4096, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 14, 0, 0, 0, 0, 0, 0, 4096, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4091, 4083, 4115, 4091, 4086, 4111, 4085] avg=4096.0000 max=4115 max_rank=3 min=4083 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=28653 max_load_reduction_pct=0.874420 moved_to_echo=16373 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 4070, 0, 0, 5, 5, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 4067, 16, 0, 0, 0, 0, 0, 0, 4096, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 14, 0, 0, 0, 0, 0, 0, 4096, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4091, 4107, 4115, 4090, 4086, 4111, 4086] avg=4096.0000 max=4115 max_rank=3 min=4082 min_rank=0 imbalance_ratio=1.004639 max_load_reduction=28653 max_load_reduction_pct=0.874420 moved_to_echo=16373 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 3993, 0, 0, 5, 5, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4067, 16, 0, 0, 0, 0, 0, 0, 4096, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 4072, 14, 0, 0, 0, 0, 0, 0, 4096, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4080, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4005, 4091, 4085, 4115, 4091, 4181, 4111, 4089] avg=4096.0000 max=4181 max_rank=5 min=4005 min_rank=0 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16472 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 166, 0, 0, 0, 3925, 0, 0, 5, 5, 0, 0, 0, 0, 0, 0, 0, 4091, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 4058, 16, 0, 0, 0, 0, 0, 0, 4082, 0, 19, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 14, 0, 0, 0, 0, 0, 0, 4070, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4080, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4098, 4118, 4103, 4091, 4086, 4085, 4086] avg=4096.0000 max=4118 max_rank=2 min=4085 min_rank=6 imbalance_ratio=1.005371 max_load_reduction=28650 max_load_reduction_pct=0.874329 moved_to_echo=16348 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 4093, 0, 0, 5, 5, 0, 0, 0, 0, 0, 0, 5, 4081, 0, 0, 4, 10, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 10, 0, 26, 0, 4056, 15, 0, 0, 0, 0, 0, 0, 4096, 0, 19, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 4066, 14, 0, 0, 0, 0, 0, 0, 4088, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 0, 1, 0, 0, 4068, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4103, 4111, 4116, 4062, 4085, 4103, 4083] avg=4096.0000 max=4116 max_rank=3 min=4062 min_rank=4 imbalance_ratio=1.004883 max_load_reduction=28652 max_load_reduction_pct=0.874390 moved_to_echo=16333 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 63, 1, 0, 0, 4082, 0, 0, 5, 5, 0, 0, 0, 0, 0, 0, 1, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4066, 16, 0, 0, 0, 0, 0, 0, 4096, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 14, 0, 0, 0, 0, 0, 0, 4079, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 4079, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4156, 4092, 4087, 4115, 4092, 4038, 4094, 4094] avg=4096.0000 max=4156 max_rank=0 min=4038 min_rank=5 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16318 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 343, 3, 0, 0, 3753, 0, 0, 5, 5, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 26, 0, 0, 4046, 16, 0, 0, 0, 0, 0, 0, 4080, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4066, 14, 0, 0, 0, 0, 0, 0, 4080, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4091, 4096, 4099, 4110, 4083, 4095, 4085] avg=4096.0000 max=4110 max_rank=4 min=4083 min_rank=5 imbalance_ratio=1.003418 max_load_reduction=28658 max_load_reduction_pct=0.874573 moved_to_echo=16373 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 9, 0, 0, 0, 4057, 0, 18, 11, 9, 0, 0, 0, 0, 0, 0, 51, 4049, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 11, 0, 0, 0, 4050, 16, 0, 0, 0, 0, 0, 0, 3953, 0, 10, 0, 8, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 55, 0, 5, 1, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 8, 12, 11, 40, 0, 0, 2, 4035, 15, 0, 0, 0, 0, 0, 0, 4045, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 43, 0, 33, 10, 0, 32, 4035, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4111, 4081, 3990, 4141, 4123, 4058, 4160] avg=4096.0000 max=4160 max_rank=7 min=3990 min_rank=3 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16482 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 128, 128, 128, 0, 32512, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 128, 128, 0, 128, 32640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 256, 0, 0, 128, 32512, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 640, 128, 0, 0, 32128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32640, 0, 256, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 0, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 4096, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3947, 0, 0, 0, 0, 3574, 0, 0, 0, 1, 0, 8, 292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 376, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 4056, 0, 4096, 4093, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 8, 0, 0, 4084, 3982, 0, 0, 0, 0, 3967, 0, 0, 0, 29, 0, 41, 225, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3760, 0, 0, 0, 0, 4032, 0, 0, 0, 4, 0, 122, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 197, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4057, 0, 4091, 4092, 0, 0, 4091, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4093, 3983, 2, 0, 0, 0, 3798, 0, 0, 3, 0, 22, 5, 79, 1, 0, 2, 0, 0, 0, 0, 0, 7, 6, 0, 4091, 1, 3, 0, 0, 3, 0, 2, 1, 0, 137, 1, 0, 0, 0, 4, 0, 191, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 4096, 3494, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3980, 0, 0, 0, 0, 3107, 0, 2, 0, 0, 0, 722, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[86, 0, 0, 0, 4095, 0, 4096, 3853, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4008, 0, 0, 0, 0, 3473, 0, 0, 0, 0, 0, 96, 311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4037, 0, 2, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 503, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4040, 0, 4080, 3959, 0, 0, 4079, 16, 5, 11, 0, 0, 0, 128, 9, 0, 4083, 3481, 37, 0, 0, 0, 3454, 0, 0, 9, 0, 0, 1135, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 8, 0, 4088, 9, 52, 3, 6, 14, 0, 16, 0, 0, 2, 0, 9, 0, 11, 0, 9, 0, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 4096, 0, 1, 0, 1, 0, 0, 19, 0, 0, 4095, 3854, 0, 0, 0, 0, 3977, 0, 0, 0, 1, 0, 275, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 63, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 115, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 198, 3947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3574, 0, 0, 0, 1, 0, 414, 0, 0, 0, 0, 0, 0, 0, 8, 292, 0, 0, 0, 0, 0, 0, 3481, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 376, 0, 3858, 23, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4206, 4095, 4145, 3989, 3904, 4099, 4073, 4257] avg=4096.0000 max=4257 max_rank=7 min=3904 min_rank=4 imbalance_ratio=1.039307 max_load_reduction=28511 max_load_reduction_pct=0.870087 moved_to_echo=16333 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 115, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 198, 3760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 0, 0, 0, 4, 0, 414, 0, 0, 0, 0, 0, 0, 0, 122, 80, 0, 0, 0, 0, 0, 0, 3481, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 197, 0, 3855, 23, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4211, 4095, 3958, 4450, 3806, 4100, 4073, 4075] avg=4096.0000 max=4450 max_rank=3 min=3806 min_rank=4 imbalance_ratio=1.086426 max_load_reduction=28318 max_load_reduction_pct=0.864197 moved_to_echo=16054 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 115, 0, 0, 3494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 198, 3980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3107, 0, 2, 0, 0, 0, 414, 0, 0, 0, 0, 0, 0, 0, 722, 554, 0, 0, 0, 0, 0, 0, 3481, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 0, 3858, 23, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3609, 4095, 4178, 3523, 4880, 4055, 4073, 4355] avg=4096.0000 max=4880 max_rank=4 min=3523 min_rank=3 imbalance_ratio=1.191406 max_load_reduction=27888 max_load_reduction_pct=0.851074 moved_to_echo=17363 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[86, 0, 0, 0, 115, 0, 0, 3853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 198, 4008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3473, 0, 0, 0, 0, 0, 413, 0, 0, 0, 0, 0, 0, 0, 96, 311, 0, 0, 0, 0, 0, 0, 3473, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4037, 0, 2, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 0, 3857, 22, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4054, 4095, 4206, 3886, 4003, 4052, 4090, 4382] avg=4096.0000 max=4382 max_rank=7 min=3886 min_rank=3 imbalance_ratio=1.069824 max_load_reduction=28386 max_load_reduction_pct=0.866272 moved_to_echo=16527 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 114, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 197, 3983, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3798, 0, 0, 3, 0, 22, 414, 0, 0, 0, 0, 0, 0, 0, 5, 79, 1, 0, 2, 0, 0, 0, 3479, 122, 0, 0, 0, 0, 0, 0, 0, 0, 7, 6, 0, 4091, 1, 3, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 1, 0, 137, 4068, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 191, 0, 3821, 23, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4206, 4092, 4182, 4237, 3688, 4112, 4211, 4040] avg=4096.0000 max=4237 max_rank=3 min=3688 min_rank=4 imbalance_ratio=1.034424 max_load_reduction=28531 max_load_reduction_pct=0.870697 moved_to_echo=16051 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 113, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 198, 3982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3967, 0, 0, 0, 29, 0, 413, 0, 0, 0, 0, 0, 0, 0, 41, 225, 0, 0, 0, 0, 0, 0, 3471, 122, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4079, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 3821, 22, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4208, 4096, 4188, 4409, 3859, 4083, 4074, 3851] avg=4096.0000 max=4409 max_rank=3 min=3851 min_rank=7 imbalance_ratio=1.076416 max_load_reduction=28359 max_load_reduction_pct=0.865448 moved_to_echo=15867 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 115, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 198, 3854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3977, 0, 0, 0, 1, 0, 414, 0, 0, 0, 0, 0, 0, 0, 275, 5, 0, 0, 0, 0, 0, 0, 3480, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 3857, 22, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4211, 4097, 4071, 4392, 3883, 4097, 4075, 3942] avg=4096.0000 max=4392 max_rank=3 min=3883 min_rank=4 imbalance_ratio=1.072266 max_load_reduction=28376 max_load_reduction_pct=0.865967 moved_to_echo=15997 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 104, 0, 0, 3959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 16, 5, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 9, 0, 185, 3481, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3454, 0, 0, 9, 0, 0, 419, 0, 0, 0, 0, 0, 0, 0, 1135, 0, 0, 0, 0, 0, 0, 11, 3474, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4088, 9, 52, 5, 8, 0, 0, 0, 0, 0, 0, 3, 6, 14, 0, 16, 0, 0, 2, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 11, 0, 9, 0, 4, 3811, 22, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4103, 3840, 3882, 4745, 4170, 4099, 3866] avg=4096.0000 max=4745 max_rank=4 min=3840 min_rank=2 imbalance_ratio=1.158447 max_load_reduction=28023 max_load_reduction_pct=0.855194 moved_to_echo=16880 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[2432, 1664, 128, 0, 128, 0, 0, 128, 27904, 1024, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 128, 128, 128, 1920, 128, 30848, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 128, 128, 0, 256, 32640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 0, 1024, 0, 0, 31872, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 256, 128, 0, 1664, 31104, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 0, 32640, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 29440, 0, 128, 128, 128, 128, 3328, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33792 slack_slots=31744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 283, 0, 94, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 1, 0, 0, 3866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 2, 0, 3953, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 180, 0, 291, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 3978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4019, 0, 2, 0, 3842, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[12, 149, 0, 433, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 3560, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 4041, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 39, 0, 803, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 3333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 13, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4006, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 93, 0, 1086, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 2940, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 8, 0, 4079, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 143, 2, 389, 0, 2, 0, 0, 0, 4055, 1, 2, 1, 0, 0, 3, 0, 4078, 0, 4086, 30, 0, 54, 0, 0, 3987, 0, 3, 0, 2, 0, 8, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 0, 23, 0, 4073, 9, 5, 4089, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3910, 0, 0, 0, 3805, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[15, 83, 0, 1552, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 2511, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4092, 0, 0, 4095, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 4031, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 19, 3, 3368, 0, 0, 0, 0, 12, 3981, 0, 0, 0, 0, 0, 13, 0, 4067, 0, 4039, 219, 0, 15, 25, 3, 877, 0, 9, 0, 0, 0, 0, 0, 4, 45, 0, 0, 0, 88, 0, 11, 0, 0, 0, 0, 3697, 22, 0, 4071, 0, 0, 0, 0, 0, 0, 19, 0, 0, 4069, 0, 0, 0, 4092, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 283, 0, 94, 0, 0, 0, 0, 2979, 12, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3866, 0, 0, 0, 0, 0, 0, 969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 28, 19, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 0, 2, 0, 3953, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3368, 4110, 4063, 4835, 4084, 4141, 4096, 4071] avg=4096.0000 max=4835 max_rank=3 min=3368 min_rank=0 imbalance_ratio=1.180420 max_load_reduction=27933 max_load_reduction_pct=0.852448 moved_to_echo=16392 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 180, 0, 291, 0, 0, 0, 0, 2926, 13, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3978, 0, 0, 0, 0, 0, 0, 952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 27, 19, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 0, 2, 0, 3842, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3410, 4110, 4062, 4930, 4083, 4119, 4096, 3958] avg=4096.0000 max=4930 max_rank=3 min=3410 min_rank=0 imbalance_ratio=1.203613 max_load_reduction=27838 max_load_reduction_pct=0.849548 moved_to_echo=16256 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 93, 0, 1086, 0, 0, 0, 0, 2974, 13, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2940, 0, 0, 4, 0, 0, 0, 967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 28, 19, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 0, 8, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4166, 4111, 4062, 3911, 4083, 4136, 4096, 4203] avg=4096.0000 max=4203 max_rank=7 min=3911 min_rank=3 imbalance_ratio=1.026123 max_load_reduction=28565 max_load_reduction_pct=0.871735 moved_to_echo=16518 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 39, 0, 803, 0, 0, 0, 0, 2982, 13, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3333, 0, 0, 0, 0, 0, 0, 970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 28, 19, 0, 0, 0, 0, 0, 0, 4094, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 0, 0, 0, 4006, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3837, 4111, 4062, 4303, 4083, 4143, 4107, 4122] avg=4096.0000 max=4303 max_rank=3 min=3837 min_rank=0 imbalance_ratio=1.050537 max_load_reduction=28465 max_load_reduction_pct=0.868683 moved_to_echo=16455 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 143, 2, 389, 0, 2, 0, 0, 2847, 12, 0, 0, 0, 0, 0, 0, 0, 4055, 1, 2, 1, 0, 0, 3, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 30, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3987, 0, 3, 0, 2, 0, 8, 926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 23, 0, 4073, 9, 5, 27, 19, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 0, 0, 3805, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3395, 4076, 4137, 4926, 4069, 4158, 4092, 3915] avg=4096.0000 max=4926 max_rank=3 min=3395 min_rank=0 imbalance_ratio=1.202637 max_load_reduction=27842 max_load_reduction_pct=0.849670 moved_to_echo=16234 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[12, 149, 0, 433, 0, 0, 0, 0, 2974, 13, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3560, 0, 0, 0, 0, 0, 0, 967, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 28, 19, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3581, 4111, 4062, 4527, 4092, 4143, 4096, 4156] avg=4096.0000 max=4527 max_rank=3 min=3581 min_rank=0 imbalance_ratio=1.105225 max_load_reduction=28241 max_load_reduction_pct=0.861847 moved_to_echo=16487 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[15, 83, 0, 1552, 0, 0, 0, 0, 2978, 13, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2511, 0, 0, 1, 0, 0, 0, 968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4092, 0, 0, 28, 19, 0, 0, 0, 0, 0, 0, 4095, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 0, 0, 0, 4031, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4641, 4111, 4062, 3480, 4083, 4143, 4101, 4147] avg=4096.0000 max=4641 max_rank=0 min=3480 min_rank=3 imbalance_ratio=1.133057 max_load_reduction=28127 max_load_reduction_pct=0.858368 moved_to_echo=16474 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 19, 3, 3368, 0, 0, 0, 0, 2967, 13, 0, 0, 0, 0, 0, 0, 12, 3981, 0, 0, 0, 0, 0, 13, 16, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3999, 219, 0, 15, 25, 0, 0, 0, 0, 0, 0, 0, 0, 3, 877, 0, 9, 0, 0, 0, 0, 967, 0, 0, 0, 0, 0, 0, 0, 0, 4, 45, 0, 0, 0, 88, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 3697, 22, 0, 31, 24, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6370, 4028, 4258, 1856, 4191, 3785, 4084, 4196] avg=4096.0000 max=6370 max_rank=0 min=1856 min_rank=3 imbalance_ratio=1.555176 max_load_reduction=26398 max_load_reduction_pct=0.805603 moved_to_echo=16256 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 25088, 0, 128, 128, 128, 0, 128, 7808, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 1024, 128, 8064, 0, 128, 0, 0, 23680, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 0, 0, 0, 128, 0, 32640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 32512, 256, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 1024, 0, 128, 0, 31872, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 0, 128, 0, 32384, 128, 128, 256, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 32640, 128, 128, 128, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 2840, 0, 0, 0, 3400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 45, 0, 0, 649, 0, 0, 4063, 0, 0, 0, 0, 4, 4096, 1301, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 3333, 0, 0, 0, 3078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 19, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1117, 0, 0, 3964, 0, 0, 12, 0, 0, 4087, 660, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 115] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 1610, 0, 0, 0, 3452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 21, 0, 0, 57, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3052, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 2949, 0, 0, 0, 3450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 42, 0, 0, 57, 0, 0, 4095, 0, 0, 2, 0, 0, 4096, 1693, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 1932, 0, 0, 0, 3788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4088, 136, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 17, 0, 0, 4093, 0, 0, 0, 0, 2, 4096, 2335, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[3, 6, 0, 3967, 0, 1, 21, 3325, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 17, 0, 4092, 0, 3, 4093, 0, 0, 0, 1, 2, 0, 4044, 0, 2, 0, 107, 3, 3, 17, 0, 5, 4010, 0, 0, 0, 0, 0, 4092, 859, 0, 0, 0, 1, 4089, 0, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 965, 0, 0, 0, 3917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 84, 0, 0, 4095, 0, 0, 117, 0, 0, 4096, 3164, 0, 0, 0, 0, 4058, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[14, 2, 0, 1832, 0, 0, 35, 3778, 1, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4077, 6, 1, 4040, 0, 0, 0, 34, 9, 65, 4057, 2, 4, 11, 236, 12, 15, 12, 0, 0, 3052, 0, 5, 232, 0, 0, 4082, 3012, 0, 0, 0, 0, 4060, 0, 0, 0, 48, 0, 0, 0, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2949, 0, 0, 0, 1623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 3955, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1693, 0, 0, 0, 0, 2092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2004, 1827, 140, 59, 14, 1, 0, 0] rerouted_tokens_per_ep_rank=[4572, 4095, 4096, 4082, 4079, 4014, 3785, 4045] avg=4096.0000 max=4572 max_rank=0 min=3785 min_rank=6 imbalance_ratio=1.116211 max_load_reduction=28196 max_load_reduction_pct=0.860474 moved_to_echo=15923 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1610, 0, 0, 0, 1624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 3956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3052, 0, 0, 0, 0, 2092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2004, 1828, 140, 59, 14, 0, 0, 0] rerouted_tokens_per_ep_rank=[3234, 4096, 4096, 4082, 4058, 4013, 5144, 4045] avg=4096.0000 max=5144 max_rank=6 min=3234 min_rank=0 imbalance_ratio=1.255859 max_load_reduction=27624 max_load_reduction_pct=0.843018 moved_to_echo=17260 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3333, 0, 0, 0, 1448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 19, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1117, 0, 0, 3828, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 660, 0, 0, 0, 0, 2092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 2004, 1630, 136, 59, 14, 0, 0, 0] rerouted_tokens_per_ep_rank=[4781, 4087, 4096, 4100, 4037, 4957, 2752, 3958] avg=4096.0000 max=4957 max_rank=5 min=2752 min_rank=6 imbalance_ratio=1.210205 max_load_reduction=27811 max_load_reduction_pct=0.848724 moved_to_echo=15704 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3, 6, 0, 3967, 0, 1, 21, 1564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4079, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3986, 0, 2, 0, 107, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 5, 3872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 859, 0, 0, 0, 1, 2089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2000, 1761, 138, 58, 14, 1, 0, 0] rerouted_tokens_per_ep_rank=[5562, 4095, 4109, 4085, 4101, 3894, 2949, 3973] avg=4096.0000 max=5562 max_rank=0 min=2949 min_rank=6 imbalance_ratio=1.357910 max_load_reduction=27206 max_load_reduction_pct=0.830261 moved_to_echo=14917 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1932, 0, 0, 0, 1782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 3953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2335, 0, 0, 0, 0, 2089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2001, 2006, 140, 59, 14, 1, 0, 0] rerouted_tokens_per_ep_rank=[3714, 4095, 4096, 4210, 4036, 3970, 4426, 4221] avg=4096.0000 max=4426 max_rank=6 min=3714 min_rank=0 imbalance_ratio=1.080566 max_load_reduction=28342 max_load_reduction_pct=0.864929 moved_to_echo=16653 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2840, 0, 0, 0, 1600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 649, 0, 0, 3924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1301, 0, 0, 0, 0, 2087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1998, 1800, 139, 59, 14, 0, 0, 0] rerouted_tokens_per_ep_rank=[4440, 4096, 4096, 4079, 4082, 4573, 3392, 4010] avg=4096.0000 max=4573 max_rank=5 min=3392 min_rank=6 imbalance_ratio=1.116455 max_load_reduction=28195 max_load_reduction_pct=0.860443 moved_to_echo=16057 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 965, 0, 0, 0, 1843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 3955, 0, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3164, 0, 0, 0, 0, 2073, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1985, 2074, 140, 59, 14, 0, 0, 0] rerouted_tokens_per_ep_rank=[2808, 4096, 4096, 4065, 4037, 4156, 5237, 4273] avg=4096.0000 max=5237 max_rank=6 min=2808 min_rank=0 imbalance_ratio=1.278564 max_load_reduction=27531 max_load_reduction_pct=0.840179 moved_to_echo=17703 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[14, 2, 0, 1832, 0, 0, 35, 1774, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 25, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4077, 4, 0, 0, 0, 0, 0, 0, 0, 6, 1, 4015, 0, 0, 0, 34, 9, 0, 0, 0, 0, 0, 0, 0, 0, 65, 3993, 2, 4, 11, 236, 12, 15, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 2942, 0, 5, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3012, 0, 0, 0, 0, 2071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 7, 1989, 2004, 110, 64, 21, 0, 0, 0] rerouted_tokens_per_ep_rank=[3657, 4108, 4083, 4065, 4338, 3191, 5083, 4243] avg=4096.0000 max=5083 max_rank=6 min=3191 min_rank=5 imbalance_ratio=1.240967 max_load_reduction=27685 max_load_reduction_pct=0.844879 moved_to_echo=16855 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 0, 128, 0, 0, 0, 128, 16000, 14976, 1152, 512, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 19456, 0, 128, 128, 13312, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 16128, 0, 0, 0, 128, 16768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2048, 0, 128, 30464, 0, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 32256, 128, 128, 128, 512, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 0, 0, 128, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 32640, 256, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 3293, 0, 0, 0, 831, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 4096, 0, 4035, 0, 0, 0, 90, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2, 4076, 1, 0, 4091, 0, 4073, 0, 0, 0, 190, 0, 3919, 0, 3, 0, 0, 0, 2, 0, 0, 0, 3, 0, 19, 4090, 0, 0, 3, 0, 4091, 0, 1, 0, 0, 0, 0, 0, 4093, 7, 4, 0, 0, 4, 0, 0, 0, 4081, 0, 3, 2, 6, 2, 2, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 2554, 0, 0, 0, 2023, 0, 3626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4095, 3, 0, 4096, 0, 3253, 0, 0, 0, 546, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 296, 4096, 0, 0, 0, 2, 4096, 0, 0, 1, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 20, 0, 19, 13, 3947, 0, 1, 4075, 0, 1764, 0, 0, 0, 2355, 0, 4050, 4, 4, 2, 0, 0, 0, 0, 0, 0, 14, 0, 114, 4078, 0, 0, 24, 73, 4082, 0, 0, 0, 0, 0, 0, 0, 4081, 3, 12, 0, 0, 0, 0, 0, 0, 3969, 0, 15, 21, 0, 9, 0, 0, 18, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2, 0, 0, 4095, 0, 0, 4096, 0, 3676, 0, 1, 0, 1073, 0, 3358, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 4087, 1, 0, 0, 51, 4096, 0, 0, 0, 0, 0, 44, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 4091, 10, 0, 4096, 0, 3294, 0, 0, 0, 1253, 0, 3650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4084, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 856, 0, 0, 0, 0, 3293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 831, 0, 3220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3238, 850, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 11, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4149, 4051, 4099, 4083, 4089, 4114, 4092] avg=4096.0000 max=4149 max_rank=1 min=4051 min_rank=2 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16378 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 856, 0, 0, 0, 0, 2554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2023, 0, 2869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3237, 757, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 11, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 3410, 4892, 4006, 4075, 4089, 4114, 4090] avg=4096.0000 max=4892 max_rank=2 min=3410 min_rank=1 imbalance_ratio=1.194336 max_load_reduction=27876 max_load_reduction_pct=0.850708 moved_to_echo=16368 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 856, 1, 0, 0, 0, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 3219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3239, 850, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 11, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4892, 3309, 4100, 4081, 4089, 4114, 4091] avg=4096.0000 max=4892 max_rank=1 min=3309 min_rank=2 imbalance_ratio=1.194336 max_load_reduction=27876 max_load_reduction_pct=0.850708 moved_to_echo=16375 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2, 4087, 0, 0, 0, 0, 0, 0, 0, 852, 1, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 190, 0, 3101, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 0, 19, 3224, 818, 9, 2, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4078, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4081, 0, 11, 6, 0, 0, 0, 0, 0, 0, 3, 2, 6, 2, 2, 0, 0, 0, 4081, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4926, 3294, 4077, 4082, 4098, 4102, 4100] avg=4096.0000 max=4926 max_rank=1 min=3294 min_rank=2 imbalance_ratio=1.202637 max_load_reduction=27842 max_load_reduction_pct=0.849670 moved_to_echo=16382 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 856, 0, 0, 0, 0, 3676, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1073, 0, 2657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 3239, 701, 10, 2, 0, 0, 0, 0, 0, 1, 0, 0, 51, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 11, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4533, 3730, 3956, 4135, 4131, 4108, 4081] avg=4096.0000 max=4533 max_rank=1 min=3730 min_rank=2 imbalance_ratio=1.106689 max_load_reduction=28235 max_load_reduction_pct=0.861664 moved_to_echo=16455 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 856, 3, 0, 0, 0, 3253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 0, 3240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 296, 3239, 855, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 11, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4112, 3786, 4402, 4085, 4094, 4107, 4090] avg=4096.0000 max=4402 max_rank=3 min=3786 min_rank=2 imbalance_ratio=1.074707 max_load_reduction=28366 max_load_reduction_pct=0.865662 moved_to_echo=16376 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 855, 10, 0, 0, 0, 3294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1253, 0, 2888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3236, 762, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 11, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4159, 4141, 4010, 4071, 4090, 4114, 4090] avg=4096.0000 max=4159 max_rank=1 min=4010 min_rank=3 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16365 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 20, 0, 19, 13, 4072, 0, 0, 0, 0, 0, 0, 0, 822, 0, 1, 0, 0, 1764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2355, 0, 3200, 4, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 114, 3125, 850, 10, 5, 0, 0, 0, 0, 0, 0, 0, 24, 73, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 3, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3969, 0, 18, 8, 0, 0, 0, 0, 0, 0, 15, 21, 0, 9, 0, 0, 18, 0, 4068, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 2587, 5565, 4118, 4156, 4088, 3995, 4134] avg=4096.0000 max=5565 max_rank=2 min=2587 min_rank=1 imbalance_ratio=1.358643 max_load_reduction=27203 max_load_reduction_pct=0.830170 moved_to_echo=16373 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 0, 128, 0, 512, 25856, 6528, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[6912, 128, 128, 0, 0, 25984, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 8448, 0, 24448, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 0, 128, 128, 32640, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 0, 0, 0, 32640, 0, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 0, 128, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 0, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 3763, 0, 0, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 0, 0, 0, 0, 1, 0, 3988, 4091, 0, 253, 0, 0, 0, 0, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4071, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3739, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 342, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 10, 0, 0, 0, 0, 13, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3975, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 4092, 16, 0, 0, 0, 0, 0, 4091, 0, 7, 0, 0, 0, 4087, 0, 0, 0, 0, 4038, 7, 0, 4078, 0, 0, 1, 2, 0, 0, 4, 0, 0, 11, 0, 0, 9, 6, 3, 4, 0, 0, 0, 4093, 4080, 3, 0, 0, 4, 0, 0, 41, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4090, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3722, 0, 0, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 0, 0, 0, 0, 0, 0, 3983, 4096, 0, 0, 0, 0, 0, 0, 18, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 4074, 50, 0, 69, 0, 3, 17, 4071, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 2752, 2, 4, 4001, 0, 2, 12, 14, 0, 0, 3, 0, 0, 5, 15, 0, 1241, 0, 0, 26, 0, 0, 0, 3996, 4038, 11, 45, 0, 16, 0, 0, 0, 3, 53, 0, 0, 10, 0, 92, 0, 0, 0, 0, 4076, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3768, 21, 16, 8, 2, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4100, 4423, 4094, 3816, 4094, 4073, 4088] avg=4096.0000 max=4423 max_rank=2 min=3816 min_rank=4 imbalance_ratio=1.079834 max_load_reduction=28345 max_load_reduction_pct=0.865021 moved_to_echo=16071 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 3768, 19, 16, 8, 2, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4100, 4399, 4094, 3838, 4094, 4075, 4088] avg=4096.0000 max=4399 max_rank=2 min=3838 min_rank=4 imbalance_ratio=1.073975 max_load_reduction=28369 max_load_reduction_pct=0.865753 moved_to_echo=16095 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4080, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3763, 0, 0, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 0, 0, 0, 0, 3693, 19, 16, 8, 2, 0, 0, 0, 1, 0, 3986, 0, 0, 253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4100, 4086, 4096, 3921, 4240, 4155, 4087] avg=4096.0000 max=4240 max_rank=5 min=3921 min_rank=4 imbalance_ratio=1.035156 max_load_reduction=28528 max_load_reduction_pct=0.870605 moved_to_echo=16403 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3739, 0, 0, 327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 342, 0, 0, 0, 0, 3760, 20, 16, 8, 2, 0, 0, 0, 0, 0, 4094, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4100, 4066, 4094, 4148, 4104, 4088, 4088] avg=4096.0000 max=4148 max_rank=4 min=4066 min_rank=2 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16428 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4076, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 7, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 7, 0, 327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 4, 0, 0, 4087, 2, 0, 0, 0, 0, 0, 0, 11, 0, 0, 9, 6, 3, 4, 0, 3751, 19, 16, 8, 2, 0, 0, 0, 0, 0, 4091, 0, 3, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4098, 4372, 4096, 3829, 4098, 4100, 4083] avg=4096.0000 max=4372 max_rank=2 min=3829 min_rank=4 imbalance_ratio=1.067383 max_load_reduction=28396 max_load_reduction_pct=0.866577 moved_to_echo=16110 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3975, 0, 0, 327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 0, 0, 0, 0, 3756, 19, 16, 8, 2, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4100, 4302, 4096, 3930, 4094, 4078, 4088] avg=4096.0000 max=4302 max_rank=2 min=3930 min_rank=4 imbalance_ratio=1.050293 max_load_reduction=28466 max_load_reduction_pct=0.868713 moved_to_echo=16190 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3722, 0, 0, 323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 0, 0, 0, 0, 3714, 20, 16, 8, 2, 0, 0, 0, 0, 0, 3981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 27, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4100, 4045, 4095, 4261, 3981, 4118, 4088] avg=4096.0000 max=4261 max_rank=4 min=3981 min_rank=5 imbalance_ratio=1.040283 max_load_reduction=28507 max_load_reduction_pct=0.869965 moved_to_echo=16448 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4054, 50, 0, 69, 0, 3, 17, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2752, 2, 4, 317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 12, 14, 0, 0, 3, 0, 0, 4068, 4, 0, 0, 0, 0, 0, 0, 5, 15, 0, 1241, 0, 0, 26, 0, 3684, 19, 20, 10, 5, 0, 0, 0, 0, 0, 3991, 0, 11, 45, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 53, 0, 0, 10, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4193, 4070, 3075, 4103, 5025, 4063, 4081, 4158] avg=4096.0000 max=5025 max_rank=4 min=3075 min_rank=2 imbalance_ratio=1.226807 max_load_reduction=27743 max_load_reduction_pct=0.846649 moved_to_echo=17327 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 0, 30208, 128, 128, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 32512, 0, 128, 384, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 2432, 128, 128, 128, 0, 29952, 256, 256, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 0, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 128, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 32640, 128, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 128, 128, 0, 0, 128, 32512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 0, 128, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33792 slack_slots=31744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True + [2026-09-21 08:24:25] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 264.2 | throughput per GPU (TFLOP/s/GPU): 121.9 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.841822E+00 | loss scale: 1.0 | grad norm: 6.835 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[187, 332, 953, 2, 0, 767, 926, 242, 1, 639, 30, 0, 8, 34, 242, 107, 10, 1511, 666, 2128, 54, 182, 222, 2, 135, 15, 53, 337, 918, 702, 740, 1467, 933, 481, 435, 1269, 135, 1124, 715, 1024, 1458, 552, 1611, 42, 952, 863, 4, 38, 1080, 594, 519, 26, 579, 1292, 1645, 69, 1104, 8, 209, 10, 7, 16, 336, 26] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[156, 264, 1260, 0, 0, 561, 874, 234, 3, 875, 36, 0, 7, 3, 286, 90, 15, 799, 985, 1796, 17, 281, 359, 25, 88, 16, 82, 466, 1112, 793, 969, 1950, 796, 346, 528, 986, 99, 1094, 903, 888, 997, 482, 1847, 118, 736, 444, 19, 27, 1060, 575, 573, 5, 528, 1753, 1505, 28, 1087, 20, 207, 3, 9, 6, 685, 12] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[184, 261, 1080, 0, 0, 682, 1177, 416, 4, 629, 11, 5, 7, 52, 342, 149, 9, 1018, 879, 1787, 29, 151, 231, 9, 69, 10, 97, 299, 1309, 816, 990, 1217, 687, 357, 419, 1024, 116, 1150, 493, 803, 1667, 424, 1615, 55, 944, 660, 23, 55, 943, 686, 644, 42, 455, 1447, 1801, 38, 1490, 34, 253, 11, 53, 10, 439, 11] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[134, 290, 857, 9, 0, 639, 896, 314, 14, 561, 103, 3, 24, 14, 491, 117, 1, 905, 1181, 1923, 44, 152, 382, 26, 73, 13, 190, 291, 1044, 693, 830, 1345, 502, 368, 473, 1109, 91, 1177, 563, 995, 1663, 868, 1522, 88, 992, 637, 9, 42, 802, 647, 779, 24, 592, 1058, 1716, 23, 1503, 28, 219, 21, 15, 29, 627, 27] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[272, 285, 694, 10, 2, 536, 767, 384, 6, 925, 18, 2, 24, 46, 484, 121, 1, 1013, 960, 1928, 78, 136, 273, 0, 257, 24, 22, 353, 967, 535, 944, 1553, 1083, 294, 325, 1135, 135, 997, 692, 990, 991, 670, 1542, 49, 742, 640, 6, 68, 722, 1046, 682, 81, 563, 876, 1717, 74, 1470, 25, 280, 1, 9, 36, 1184, 23] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[229, 461, 659, 1, 0, 421, 938, 216, 14, 419, 16, 3, 31, 2, 430, 70, 18, 1346, 667, 1542, 80, 312, 514, 6, 116, 13, 69, 316, 985, 688, 970, 1430, 839, 560, 302, 837, 114, 1074, 610, 1106, 1377, 952, 1917, 416, 628, 345, 44, 114, 1073, 638, 615, 86, 846, 1753, 904, 59, 1464, 60, 323, 11, 27, 64, 600, 28] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[106, 328, 753, 7, 0, 450, 852, 332, 2, 663, 32, 0, 6, 38, 432, 208, 1, 1362, 1019, 1628, 23, 99, 281, 4, 111, 3, 63, 353, 1106, 375, 1332, 1558, 631, 476, 555, 1458, 177, 1248, 902, 725, 1072, 320, 1494, 47, 992, 755, 3, 49, 799, 717, 781, 5, 458, 1378, 1707, 23, 1263, 29, 228, 2, 14, 10, 911, 12] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[240, 254, 697, 4, 3, 425, 884, 286, 3, 598, 28, 0, 21, 24, 542, 54, 7, 659, 524, 1906, 23, 287, 269, 14, 125, 10, 147, 320, 1159, 614, 851, 1454, 545, 590, 556, 1405, 211, 1853, 701, 1166, 1201, 485, 1960, 47, 945, 702, 3, 74, 941, 863, 420, 39, 604, 726, 1625, 33, 1574, 5, 302, 5, 89, 4, 648, 14] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[187, 332, 953, 2, 0, 767, 926, 242, 535, 289, 124, 85, 0, 0, 0, 0, 1, 639, 30, 0, 8, 34, 242, 107, 1645, 1124, 33, 0, 0, 0, 0, 0, 10, 1511, 666, 1839, 54, 182, 222, 2, 0, 0, 0, 0, 0, 0, 0, 0, 135, 15, 53, 337, 918, 702, 740, 833, 0, 0, 0, 0, 0, 0, 0, 0, 933, 481, 435, 734, 135, 0, 715, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1458, 552, 499, 42, 952, 863, 4, 38, 0, 0, 0, 0, 0, 0, 0, 0, 1080, 594, 519, 26, 579, 1168, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 1104, 8, 209, 10, 7, 16, 336, 26, 1079, 549, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4442, 3863, 4486, 3733, 4457, 4408, 4035, 3344] avg=4096.0000 max=4486 max_rank=2 min=3344 min_rank=7 imbalance_ratio=1.095215 max_load_reduction=28282 max_load_reduction_pct=0.863098 moved_to_echo=16244 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[134, 290, 857, 9, 0, 639, 896, 314, 467, 261, 102, 78, 0, 0, 0, 0, 14, 561, 103, 3, 24, 14, 491, 117, 1716, 1177, 31, 0, 0, 0, 0, 0, 1, 905, 1181, 1662, 44, 152, 382, 26, 0, 0, 0, 0, 0, 0, 0, 0, 73, 13, 190, 291, 1044, 693, 830, 764, 0, 0, 0, 0, 0, 0, 0, 0, 502, 368, 473, 642, 91, 0, 563, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1663, 868, 471, 88, 992, 637, 9, 42, 0, 0, 0, 0, 0, 0, 0, 0, 802, 647, 779, 24, 592, 956, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 1503, 28, 219, 21, 15, 29, 627, 27, 1020, 503, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4047, 4251, 4353, 3898, 3634, 4770, 3823, 3992] avg=4096.0000 max=4770 max_rank=5 min=3634 min_rank=4 imbalance_ratio=1.164551 max_load_reduction=27998 max_load_reduction_pct=0.854431 moved_to_echo=16219 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[156, 264, 1260, 0, 0, 561, 874, 234, 415, 244, 169, 113, 0, 0, 0, 0, 3, 875, 36, 0, 7, 3, 286, 90, 1505, 1094, 38, 0, 0, 0, 0, 0, 15, 799, 985, 1552, 17, 281, 359, 25, 0, 0, 0, 0, 0, 0, 0, 0, 88, 16, 82, 466, 1112, 793, 969, 1107, 0, 0, 0, 0, 0, 0, 0, 0, 796, 346, 528, 571, 99, 0, 903, 888, 0, 0, 0, 0, 0, 0, 0, 0, 997, 482, 572, 118, 736, 444, 19, 27, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 575, 573, 5, 528, 1584, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 1087, 20, 207, 3, 9, 6, 685, 12, 1237, 730, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4290, 3937, 4033, 4633, 4131, 3395, 4353, 3996] avg=4096.0000 max=4633 max_rank=3 min=3395 min_rank=5 imbalance_ratio=1.131104 max_load_reduction=28135 max_load_reduction_pct=0.858612 moved_to_echo=15875 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[229, 461, 659, 1, 0, 421, 938, 216, 353, 209, 169, 83, 0, 0, 0, 0, 14, 419, 16, 3, 31, 2, 430, 70, 904, 1074, 40, 0, 0, 0, 0, 0, 18, 1346, 667, 1333, 80, 312, 514, 6, 0, 0, 0, 0, 0, 0, 0, 0, 116, 13, 69, 316, 985, 688, 970, 812, 0, 0, 0, 0, 0, 0, 0, 0, 839, 560, 302, 484, 114, 0, 610, 1106, 0, 0, 0, 0, 0, 0, 0, 0, 1377, 952, 593, 416, 628, 345, 44, 114, 0, 0, 0, 0, 0, 0, 0, 0, 1073, 638, 615, 86, 846, 1584, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 1464, 60, 323, 11, 27, 64, 600, 28, 1284, 535, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3739, 3003, 4276, 3969, 4015, 4469, 4901, 4396] avg=4096.0000 max=4901 max_rank=6 min=3003 min_rank=1 imbalance_ratio=1.196533 max_load_reduction=27867 max_load_reduction_pct=0.850433 moved_to_echo=17781 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[184, 261, 1080, 0, 0, 682, 1177, 416, 431, 243, 139, 71, 0, 0, 0, 0, 4, 629, 11, 5, 7, 52, 342, 149, 1801, 1150, 33, 0, 0, 0, 0, 0, 9, 1018, 879, 1544, 29, 151, 231, 9, 0, 0, 0, 0, 0, 0, 0, 0, 69, 10, 97, 299, 1309, 816, 990, 691, 0, 0, 0, 0, 0, 0, 0, 0, 687, 357, 419, 593, 116, 0, 493, 803, 0, 0, 0, 0, 0, 0, 0, 0, 1667, 424, 500, 55, 944, 660, 23, 55, 0, 0, 0, 0, 0, 0, 0, 0, 943, 686, 644, 42, 455, 1308, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 1490, 34, 253, 11, 53, 10, 439, 11, 1082, 455, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4684, 4183, 3870, 4281, 3468, 4328, 4116, 3838] avg=4096.0000 max=4684 max_rank=0 min=3468 min_rank=4 imbalance_ratio=1.143555 max_load_reduction=28084 max_load_reduction_pct=0.857056 moved_to_echo=15750 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[272, 285, 694, 10, 2, 536, 767, 384, 478, 262, 84, 90, 0, 0, 0, 0, 6, 925, 18, 2, 24, 46, 484, 121, 1717, 997, 32, 0, 0, 0, 0, 0, 1, 1013, 960, 1666, 78, 136, 273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 257, 24, 22, 353, 967, 535, 944, 882, 0, 0, 0, 0, 0, 0, 0, 0, 1083, 294, 325, 657, 135, 0, 692, 990, 0, 0, 0, 0, 0, 0, 0, 0, 991, 670, 477, 49, 742, 640, 6, 68, 0, 0, 0, 0, 0, 0, 0, 0, 722, 1046, 682, 81, 563, 792, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 1470, 25, 280, 1, 9, 36, 1184, 23, 1033, 581, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3864, 4372, 4127, 3984, 4176, 3643, 3960, 4642] avg=4096.0000 max=4642 max_rank=7 min=3643 min_rank=5 imbalance_ratio=1.133301 max_load_reduction=28126 max_load_reduction_pct=0.858337 moved_to_echo=16421 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[106, 328, 753, 7, 0, 450, 852, 332, 614, 221, 133, 91, 0, 0, 0, 0, 2, 663, 32, 0, 6, 38, 432, 208, 1707, 1248, 31, 0, 0, 0, 0, 0, 1, 1362, 1019, 1407, 23, 99, 281, 4, 0, 0, 0, 0, 0, 0, 0, 0, 111, 3, 63, 353, 1106, 375, 1332, 884, 0, 0, 0, 0, 0, 0, 0, 0, 631, 476, 555, 844, 177, 0, 902, 725, 0, 0, 0, 0, 0, 0, 0, 0, 1072, 320, 462, 47, 992, 755, 3, 49, 0, 0, 0, 0, 0, 0, 0, 0, 799, 717, 781, 5, 458, 1245, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 1263, 29, 228, 2, 14, 10, 911, 12, 1001, 583, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3887, 4367, 4196, 4227, 4310, 3700, 4028, 4053] avg=4096.0000 max=4367 max_rank=1 min=3700 min_rank=5 imbalance_ratio=1.066162 max_load_reduction=28401 max_load_reduction_pct=0.866730 moved_to_echo=16091 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[240, 254, 697, 4, 3, 425, 884, 286, 597, 262, 74, 89, 0, 0, 0, 0, 3, 598, 28, 0, 21, 24, 542, 54, 1625, 1853, 44, 0, 0, 0, 0, 0, 7, 659, 524, 1644, 23, 287, 269, 14, 0, 0, 0, 0, 0, 0, 0, 0, 125, 10, 147, 320, 1159, 614, 851, 817, 0, 0, 0, 0, 0, 0, 0, 0, 545, 590, 556, 808, 211, 0, 701, 1166, 0, 0, 0, 0, 0, 0, 0, 0, 1201, 485, 598, 47, 945, 702, 3, 74, 0, 0, 0, 0, 0, 0, 0, 0, 941, 863, 420, 39, 604, 652, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 1574, 5, 302, 5, 89, 4, 648, 14, 1318, 548, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3815, 4792, 3427, 4043, 4577, 4055, 3552, 4507] avg=4096.0000 max=4792 max_rank=1 min=3427 min_rank=2 imbalance_ratio=1.169922 max_load_reduction=27976 max_load_reduction_pct=0.853760 moved_to_echo=16691 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 5376, 384, 128, 128, 256, 3328, 1024, 12672, 9728, 384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[6016, 3584, 3712, 5376, 1152, 0, 5632, 7808, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[10496, 4864, 4224, 896, 7040, 5120, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[1536, 2560, 7040, 128, 128, 4608, 7424, 2432, 3968, 2048, 1024, 768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 8704, 6912, 12672, 384, 1664, 2560, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[7424, 5888, 5120, 384, 4736, 9344, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1024, 128, 768, 2816, 8704, 5248, 7680, 6912, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[11008, 256, 2048, 128, 256, 256, 5504, 256, 9088, 4608, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[938, 0, 0, 1, 0, 771, 3, 9, 15, 2, 3813, 57, 0, 0, 867, 2, 0, 0, 696, 85, 113, 52, 45, 5, 0, 1481, 0, 6, 12, 2230, 2170, 15, 3, 0, 0, 0, 1937, 3067, 3461, 0, 1710, 0, 334, 0, 0, 0, 170, 0, 1, 14, 246, 32, 537, 4, 1, 5, 23, 0, 646, 3030, 3247, 0, 912, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[646, 0, 0, 0, 0, 989, 0, 85, 4, 0, 3911, 470, 0, 1, 352, 0, 0, 0, 41, 0, 959, 168, 34, 0, 1, 1758, 0, 0, 2, 611, 2147, 0, 0, 0, 1, 0, 2484, 4083, 3697, 1, 172, 3, 194, 0, 0, 0, 26, 0, 1, 0, 794, 394, 371, 0, 0, 0, 110, 1, 150, 2984, 1782, 0, 3341, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[2142, 0, 526, 0, 0, 1546, 0, 154, 28, 145, 755, 612, 0, 0, 5, 0, 0, 0, 2945, 54, 3351, 4, 26, 0, 0, 270, 0, 0, 3, 212, 1778, 18, 0, 0, 0, 0, 707, 3768, 1172, 6, 91, 0, 773, 0, 0, 0, 185, 0, 1, 1, 680, 105, 725, 0, 0, 0, 262, 0, 1466, 989, 3637, 0, 3626, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[632, 0, 0, 1, 0, 1469, 0, 81, 0, 9, 3566, 1120, 0, 0, 163, 16, 0, 3, 255, 163, 5, 95, 397, 0, 2, 2398, 1, 0, 320, 406, 928, 0, 0, 3, 0, 0, 1957, 4034, 1552, 3, 1070, 0, 654, 3, 0, 0, 173, 3, 0, 6, 19, 1470, 1208, 0, 0, 0, 1213, 0, 27, 2901, 1892, 0, 2550, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[41, 0, 22, 0, 0, 1045, 0, 56, 0, 0, 3903, 563, 9, 0, 286, 22, 0, 0, 827, 0, 1752, 15, 901, 0, 2, 2751, 1, 0, 177, 153, 197, 19, 1, 11, 0, 1, 1549, 3684, 1226, 0, 262, 9, 643, 1, 1, 0, 0, 0, 0, 0, 412, 380, 727, 1, 0, 1, 651, 0, 2043, 3033, 2766, 0, 2622, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[301, 10, 2, 0, 0, 105, 0, 270, 0, 38, 3777, 531, 0, 0, 136, 2, 1, 0, 762, 1, 1200, 4, 18, 1, 0, 1857, 1, 1, 0, 179, 176, 2, 0, 0, 0, 0, 3177, 4012, 1793, 11, 1732, 0, 1446, 0, 3, 2, 25, 0, 1, 0, 149, 638, 762, 0, 0, 0, 404, 1, 14, 2956, 2493, 0, 3773, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[129, 0, 5, 0, 6, 810, 0, 78, 14, 109, 3746, 1029, 0, 0, 195, 0, 0, 0, 771, 0, 462, 3, 28, 85, 0, 959, 0, 60, 1, 24, 2219, 131, 4, 0, 2, 0, 3509, 2360, 1545, 0, 1588, 0, 148, 0, 7, 0, 8, 4, 0, 0, 3411, 137, 1262, 5, 0, 22, 1566, 0, 104, 87, 2485, 0, 3650, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[802, 0, 1, 0, 0, 1515, 0, 0, 0, 65, 2467, 352, 4, 0, 1776, 0, 0, 0, 91, 24, 443, 3, 68, 15, 0, 2452, 0, 0, 0, 1766, 1792, 27, 0, 0, 1, 0, 2375, 3750, 1391, 0, 1434, 0, 246, 0, 0, 0, 0, 0, 5, 0, 27, 1750, 209, 0, 107, 8, 311, 10, 667, 2024, 1223, 0, 3567, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[938, 0, 0, 1, 0, 771, 3, 9, 971, 321, 0, 0, 0, 0, 0, 0, 15, 2, 3495, 57, 0, 0, 867, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 696, 85, 113, 52, 45, 5, 2306, 283, 0, 0, 0, 0, 0, 0, 0, 1481, 0, 6, 12, 2230, 2170, 15, 88, 35, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1849, 0, 3461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1710, 0, 334, 0, 0, 0, 170, 0, 2096, 0, 0, 0, 0, 0, 0, 0, 1, 14, 246, 32, 537, 4, 1, 5, 591, 93, 0, 0, 0, 0, 0, 0, 23, 0, 646, 3030, 848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3014, 4438, 3585, 6037, 5313, 4310, 1524, 4547] avg=4096.0000 max=6037 max_rank=3 min=1524 min_rank=6 imbalance_ratio=1.473877 max_load_reduction=26731 max_load_reduction_pct=0.815765 moved_to_echo=15694 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[646, 0, 0, 0, 0, 989, 0, 85, 1294, 1176, 0, 0, 0, 0, 0, 0, 4, 0, 3585, 470, 0, 1, 352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 959, 168, 34, 0, 1265, 290, 0, 0, 0, 0, 0, 0, 1, 1758, 0, 0, 2, 611, 2147, 0, 113, 36, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2371, 0, 3697, 1, 0, 0, 0, 0, 0, 0, 0, 0, 172, 3, 194, 0, 0, 0, 26, 0, 2789, 0, 0, 0, 0, 0, 0, 0, 1, 0, 794, 394, 371, 0, 0, 0, 2165, 51, 0, 0, 0, 0, 0, 0, 110, 1, 150, 2984, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4190, 4412, 2757, 4668, 6070, 3184, 3776, 3711] avg=4096.0000 max=6070 max_rank=4 min=2757 min_rank=2 imbalance_ratio=1.481934 max_load_reduction=26698 max_load_reduction_pct=0.814758 moved_to_echo=16741 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[301, 10, 2, 0, 0, 105, 0, 270, 1271, 1328, 0, 0, 0, 0, 0, 0, 0, 38, 3463, 531, 0, 0, 136, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 762, 1, 1200, 4, 18, 1, 1770, 280, 0, 0, 0, 0, 0, 0, 0, 1857, 1, 1, 0, 179, 176, 2, 145, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3032, 0, 1793, 11, 0, 0, 0, 0, 0, 0, 0, 0, 1732, 0, 1446, 0, 3, 2, 25, 0, 2741, 0, 0, 0, 0, 0, 0, 0, 1, 0, 149, 638, 762, 0, 0, 0, 2445, 72, 0, 0, 0, 0, 0, 0, 404, 1, 14, 2956, 651, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3287, 4170, 4037, 2395, 4836, 5949, 4067, 4027] avg=4096.0000 max=5949 max_rank=5 min=2395 min_rank=3 imbalance_ratio=1.452393 max_load_reduction=26819 max_load_reduction_pct=0.818451 moved_to_echo=18879 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2142, 0, 526, 0, 0, 1546, 0, 154, 1193, 1277, 0, 0, 0, 0, 0, 0, 28, 145, 693, 612, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2945, 54, 3351, 4, 26, 0, 2583, 56, 0, 0, 0, 0, 0, 0, 0, 270, 0, 0, 3, 212, 1778, 18, 32, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 675, 0, 1172, 6, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 773, 0, 0, 0, 185, 0, 2575, 0, 0, 0, 0, 0, 0, 0, 1, 1, 680, 105, 725, 0, 0, 0, 2349, 105, 0, 0, 0, 0, 0, 0, 262, 0, 1466, 989, 949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6838, 1483, 9019, 2319, 1853, 3624, 3966, 3666] avg=4096.0000 max=9019 max_rank=2 min=1483 min_rank=1 imbalance_ratio=2.201904 max_load_reduction=23749 max_load_reduction_pct=0.724762 moved_to_echo=13109 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[632, 0, 0, 1, 0, 1469, 0, 81, 1278, 899, 0, 0, 0, 0, 0, 0, 0, 9, 3269, 1120, 0, 0, 163, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 255, 163, 5, 95, 397, 0, 1343, 265, 0, 0, 0, 0, 0, 0, 2, 2398, 1, 0, 320, 406, 928, 0, 89, 32, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1868, 0, 1552, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1070, 0, 654, 3, 0, 0, 173, 3, 2756, 0, 0, 0, 0, 0, 0, 0, 0, 6, 19, 1470, 1208, 0, 0, 0, 1651, 54, 0, 0, 0, 0, 0, 0, 1213, 0, 27, 2901, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4360, 4577, 2526, 4176, 3426, 4659, 4408, 4636] avg=4096.0000 max=4659 max_rank=5 min=2526 min_rank=2 imbalance_ratio=1.137451 max_load_reduction=28109 max_load_reduction_pct=0.857819 moved_to_echo=17129 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[129, 0, 5, 0, 6, 810, 0, 78, 747, 1285, 0, 0, 0, 0, 0, 0, 14, 109, 3434, 1029, 0, 0, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 771, 0, 462, 3, 28, 85, 1765, 278, 0, 0, 0, 0, 0, 0, 0, 959, 0, 60, 1, 24, 2219, 131, 161, 34, 0, 0, 0, 0, 0, 0, 4, 0, 2, 0, 3348, 0, 1545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1588, 0, 148, 0, 7, 0, 8, 4, 1613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3411, 137, 1262, 5, 0, 22, 2365, 71, 0, 0, 0, 0, 0, 0, 1566, 0, 104, 87, 649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3060, 4781, 3392, 3589, 4899, 3368, 7273, 2406] avg=4096.0000 max=7273 max_rank=6 min=2406 min_rank=7 imbalance_ratio=1.775635 max_load_reduction=25495 max_load_reduction_pct=0.778046 moved_to_echo=17946 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[41, 0, 22, 0, 0, 1045, 0, 56, 1168, 924, 0, 0, 0, 0, 0, 0, 0, 0, 3568, 563, 9, 0, 286, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 827, 0, 1752, 15, 901, 0, 1968, 294, 0, 0, 0, 0, 0, 0, 2, 2751, 1, 0, 177, 153, 197, 19, 76, 41, 0, 0, 0, 0, 0, 0, 1, 11, 0, 1, 1473, 0, 1226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 262, 9, 643, 1, 1, 0, 0, 0, 2516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 412, 380, 727, 1, 0, 1, 1698, 84, 0, 0, 0, 0, 0, 0, 651, 0, 2043, 3033, 714, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3256, 4448, 5757, 3417, 2712, 3432, 3303, 6443] avg=4096.0000 max=6443 max_rank=7 min=2712 min_rank=4 imbalance_ratio=1.572998 max_load_reduction=26325 max_load_reduction_pct=0.803375 moved_to_echo=15890 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[802, 0, 1, 0, 0, 1515, 0, 0, 1188, 1257, 0, 0, 0, 0, 0, 0, 0, 65, 2262, 352, 4, 0, 1776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 24, 443, 3, 68, 15, 868, 183, 0, 0, 0, 0, 0, 0, 0, 2452, 0, 0, 0, 1766, 1792, 27, 108, 22, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2267, 0, 1391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1434, 0, 246, 0, 0, 0, 0, 0, 2562, 0, 0, 0, 0, 0, 0, 0, 5, 0, 27, 1750, 209, 0, 107, 8, 2310, 35, 0, 0, 0, 0, 0, 0, 311, 10, 667, 2024, 320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4763, 4459, 1695, 6167, 3659, 4242, 4451, 3332] avg=4096.0000 max=6167 max_rank=3 min=1695 min_rank=2 imbalance_ratio=1.505615 max_load_reduction=26601 max_load_reduction_pct=0.811798 moved_to_echo=15684 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 6400, 384, 8320, 384, 1536, 128, 13952, 2048, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[5632, 128, 640, 128, 128, 8320, 128, 768, 9216, 8576, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 13952, 128, 128, 640, 5632, 11520, 256, 896, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 5760, 4992, 5888, 128, 128, 128, 15616, 640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4608, 128, 5120, 18048, 5120, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 16896, 0, 15872, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[8064, 128, 4480, 128, 128, 128, 640, 128, 19712, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 384, 23808, 4736, 128, 128, 3840, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3116, 0, 1738, 0, 1846, 0, 0, 980, 0, 4091, 0, 991, 132, 4, 3907, 0, 10, 0, 0, 0, 0, 0, 0, 71, 9, 0, 0, 0, 0, 2850, 104, 0, 0, 0, 38, 0, 0, 0, 0, 0, 789, 0, 0, 0, 0, 2, 4061, 250, 2, 2877, 4067, 0, 0, 36, 0, 0, 779, 3, 15, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 4088, 2, 121, 0, 1, 0, 0, 38, 0, 4080, 0, 590, 1827, 13, 4088, 1, 1, 2, 0, 3, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1064, 0, 0, 122, 1, 0, 0, 0, 1, 2, 6, 4004, 1, 0, 0, 2, 38, 4032, 706, 6, 3108, 3891, 0, 0, 8, 0, 0, 856, 0, 61, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2186, 0, 3508, 0, 52, 0, 0, 163, 0, 4087, 1, 143, 3778, 12, 1425, 11, 20, 0, 0, 0, 2, 0, 0, 44, 0, 0, 0, 0, 0, 912, 0, 3, 0, 171, 15, 0, 0, 0, 0, 0, 2827, 1, 0, 0, 0, 0, 4052, 2510, 0, 829, 3905, 0, 3, 0, 0, 3, 1592, 5, 508, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3216, 0, 1022, 0, 4, 0, 11, 181, 1, 3420, 1, 1565, 3495, 2, 4012, 23, 14, 0, 0, 0, 0, 0, 1, 249, 5, 0, 0, 0, 0, 520, 0, 0, 1, 1, 6, 1, 0, 1, 0, 8, 3058, 0, 4, 0, 9, 0, 3944, 1435, 0, 1677, 3979, 0, 0, 0, 0, 0, 856, 43, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[4, 3, 42, 0, 4033, 0, 266, 0, 32, 0, 0, 1776, 1, 4091, 0, 1998, 3338, 1, 3809, 0, 0, 0, 0, 0, 0, 0, 1, 4, 131, 14, 0, 0, 5, 819, 29, 0, 0, 216, 0, 0, 1, 0, 2, 1, 1833, 0, 0, 0, 46, 0, 4068, 34, 1, 2722, 3130, 0, 3, 0, 1, 0, 270, 0, 43, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 4061, 0, 3366, 5, 9, 0, 1, 1485, 0, 1378, 0, 2765, 2225, 0, 3856, 5, 0, 10, 0, 0, 0, 0, 1, 216, 0, 0, 0, 0, 48, 96, 0, 0, 17, 4, 68, 0, 0, 0, 0, 0, 3395, 22, 0, 0, 14, 0, 4092, 28, 0, 1389, 3650, 0, 0, 0, 0, 0, 389, 4, 167, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 56, 0, 4087, 1, 2121, 0, 607, 1, 0, 188, 0, 3326, 0, 2146, 2770, 175, 2958, 0, 50, 0, 0, 0, 2, 2, 0, 10, 0, 0, 0, 1, 1, 377, 0, 5, 0, 0, 7, 0, 1, 0, 0, 1, 3785, 0, 0, 0, 0, 0, 4057, 39, 0, 2667, 3230, 0, 1, 42, 0, 2, 47, 2, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3236, 0, 1369, 0, 129, 0, 0, 1044, 0, 4069, 0, 85, 2083, 0, 3373, 21, 0, 0, 0, 0, 0, 0, 0, 22, 0, 1, 0, 6, 2, 2117, 0, 0, 1, 83, 13, 0, 0, 0, 0, 7, 3173, 0, 0, 0, 15, 0, 4094, 15, 0, 3045, 4085, 0, 22, 0, 0, 0, 641, 0, 17, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1494, 0, 3508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 163, 0, 1996, 1, 143, 0, 0, 0, 0, 0, 0, 0, 0, 3778, 12, 663, 11, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 44, 0, 0, 0, 0, 3998, 0, 0, 0, 0, 0, 0, 0, 0, 912, 0, 3, 0, 171, 15, 0, 1954, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2827, 1, 0, 0, 692, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2510, 0, 829, 1214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 1592, 5, 508, 0, 2691, 274, 54, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5002, 2355, 4484, 4044, 3543, 3657, 4553, 5130] avg=4096.0000 max=5130 max_rank=7 min=2355 min_rank=1 imbalance_ratio=1.252441 max_load_reduction=27638 max_load_reduction_pct=0.843445 moved_to_echo=16883 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2197, 0, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 11, 181, 1, 1670, 1, 1565, 0, 0, 0, 0, 0, 0, 0, 0, 3495, 2, 1865, 23, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 249, 5, 0, 0, 0, 3892, 0, 0, 0, 0, 0, 0, 0, 0, 520, 0, 0, 1, 1, 6, 1, 1635, 1375, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8, 3058, 0, 4, 0, 1019, 115, 0, 0, 0, 0, 0, 0, 9, 0, 0, 1435, 0, 1677, 1237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 856, 43, 3, 0, 2742, 772, 52, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3219, 3433, 5399, 4147, 3539, 4205, 4358, 4468] avg=4096.0000 max=5399 max_rank=2 min=3219 min_rank=0 imbalance_ratio=1.318115 max_load_reduction=27369 max_load_reduction_pct=0.835236 moved_to_echo=16570 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2129, 0, 1738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1846, 0, 0, 980, 0, 1998, 0, 991, 0, 0, 0, 0, 0, 0, 0, 0, 132, 4, 1816, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 9, 0, 0, 0, 4007, 0, 0, 0, 0, 0, 0, 0, 0, 2850, 104, 0, 0, 0, 38, 0, 1956, 1339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 789, 0, 0, 0, 987, 137, 0, 0, 0, 0, 0, 0, 0, 2, 0, 250, 2, 2877, 1264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 779, 3, 15, 0, 2803, 752, 54, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3867, 5815, 1962, 4087, 6287, 1913, 4395, 4442] avg=4096.0000 max=6287 max_rank=4 min=1913 min_rank=5 imbalance_ratio=1.534912 max_load_reduction=26481 max_load_reduction_pct=0.808136 moved_to_echo=17037 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 2774, 0, 3366, 5, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 1, 1485, 0, 674, 0, 2765, 0, 0, 0, 0, 0, 0, 0, 0, 2225, 0, 1792, 5, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 216, 0, 0, 0, 0, 4038, 0, 0, 0, 0, 0, 0, 0, 48, 96, 0, 0, 17, 4, 68, 0, 658, 1322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3395, 22, 0, 0, 1287, 46, 0, 0, 0, 0, 0, 0, 14, 0, 0, 28, 0, 1389, 1135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 389, 4, 167, 0, 2515, 742, 54, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6147, 4934, 4032, 4255, 2213, 4750, 2566, 3871] avg=4096.0000 max=6147 max_rank=0 min=2213 min_rank=4 imbalance_ratio=1.500732 max_load_reduction=26621 max_load_reduction_pct=0.812408 moved_to_echo=13400 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 2793, 2, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 38, 0, 1993, 0, 590, 0, 0, 0, 0, 0, 0, 0, 0, 1827, 13, 1900, 1, 1, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3980, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 0, 0, 122, 1, 0, 0, 1950, 1401, 0, 0, 0, 0, 0, 0, 0, 1, 2, 6, 4004, 1, 0, 0, 1295, 137, 0, 0, 0, 0, 0, 0, 2, 38, 0, 706, 6, 3108, 1210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 856, 0, 61, 1, 2681, 787, 52, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2918, 2622, 3747, 3981, 4538, 5446, 5070, 4446] avg=4096.0000 max=5446 max_rank=5 min=2622 min_rank=1 imbalance_ratio=1.329590 max_load_reduction=27322 max_load_reduction_pct=0.833801 moved_to_echo=19500 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2211, 0, 1369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 0, 0, 1044, 0, 1987, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 2083, 0, 1568, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 1, 0, 6, 4041, 0, 0, 0, 0, 0, 0, 0, 2, 2117, 0, 0, 1, 83, 13, 0, 1945, 1156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3173, 0, 0, 0, 1025, 137, 0, 0, 0, 0, 0, 0, 15, 0, 0, 15, 0, 3045, 1270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 641, 0, 17, 0, 2815, 649, 53, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3580, 3245, 3672, 4070, 5317, 4342, 4345, 4197] avg=4096.0000 max=5317 max_rank=4 min=3245 min_rank=1 imbalance_ratio=1.298096 max_load_reduction=27451 max_load_reduction_pct=0.837738 moved_to_echo=18201 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4, 3, 42, 0, 2755, 0, 266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 1776, 1, 1998, 0, 1998, 0, 0, 0, 0, 0, 0, 0, 0, 3338, 1, 1771, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 131, 14, 0, 0, 4015, 0, 0, 0, 0, 0, 0, 0, 5, 819, 29, 0, 0, 216, 0, 0, 1956, 1305, 0, 0, 0, 0, 0, 0, 1, 0, 2, 1, 1833, 0, 0, 0, 1278, 137, 0, 0, 0, 0, 0, 0, 46, 0, 0, 34, 1, 2722, 973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 270, 0, 43, 0, 2157, 733, 53, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3070, 5805, 5110, 4165, 4330, 3252, 3776, 3260] avg=4096.0000 max=5805 max_rank=1 min=3070 min_rank=0 imbalance_ratio=1.417236 max_load_reduction=26963 max_load_reduction_pct=0.822845 moved_to_echo=14618 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 56, 0, 2787, 1, 2121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 607, 1, 0, 188, 0, 1617, 0, 2146, 0, 0, 0, 0, 0, 0, 0, 0, 2770, 175, 1367, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 10, 0, 0, 0, 1, 4004, 0, 0, 0, 0, 0, 0, 0, 1, 377, 0, 5, 0, 0, 7, 0, 1593, 1018, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 3785, 0, 0, 0, 1300, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 2667, 999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 42, 0, 2, 47, 2, 3, 0, 2231, 573, 53, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4965, 4559, 4362, 4019, 3001, 5203, 3705, 2954] avg=4096.0000 max=5203 max_rank=5 min=2954 min_rank=7 imbalance_ratio=1.270264 max_load_reduction=27565 max_load_reduction_pct=0.841217 moved_to_echo=14863 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 8832, 256, 128, 256, 512, 256, 128, 13696, 9472, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 5504, 128, 896, 128, 20736, 5376, 512, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[19712, 256, 12800, 128, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[2688, 128, 128, 5888, 128, 13952, 128, 10368, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 19200, 128, 13568, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 5120, 128, 18432, 9344, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 22912, 128, 128, 0, 8960, 1024, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 640, 256, 128, 0, 128, 32000, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33792 slack_slots=31744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 4092, 1, 0, 2, 693, 2453, 0, 3, 8, 0, 0, 0, 0, 3965, 4082, 0, 20, 0, 0, 0, 0, 0, 17, 1, 288, 0, 3997, 0, 0, 4005, 0, 4, 3, 0, 5, 0, 0, 3358, 0, 0, 0, 0, 4, 0, 1688, 4, 0, 3991, 0, 50, 0, 0, 0, 0, 0, 30, 2, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4078, 1, 0, 0, 551, 1916, 0, 0, 1808, 0, 50, 0, 4, 3566, 3910, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 4089, 1, 0, 4024, 0, 0, 0, 0, 0, 3, 0, 4094, 238, 3, 0, 0, 0, 0, 290, 0, 0, 3917, 5, 1, 0, 0, 2, 0, 0, 210, 0, 0, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 1, 4093, 0, 0, 1, 2, 2576, 0, 0, 105, 0, 27, 0, 0, 4089, 4076, 0, 0, 0, 0, 0, 0, 0, 6, 0, 85, 0, 4088, 1, 1, 3987, 0, 0, 0, 0, 0, 0, 0, 4005, 0, 0, 0, 0, 1, 0, 1392, 0, 0, 4080, 0, 107, 0, 0, 0, 0, 0, 40, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3609, 0, 0, 0, 1998, 2116, 0, 0, 1849, 0, 14, 0, 0, 4095, 2299, 0, 0, 0, 0, 0, 0, 0, 0, 1, 230, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4055, 46, 0, 0, 0, 1, 0, 164, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 1, 4090, 0, 0, 0, 27, 3928, 0, 0, 226, 0, 14, 2, 0, 3986, 3996, 0, 0, 2, 0, 0, 2, 0, 0, 2, 167, 0, 4076, 0, 8, 4052, 0, 1, 0, 0, 0, 8, 0, 3740, 3, 9, 0, 0, 0, 0, 150, 0, 0, 4090, 1, 184, 0, 0, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4081, 0, 1, 0, 664, 1089, 0, 0, 1519, 0, 0, 0, 0, 4077, 4087, 3, 0, 0, 0, 0, 0, 1, 0, 1, 7, 0, 4095, 0, 0, 3859, 0, 8, 0, 0, 0, 0, 0, 3790, 4, 0, 0, 0, 0, 0, 1824, 0, 0, 3609, 0, 10, 0, 0, 0, 1, 0, 28, 10, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4027, 0, 0, 0, 608, 3358, 0, 0, 229, 0, 1, 0, 0, 3977, 4042, 0, 0, 20, 0, 0, 0, 0, 1, 0, 359, 0, 4096, 0, 0, 3999, 0, 0, 0, 0, 0, 0, 0, 3716, 0, 0, 0, 0, 0, 0, 158, 0, 0, 4077, 0, 6, 0, 0, 0, 0, 14, 80, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1, 0, 15, 0, 2, 4066, 0, 1, 1, 306, 3507, 2, 0, 930, 0, 123, 0, 0, 4009, 4094, 0, 0, 0, 0, 0, 0, 0, 1, 6, 3, 6, 3705, 0, 0, 3187, 4, 0, 6, 2, 0, 56, 0, 4092, 7, 0, 0, 0, 0, 0, 914, 0, 1, 3489, 0, 220, 1, 4, 0, 1, 0, 2, 0, 4, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 1, 4093, 0, 0, 75, 0, 0, 0, 0, 0, 0, 0, 1, 2, 2576, 0, 0, 105, 0, 27, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 275, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 85, 0, 4005, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 3987, 0, 0, 0, 0, 0, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4005, 0, 0, 0, 0, 1, 199, 0, 0, 0, 0, 0, 0, 0, 0, 1392, 0, 0, 3334, 0, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 1, 3814, 350, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4173, 2719, 4351, 4096, 4186, 4205, 4833, 4205] avg=4096.0000 max=4833 max_rank=6 min=2719 min_rank=1 imbalance_ratio=1.179932 max_load_reduction=27935 max_load_reduction_pct=0.852509 moved_to_echo=17429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4081, 0, 1, 75, 0, 0, 0, 0, 0, 0, 0, 0, 664, 1089, 0, 0, 1519, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 274, 4087, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 7, 0, 4012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3859, 0, 8, 0, 0, 0, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3790, 4, 0, 0, 0, 0, 176, 0, 0, 0, 0, 0, 0, 0, 0, 1824, 0, 0, 2949, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 28, 10, 0, 0, 3803, 310, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4157, 3280, 4364, 4021, 4041, 3970, 4783, 4152] avg=4096.0000 max=4783 max_rank=6 min=3280 min_rank=1 imbalance_ratio=1.167725 max_load_reduction=27985 max_load_reduction_pct=0.854034 moved_to_echo=16946 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 4092, 1, 0, 73, 0, 0, 0, 0, 0, 0, 0, 2, 693, 2453, 0, 3, 8, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, 4082, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 1, 288, 0, 3917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4005, 0, 4, 3, 0, 5, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3358, 0, 0, 0, 0, 4, 195, 0, 0, 0, 0, 0, 0, 0, 0, 1688, 4, 0, 3262, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 2, 0, 0, 3699, 342, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4168, 3166, 4368, 4223, 4209, 3557, 5004, 4073] avg=4096.0000 max=5004 max_rank=6 min=3166 min_rank=1 imbalance_ratio=1.221680 max_load_reduction=27764 max_load_reduction_pct=0.847290 moved_to_echo=16843 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4078, 1, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 551, 1916, 0, 0, 1808, 0, 50, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4, 240, 3910, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4006, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4024, 0, 0, 0, 0, 0, 189, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4094, 238, 3, 0, 0, 0, 191, 0, 0, 0, 0, 0, 0, 0, 0, 290, 0, 0, 3201, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 210, 0, 0, 3, 3326, 336, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4154, 4333, 4156, 4008, 4214, 4529, 3497, 3877] avg=4096.0000 max=4529 max_rank=5 min=3497 min_rank=6 imbalance_ratio=1.105713 max_load_reduction=28239 max_load_reduction_pct=0.861786 moved_to_echo=16117 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3609, 0, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 1998, 2116, 0, 0, 1849, 0, 14, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 275, 2299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 230, 0, 4010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 46, 0, 0, 0, 1, 200, 0, 0, 0, 0, 0, 0, 0, 0, 164, 0, 0, 3348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 3820, 351, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3684, 5985, 2574, 4241, 4293, 4302, 3512, 4177] avg=4096.0000 max=5985 max_rank=1 min=2574 min_rank=2 imbalance_ratio=1.461182 max_load_reduction=26783 max_load_reduction_pct=0.817352 moved_to_echo=16284 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4027, 0, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 608, 3358, 0, 0, 229, 0, 1, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 267, 4042, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 359, 0, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3999, 0, 0, 0, 0, 0, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3716, 0, 0, 0, 0, 0, 199, 0, 0, 0, 0, 0, 0, 0, 0, 158, 0, 0, 3331, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 80, 0, 0, 0, 3710, 350, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4204, 4329, 4373, 4196, 3915, 3495, 4154] avg=4096.0000 max=4373 max_rank=3 min=3495 min_rank=6 imbalance_ratio=1.067627 max_load_reduction=28395 max_load_reduction_pct=0.866547 moved_to_echo=15760 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 1, 4090, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 27, 3928, 0, 0, 226, 0, 14, 8, 0, 0, 0, 0, 0, 0, 0, 2, 0, 268, 3996, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 167, 0, 3994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4052, 0, 1, 0, 0, 0, 197, 0, 0, 0, 0, 0, 0, 0, 8, 0, 3740, 3, 9, 0, 0, 0, 200, 0, 0, 0, 0, 0, 0, 0, 0, 150, 0, 0, 3342, 1, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3718, 351, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4167, 4203, 4268, 4165, 4258, 3960, 3677, 4070] avg=4096.0000 max=4268 max_rank=2 min=3677 min_rank=6 imbalance_ratio=1.041992 max_load_reduction=28500 max_load_reduction_pct=0.869751 moved_to_echo=15965 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 15, 0, 2, 4066, 0, 1, 71, 7, 0, 0, 0, 0, 0, 0, 1, 306, 3507, 2, 0, 930, 0, 123, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 264, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 3, 6, 3625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3187, 4, 0, 6, 2, 0, 172, 0, 0, 0, 0, 0, 0, 0, 56, 0, 4092, 7, 0, 0, 0, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 914, 0, 1, 2831, 0, 220, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 2, 0, 4, 0, 3745, 304, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4163, 4878, 4358, 3641, 3371, 4330, 3967, 4060] avg=4096.0000 max=4878 max_rank=1 min=3371 min_rank=4 imbalance_ratio=1.190918 max_load_reduction=27890 max_load_reduction_pct=0.851135 moved_to_echo=15728 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 6656, 128, 128, 25600, 128, 640, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 4864, 20992, 128, 128, 6784, 0, 256, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 2176, 30592, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33920 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 128, 512, 128, 128, 128, 29696, 2816, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 30976, 384, 128, 0, 0, 128, 1536, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 31232, 128, 128, 128, 128, 128, 1536, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 128, 128, 32256, 128, 128, 640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 1152, 128, 31616, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33920 slack_slots=31616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33792 slack_slots=31744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3616, 6, 0, 4069, 4094, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 12, 1, 0, 0, 0, 0, 0, 0, 885, 0, 0, 3, 0, 0, 0, 15, 0, 0, 0, 4090, 4094, 0, 4065, 581, 0, 1, 0, 1, 0, 275, 0, 913, 0, 2673, 0, 0, 0, 0, 2, 1, 3369] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 2199, 0, 0, 4020, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 39, 0, 0, 2699, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4086, 4064, 1, 4089, 2449, 0, 0, 0, 0, 0, 824, 21, 975, 0, 36, 0, 0, 0, 0, 0, 0, 3150] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 9, 404, 0, 2, 4008, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 33, 237, 0, 0, 0, 0, 156, 0, 0, 2578, 0, 0, 18, 0, 1, 0, 0, 5, 0, 2, 4000, 4090, 0, 4058, 485, 0, 18, 0, 1, 1, 166, 1, 2489, 2, 4056, 1, 0, 0, 0, 0, 1, 1829] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[3, 0, 0, 0, 3863, 0, 0, 3561, 4073, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 3514, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 3954, 4085, 0, 4023, 574, 0, 0, 0, 1, 0, 575, 50, 2533, 0, 884, 0, 0, 0, 0, 0, 0, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4036, 0, 0, 3868, 4096, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 0, 0, 0, 0, 20, 0, 0, 3035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4095, 3807, 0, 0, 0, 0, 0, 860, 0, 268, 0, 80, 0, 0, 0, 0, 0, 0, 232] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1, 7, 0, 0, 2523, 2, 0, 3483, 4091, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 3, 0, 159, 4012, 4096, 0, 4096, 720, 0, 0, 0, 0, 13, 1306, 0, 2418, 0, 3229, 0, 2, 0, 0, 0, 0, 2566] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 3420, 0, 0, 3601, 4096, 7, 4, 0, 1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 25, 0, 0, 0, 4, 0, 0, 0, 2543, 2, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4094, 4092, 0, 4016, 10, 0, 0, 0, 3, 0, 79, 16, 2980, 0, 1090, 0, 0, 0, 1, 0, 0, 2673] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3833, 0, 0, 3670, 4084, 11, 0, 0, 86, 0, 4, 0, 85, 0, 0, 1, 0, 0, 0, 1, 2, 0, 2, 0, 17, 0, 0, 2227, 1, 0, 0, 4, 7, 0, 0, 0, 0, 0, 4068, 4079, 0, 4088, 785, 0, 0, 12, 12, 0, 144, 0, 1756, 1, 2930, 2, 4, 0, 3, 0, 1, 848] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3616, 6, 0, 1188, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 12, 4018, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 885, 1965, 47, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 15, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2929, 0, 0, 0, 581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 275, 0, 913, 0, 913, 916, 0, 0, 0, 0, 0, 0, 2673, 0, 0, 0, 0, 2, 1, 3369, 248, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4810, 4083, 4031, 2898, 4112, 3510, 3019, 6305] avg=4096.0000 max=6305 max_rank=7 min=2898 min_rank=3 imbalance_ratio=1.539307 max_load_reduction=26463 max_load_reduction_pct=0.807587 moved_to_echo=16946 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 0, 3863, 0, 0, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 3975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3514, 1719, 48, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2832, 0, 0, 0, 574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 575, 50, 2533, 0, 882, 801, 0, 0, 0, 0, 0, 0, 884, 0, 0, 0, 0, 0, 0, 1032, 240, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4907, 4072, 4004, 5281, 4088, 3406, 4842, 2168] avg=4096.0000 max=5281 max_rank=3 min=2168 min_rank=7 imbalance_ratio=1.289307 max_load_reduction=27487 max_load_reduction_pct=0.838837 moved_to_echo=14504 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 9, 404, 0, 2, 1171, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 33, 237, 4010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 0, 2578, 1935, 48, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 1, 0, 0, 5, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2864, 0, 0, 0, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 1, 1, 166, 1, 2489, 2, 893, 902, 0, 0, 0, 0, 0, 0, 4056, 1, 0, 0, 0, 0, 1, 1829, 243, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1586, 4084, 4301, 4717, 4114, 3351, 4473, 6142] avg=4096.0000 max=6142 max_rank=7 min=1586 min_rank=0 imbalance_ratio=1.499512 max_load_reduction=26626 max_load_reduction_pct=0.812561 moved_to_echo=18080 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 2199, 0, 0, 1174, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 2699, 1941, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2926, 0, 1, 0, 2449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 824, 21, 975, 0, 912, 905, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 3150, 248, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3375, 4087, 4052, 4727, 4068, 5376, 3637, 3446] avg=4096.0000 max=5376 max_rank=5 min=3375 min_rank=0 imbalance_ratio=1.312500 max_load_reduction=27392 max_load_reduction_pct=0.835938 moved_to_echo=16527 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4036, 0, 0, 1131, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 3035, 1867, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2933, 0, 0, 0, 3807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 860, 0, 268, 0, 914, 870, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 232, 249, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5168, 4098, 4210, 4971, 4096, 6740, 2912, 573] avg=4096.0000 max=6740 max_rank=5 min=573 min_rank=7 imbalance_ratio=1.645508 max_load_reduction=26028 max_load_reduction_pct=0.794312 moved_to_echo=14321 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 7, 0, 0, 2523, 2, 0, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 33, 1682, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 159, 2874, 0, 0, 0, 720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1306, 0, 2418, 0, 895, 784, 0, 0, 0, 0, 0, 0, 3229, 0, 2, 0, 0, 0, 0, 2566, 243, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3550, 4080, 4048, 1770, 4099, 3753, 5416, 6052] avg=4096.0000 max=6052 max_rank=7 min=1770 min_rank=3 imbalance_ratio=1.477539 max_load_reduction=26716 max_load_reduction_pct=0.815308 moved_to_echo=19320 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 3420, 0, 0, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 7, 4, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 25, 3969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 2543, 1739, 47, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 6, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2932, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 79, 16, 2980, 0, 914, 810, 0, 0, 0, 0, 0, 0, 1090, 0, 0, 0, 1, 0, 0, 2673, 248, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4474, 4098, 3995, 4333, 4100, 2942, 4802, 4024] avg=4096.0000 max=4802 max_rank=6 min=2942 min_rank=5 imbalance_ratio=1.172363 max_load_reduction=27966 max_load_reduction_pct=0.853455 moved_to_echo=15868 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3833, 0, 0, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 11, 0, 0, 86, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 1, 0, 0, 0, 1, 4040, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 17, 0, 0, 2227, 1775, 48, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 7, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2905, 0, 0, 0, 785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 12, 0, 144, 0, 1756, 1, 912, 830, 0, 0, 0, 0, 0, 0, 2930, 2, 4, 0, 3, 0, 1, 848, 251, 19, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4898, 4166, 4127, 4071, 4091, 3690, 3667, 4058] avg=4096.0000 max=4898 max_rank=0 min=3667 min_rank=6 imbalance_ratio=1.195801 max_load_reduction=27870 max_load_reduction_pct=0.850525 moved_to_echo=15506 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 256, 0, 0, 17536, 14720, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 23936, 128, 128, 8960, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[15104, 128, 128, 0, 128, 128, 128, 15744, 2048, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 4352, 128, 14336, 128, 7296, 6912, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 256, 23296, 0, 128, 0, 9472, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[32640, 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 0, 0, 128, 512, 32256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 2199, 0, 851, 0, 0, 0, 0, 0, 4089, 0, 6, 0, 0, 0, 2782, 1207, 0, 0, 0, 0, 0, 391, 20, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1483, 0, 0, 0, 0, 0, 3945, 362, 0, 1925, 0, 0, 4096, 0, 0, 0, 0, 0, 3880, 0, 0, 1436, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 2, 2867, 0, 30, 0, 0, 0, 0, 1, 4067, 0, 0, 1, 0, 0, 420, 3465, 3, 0, 0, 28, 0, 1775, 49, 0, 0, 0, 3, 0, 0, 4093, 0, 0, 0, 0, 0, 960, 1, 1, 0, 0, 0, 3918, 4, 0, 909, 0, 0, 4093, 0, 0, 0, 0, 0, 4092, 0, 0, 1985, 0, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 1915, 0, 27, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4026, 2442, 2, 0, 0, 2, 0, 66, 24, 0, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4, 1482, 0, 0, 2, 0, 0, 4030, 217, 0, 3586, 0, 0, 4066, 0, 0, 0, 0, 0, 1026, 0, 1, 1665, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 1232, 0, 578, 0, 0, 0, 0, 0, 4091, 0, 0, 2, 0, 0, 1979, 3811, 84, 0, 0, 0, 0, 242, 50, 0, 0, 0, 59, 0, 0, 3928, 0, 0, 0, 0, 0, 792, 0, 0, 12, 0, 0, 4095, 11, 0, 955, 0, 0, 4088, 0, 0, 0, 0, 0, 3324, 0, 0, 3435, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 601, 0, 281, 0, 1, 1, 2, 1, 4094, 2, 14, 0, 0, 2, 4066, 2079, 52, 0, 0, 2, 0, 5, 2, 2, 0, 0, 782, 0, 0, 4077, 0, 0, 0, 0, 0, 313, 0, 0, 0, 0, 0, 3654, 531, 2, 301, 0, 0, 4034, 0, 0, 0, 0, 0, 4090, 2, 0, 3773, 0, 0, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 39, 0, 71, 2, 0, 0, 0, 0, 4094, 0, 6, 0, 0, 1, 393, 2139, 0, 0, 8, 0, 0, 99, 393, 0, 0, 0, 333, 0, 0, 4091, 0, 0, 0, 0, 0, 3488, 1, 1, 9, 1, 0, 4055, 72, 1, 3160, 0, 0, 4086, 4, 0, 0, 0, 0, 4005, 0, 0, 2215, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1, 5, 0, 587, 1, 4, 0, 0, 6, 3, 4, 4086, 0, 0, 0, 0, 2, 1196, 3120, 0, 2, 0, 134, 0, 573, 49, 2, 0, 2, 44, 5, 0, 4074, 0, 1, 0, 0, 10, 3382, 11, 0, 0, 0, 0, 4060, 0, 3, 1960, 0, 1, 4091, 3, 0, 0, 9, 4, 4001, 0, 0, 1326, 1, 5, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 219, 0, 1433, 0, 0, 0, 0, 0, 4096, 0, 36, 0, 0, 0, 3097, 3403, 2, 0, 0, 595, 0, 234, 47, 0, 0, 0, 383, 0, 0, 4096, 0, 0, 0, 0, 6, 165, 0, 0, 0, 0, 0, 4068, 1012, 0, 1942, 0, 0, 4096, 0, 0, 0, 0, 0, 3502, 0, 0, 336, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 2867, 0, 30, 0, 0, 1661, 1231, 98, 0, 0, 0, 0, 0, 0, 0, 1, 4065, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 420, 2223, 3, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1775, 49, 0, 0, 0, 3, 0, 0, 1974, 1395, 0, 0, 0, 0, 0, 0, 2600, 0, 0, 0, 0, 0, 960, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1944, 4, 0, 909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 11, 2, 0, 0, 0, 0, 0, 0, 2431, 0, 0, 1985, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5889, 4067, 2674, 5196, 3561, 2858, 4106, 4417] avg=4096.0000 max=5889 max_rank=0 min=2674 min_rank=2 imbalance_ratio=1.437744 max_load_reduction=26879 max_load_reduction_pct=0.820282 moved_to_echo=14942 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1232, 0, 578, 0, 0, 1349, 1354, 94, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1979, 2444, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 242, 50, 0, 0, 0, 59, 0, 0, 2063, 1339, 0, 0, 0, 0, 0, 0, 2495, 0, 0, 0, 0, 0, 792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 2032, 11, 0, 955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 13, 2, 0, 0, 0, 0, 0, 0, 1975, 0, 0, 3435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4607, 4091, 4507, 3753, 3287, 3010, 4103, 5410] avg=4096.0000 max=5410 max_rank=7 min=3010 min_rank=5 imbalance_ratio=1.320801 max_load_reduction=27358 max_load_reduction_pct=0.834900 moved_to_echo=15810 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2199, 0, 851, 0, 0, 1575, 429, 98, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2782, 774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 391, 20, 0, 0, 0, 0, 0, 0, 1987, 1396, 0, 0, 0, 0, 0, 0, 2602, 0, 0, 0, 0, 0, 1483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1958, 362, 0, 1925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 2305, 0, 0, 1436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5152, 4093, 3556, 3794, 4085, 4245, 4102, 3741] avg=4096.0000 max=5152 max_rank=0 min=3556 min_rank=2 imbalance_ratio=1.257812 max_load_reduction=27616 max_load_reduction_pct=0.842773 moved_to_echo=16173 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 601, 0, 281, 0, 1, 1660, 739, 98, 0, 0, 0, 0, 0, 1, 2, 1, 4092, 2, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4066, 1333, 52, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 2, 0, 0, 782, 0, 0, 1841, 1389, 0, 0, 0, 0, 0, 0, 2590, 0, 0, 0, 0, 0, 313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1813, 531, 2, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 0, 0, 0, 0, 7, 2, 0, 0, 0, 0, 0, 0, 2430, 2, 0, 3773, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3380, 4112, 5455, 4021, 2903, 2647, 4043, 6207] avg=4096.0000 max=6207 max_rank=7 min=2647 min_rank=5 imbalance_ratio=1.515381 max_load_reduction=26561 max_load_reduction_pct=0.810577 moved_to_echo=15800 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1915, 0, 27, 0, 0, 416, 868, 98, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 1566, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 24, 0, 1, 0, 0, 0, 0, 2030, 1396, 0, 0, 0, 0, 0, 0, 2601, 0, 0, 0, 0, 4, 1482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2000, 217, 0, 3586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 8, 2, 0, 0, 0, 0, 0, 0, 610, 0, 1, 1665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3324, 4087, 5596, 3517, 4087, 5805, 4076, 2276] avg=4096.0000 max=5805 max_rank=5 min=2276 min_rank=7 imbalance_ratio=1.417236 max_load_reduction=26963 max_load_reduction_pct=0.822845 moved_to_echo=16244 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 219, 0, 1433, 0, 0, 1421, 1209, 98, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3097, 2183, 2, 0, 0, 595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 234, 47, 0, 0, 0, 383, 0, 0, 2049, 1396, 0, 0, 0, 0, 0, 0, 2602, 0, 0, 0, 0, 6, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2019, 1012, 0, 1942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 11, 2, 0, 0, 0, 0, 0, 0, 2081, 0, 0, 336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4380, 4130, 5877, 4109, 2773, 4973, 4109, 2417] avg=4096.0000 max=5877 max_rank=2 min=2417 min_rank=7 imbalance_ratio=1.434814 max_load_reduction=26891 max_load_reduction_pct=0.820648 moved_to_echo=14272 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 39, 0, 71, 2, 0, 1626, 760, 98, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 393, 1372, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 393, 0, 0, 0, 333, 0, 0, 2043, 1394, 0, 0, 0, 0, 0, 0, 2599, 0, 0, 0, 0, 0, 3488, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 9, 1, 0, 2012, 72, 1, 3160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4, 0, 0, 0, 0, 7, 2, 0, 0, 0, 0, 0, 0, 2379, 0, 0, 2215, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2596, 4098, 1774, 4262, 6088, 5256, 4099, 4595] avg=4096.0000 max=6088 max_rank=4 min=1774 min_rank=2 imbalance_ratio=1.486328 max_load_reduction=26680 max_load_reduction_pct=0.814209 moved_to_echo=20038 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 5, 0, 587, 1, 4, 0, 0, 1628, 1112, 102, 0, 0, 0, 0, 0, 6, 3, 4, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1196, 1995, 0, 2, 0, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 573, 49, 2, 0, 2, 44, 5, 0, 2049, 1392, 0, 0, 0, 0, 0, 0, 2580, 0, 1, 0, 0, 10, 3382, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2011, 0, 3, 1960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4091, 3, 0, 0, 9, 4, 13, 9, 0, 0, 0, 0, 0, 0, 2373, 0, 0, 1326, 1, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3440, 4090, 3329, 4116, 5984, 3974, 4130, 3705] avg=4096.0000 max=5984 max_rank=4 min=3329 min_rank=2 imbalance_ratio=1.460938 max_load_reduction=26784 max_load_reduction_pct=0.817383 moved_to_echo=17793 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[3456, 640, 128, 128, 128, 1664, 128, 0, 16128, 11136, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33920 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 9728, 128, 3328, 128, 128, 11392, 7808, 896, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[20736, 0, 128, 0, 0, 128, 12160, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[16640, 128, 128, 16256, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 18048, 13952, 256, 128, 128, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 15872, 2304, 128, 14848, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 32768, 128, 0, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33920 slack_slots=31616 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[745, 0, 1, 0, 4053, 3955, 0, 0, 0, 0, 0, 0, 1101, 34, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 4096, 0, 2, 1, 2535, 49, 0, 0, 0, 4095, 0, 3718, 0, 0, 0, 99, 0, 0, 0, 136, 0, 0, 0, 4, 0, 0, 16, 4075, 0, 0, 0, 0, 2, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4043, 4043, 0, 1, 0, 0, 4, 0, 978, 248, 17, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2, 0, 4086, 4094, 0, 0, 0, 3110, 22, 0, 0, 0, 4095, 0, 4047, 0, 0, 4, 0, 4, 0, 0, 27, 0, 21, 0, 19, 0, 0, 169, 3708, 1, 0, 15, 0, 0, 0, 4, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 159, 0, 3023, 3944, 0, 0, 0, 0, 0, 0, 4031, 819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 408, 0, 0, 1534, 4096, 0, 2, 0, 3899, 0, 0, 0, 0, 4096, 0, 3136, 0, 0, 0, 82, 0, 7, 0, 12, 0, 0, 0, 0, 0, 0, 1, 3518, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[5, 0, 8, 0, 3728, 4059, 0, 0, 0, 0, 0, 0, 1743, 371, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4072, 4094, 0, 0, 2, 2905, 0, 0, 0, 0, 4091, 0, 3820, 0, 0, 0, 0, 0, 48, 0, 21, 19, 5, 0, 0, 3, 3, 1, 3768, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[5, 0, 93, 0, 4025, 2564, 0, 0, 0, 0, 0, 0, 3370, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 4096, 4096, 0, 920, 0, 1596, 0, 0, 0, 0, 4096, 0, 3793, 0, 0, 0, 0, 0, 0, 0, 98, 8, 0, 0, 0, 0, 0, 4, 3990, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[11, 0, 5, 6, 3678, 4064, 0, 4, 15, 0, 0, 0, 390, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4091, 4088, 0, 590, 0, 2867, 85, 0, 8, 0, 4081, 0, 3831, 0, 0, 0, 101, 7, 0, 7, 0, 6, 0, 0, 0, 2, 0, 725, 4080, 2, 6, 0, 5, 0, 0, 5, 7, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[98, 0, 0, 0, 4038, 4069, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 2, 0, 0, 0, 3, 0, 318, 0, 2, 4044, 4083, 0, 0, 0, 4093, 0, 2, 0, 0, 4078, 12, 4008, 1, 0, 0, 19, 0, 0, 0, 0, 43, 0, 0, 20, 0, 0, 1, 3792, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[11, 0, 5, 0, 4094, 4088, 0, 0, 0, 0, 0, 0, 237, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4065, 4096, 0, 110, 1, 3672, 1, 0, 0, 0, 4096, 0, 4034, 1, 0, 0, 232, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4003, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[745, 0, 1, 0, 4053, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1101, 34, 0, 0, 2610, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3819, 188, 2, 0, 0, 0, 0, 0, 0, 0, 872, 0, 0, 2, 1, 2535, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 276, 0, 3718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 0, 0, 0, 136, 0, 3647, 457, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 16, 4075, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4919, 3745, 4011, 3410, 4043, 4339, 4205, 4096] avg=4096.0000 max=4919 max_rank=0 min=3410 min_rank=3 imbalance_ratio=1.200928 max_load_reduction=27849 max_load_reduction_pct=0.849884 moved_to_echo=16683 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4043, 123, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 978, 248, 17, 0, 2634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3819, 192, 2, 0, 0, 0, 0, 0, 2, 0, 880, 0, 0, 0, 0, 3110, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 276, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 0, 0, 27, 0, 3728, 461, 0, 0, 0, 0, 0, 0, 21, 0, 19, 0, 0, 169, 3708, 1, 111, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 4, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4167, 3881, 4019, 3992, 4345, 4224, 4029, 4111] avg=4096.0000 max=4345 max_rank=4 min=3881 min_rank=1 imbalance_ratio=1.060791 max_load_reduction=28423 max_load_reduction_pct=0.867401 moved_to_echo=16709 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 159, 0, 3023, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4031, 819, 0, 0, 988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 408, 3820, 187, 2, 0, 0, 0, 0, 0, 0, 0, 332, 0, 0, 2, 0, 3899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 276, 0, 3136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 7, 0, 12, 0, 3637, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3518, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3303, 5838, 4417, 4233, 3412, 3911, 3560, 4094] avg=4096.0000 max=5838 max_rank=1 min=3303 min_rank=0 imbalance_ratio=1.425293 max_load_reduction=26930 max_load_reduction_pct=0.821838 moved_to_echo=14977 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 93, 0, 4025, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3370, 4, 0, 0, 2640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3820, 122, 3, 0, 0, 0, 0, 0, 0, 0, 882, 0, 0, 920, 0, 1596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 276, 0, 3793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 8, 2364, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3990, 0, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4201, 6014, 3955, 3398, 4069, 2933, 4105, 4093] avg=4096.0000 max=6014 max_rank=1 min=2933 min_rank=5 imbalance_ratio=1.468262 max_load_reduction=26754 max_load_reduction_pct=0.816467 moved_to_echo=15200 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[11, 0, 5, 0, 4094, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 237, 0, 0, 0, 2620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3820, 194, 2, 0, 0, 0, 0, 0, 0, 0, 876, 0, 0, 110, 1, 3672, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 276, 0, 4034, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 232, 1, 0, 0, 0, 0, 3769, 459, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4003, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4235, 2857, 4018, 4659, 4312, 4461, 4132, 4094] avg=4096.0000 max=4659 max_rank=3 min=2857 min_rank=1 imbalance_ratio=1.137451 max_load_reduction=28109 max_load_reduction_pct=0.857819 moved_to_echo=16999 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 8, 0, 3728, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1743, 371, 0, 0, 2625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3816, 193, 3, 0, 0, 0, 0, 0, 0, 0, 877, 0, 0, 0, 2, 2905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 275, 0, 3820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 21, 19, 3743, 460, 0, 0, 0, 0, 0, 0, 5, 0, 0, 3, 3, 1, 3768, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3864, 4739, 4014, 3784, 4095, 4291, 3890, 4091] avg=4096.0000 max=4739 max_rank=1 min=3784 min_rank=3 imbalance_ratio=1.156982 max_load_reduction=28029 max_load_reduction_pct=0.855377 moved_to_echo=16367 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[98, 0, 0, 0, 4038, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 2607, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 0, 318, 3803, 193, 2, 0, 0, 0, 0, 0, 0, 2, 870, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 275, 12, 4008, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 43, 3752, 457, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 1, 3792, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4260, 2649, 4321, 4965, 4298, 4271, 3923, 4081] avg=4096.0000 max=4965 max_rank=3 min=2649 min_rank=1 imbalance_ratio=1.212158 max_load_reduction=27803 max_load_reduction_pct=0.848480 moved_to_echo=16573 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[11, 0, 5, 6, 3678, 115, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 390, 0, 0, 0, 2640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3811, 198, 3, 0, 0, 0, 0, 0, 0, 0, 870, 0, 0, 590, 0, 2867, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 8, 0, 270, 0, 3831, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 7, 0, 7, 0, 6, 3751, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 725, 4080, 2, 115, 0, 0, 0, 0, 0, 0, 0, 6, 0, 5, 0, 0, 5, 7, 0, 4085, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3819, 3045, 4013, 4327, 4194, 4338, 4924, 4108] avg=4096.0000 max=4924 max_rank=6 min=3045 min_rank=1 imbalance_ratio=1.202148 max_load_reduction=27844 max_load_reduction_pct=0.849731 moved_to_echo=17564 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 0, 11904, 1536, 128, 0, 19456, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 6528, 0, 0, 1664, 128, 24704, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 640, 128, 128, 128, 384, 128, 28416, 3456, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 128, 1024, 30976, 128, 896, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[256, 128, 128, 0, 2304, 128, 30464, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 128, 128, 128, 128, 768, 30592, 1536, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[896, 0, 384, 128, 30720, 1024, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 0, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33792 slack_slots=31744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 218, 3, 0, 0, 0, 0, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 6, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3883, 4096, 4091, 7, 0, 0, 0, 32, 0, 0, 0, 4092, 180, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 3930, 4030] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 333, 0, 0, 13, 0, 0, 0, 129, 0, 0, 0, 45, 0, 0, 0, 0, 0, 7, 0, 2, 4077, 0, 0, 0, 6, 30, 0, 0, 0, 0, 0, 0, 4000, 4096, 3302, 2, 0, 0, 0, 421, 0, 0, 0, 4047, 2597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 1870, 3706] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 1, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4, 0, 4052, 4090, 591, 24, 0, 0, 0, 3594, 0, 6, 0, 4045, 1302, 0, 9, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 3196, 3579] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 381, 10, 0, 27, 1, 0, 0, 24, 0, 0, 0, 1218, 0, 0, 0, 6, 0, 0, 0, 0, 4093, 0, 0, 0, 15, 0, 0, 0, 0, 0, 3, 0, 3775, 4087, 4085, 0, 0, 0, 0, 2458, 0, 0, 0, 4081, 150, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 1533, 2721] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1, 0, 0, 1506, 0, 0, 33, 6, 0, 0, 297, 0, 0, 2, 505, 0, 0, 0, 0, 0, 0, 6, 0, 4095, 3, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 3434, 4094, 2222, 0, 0, 0, 0, 1310, 0, 0, 0, 4091, 2073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 0, 0, 0, 2233, 2829] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 1936, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 329, 0, 0, 0, 0, 0, 12, 0, 0, 4055, 1, 0, 0, 5, 54, 0, 0, 0, 0, 0, 0, 3976, 4089, 3020, 0, 0, 0, 0, 1160, 0, 0, 0, 2707, 2081, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 2283, 2939] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1, 2, 1, 2933, 0, 0, 122, 1, 0, 2, 3, 0, 0, 0, 35, 0, 0, 0, 10, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4075, 4080, 3645, 0, 0, 0, 0, 1662, 0, 6, 0, 3169, 3219, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4093, 0, 0, 0, 1176, 434] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[2, 1, 6, 1565, 5, 0, 0, 0, 0, 4, 200, 0, 6, 3, 117, 17, 0, 12, 0, 0, 0, 0, 0, 4088, 0, 8, 0, 0, 0, 1, 10, 12, 0, 0, 0, 4033, 4065, 4011, 6, 0, 4, 0, 3401, 21, 37, 0, 3429, 2383, 0, 0, 0, 7, 0, 2, 2, 3, 0, 0, 4078, 0, 0, 0, 632, 597] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 59, 0, 0, 0, 0, 2420, 843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 1, 3407, 625, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1007, 0, 591, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3594, 0, 6, 0, 638, 1302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 1, 0, 0, 0, 4090, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3196, 2735, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3322, 4049, 4098, 4098, 1626, 5540, 4104, 5931] avg=4096.0000 max=5931 max_rank=7 min=1626 min_rank=4 imbalance_ratio=1.447998 max_load_reduction=26837 max_load_reduction_pct=0.819000 moved_to_echo=17201 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 218, 3, 0, 0, 0, 2319, 949, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 2, 0, 3447, 599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 965, 0, 4091, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 645, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3930, 3079, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3489, 4051, 4104, 4092, 5063, 857, 4103, 7009] avg=4096.0000 max=7009 max_rank=7 min=857 min_rank=5 imbalance_ratio=1.711182 max_load_reduction=25759 max_load_reduction_pct=0.786102 moved_to_echo=17032 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1936, 0, 0, 0, 34, 2375, 692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 329, 0, 2280, 613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4055, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5, 54, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 988, 0, 3020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1160, 0, 0, 0, 427, 2081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4089, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2283, 2246, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5037, 3222, 4068, 4141, 4008, 3668, 4095, 4529] avg=4096.0000 max=5037 max_rank=0 min=3222 min_rank=1 imbalance_ratio=1.229736 max_load_reduction=27731 max_load_reduction_pct=0.846283 moved_to_echo=16300 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 333, 0, 0, 13, 0, 2389, 873, 0, 0, 0, 0, 0, 0, 0, 0, 129, 0, 0, 0, 45, 0, 3409, 617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 2, 4077, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 30, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 994, 0, 3302, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 421, 0, 0, 0, 638, 2597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1870, 2832, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3608, 4200, 4087, 4117, 4298, 3656, 4100, 4702] avg=4096.0000 max=4702 max_rank=7 min=3608 min_rank=0 imbalance_ratio=1.147949 max_load_reduction=28066 max_load_reduction_pct=0.856506 moved_to_echo=16756 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 1506, 0, 0, 33, 6, 2051, 666, 0, 0, 0, 0, 0, 0, 0, 0, 297, 0, 0, 2, 505, 0, 3446, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 1, 0, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 854, 0, 2222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1310, 0, 0, 0, 645, 2073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2233, 2162, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4263, 4779, 4102, 4027, 3077, 4028, 4097, 4395] avg=4096.0000 max=4779 max_rank=1 min=3077 min_rank=4 imbalance_ratio=1.166748 max_load_reduction=27989 max_load_reduction_pct=0.854156 moved_to_echo=15597 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 381, 10, 0, 27, 1, 2255, 641, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 1218, 0, 3438, 582, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 938, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2458, 0, 0, 0, 643, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 4087, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1533, 2079, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3315, 5262, 4100, 4100, 5026, 3251, 4102, 3612] avg=4096.0000 max=5262 max_rank=1 min=3251 min_rank=5 imbalance_ratio=1.284668 max_load_reduction=27506 max_load_reduction_pct=0.839417 moved_to_echo=15991 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 2, 1, 2933, 0, 0, 122, 1, 2434, 102, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 35, 0, 2669, 628, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1013, 0, 3645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1662, 0, 6, 0, 500, 3219, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4080, 4, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1176, 332, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5596, 3337, 4105, 4089, 4659, 5387, 4085, 1510] avg=4096.0000 max=5596 max_rank=0 min=1510 min_rank=7 imbalance_ratio=1.366211 max_load_reduction=27172 max_load_reduction_pct=0.829224 moved_to_echo=15641 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2, 1, 6, 1565, 5, 0, 0, 0, 2414, 145, 0, 0, 0, 0, 0, 0, 0, 4, 200, 0, 6, 3, 117, 17, 2894, 625, 2, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 4088, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 1, 10, 12, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 994, 0, 4011, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3401, 21, 37, 0, 535, 2383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 2, 2, 3, 4065, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 632, 448, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4138, 3868, 4104, 4104, 5011, 6381, 4082, 1080] avg=4096.0000 max=6381 max_rank=5 min=1080 min_rank=7 imbalance_ratio=1.557861 max_load_reduction=26387 max_load_reduction_pct=0.805267 moved_to_echo=16554 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 8960, 128, 0, 256, 128, 18688, 4992, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 0, 0, 16896, 16000, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 14080, 128, 128, 0, 4736, 14080, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 7808, 0, 25088, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[0, 128, 768, 0, 128, 128, 2304, 128, 25088, 4864, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 128, 128, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 128, 128, 128, 128, 32640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33792 slack_slots=31744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4038, 0, 0, 4096, 7, 1, 13, 0, 0, 4076, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4061, 0, 35, 0, 0, 0, 0, 0, 2803, 13, 4030, 0, 0, 56, 0, 0, 0, 1239, 1, 0, 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4075, 0, 5, 4096, 11, 16, 148, 0, 0, 4074, 0, 0, 0, 25, 0, 0, 4092, 0, 0, 0, 4094, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 152, 14, 4013, 0, 0, 4, 0, 0, 0, 3590, 69, 0, 0, 0, 0, 188, 0, 0, 0, 0, 2, 1, 0, 2, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 5, 0, 4069, 0, 1, 4095, 2, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 4095, 1, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 1678, 91, 4041, 0, 0, 0, 1, 0, 0, 2395, 6, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 1, 0, 4091, 0, 0, 4096, 0, 0, 4, 0, 0, 4043, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 1325, 0, 0, 0, 0, 0, 219, 61, 3868, 0, 0, 1, 0, 0, 0, 1634, 42, 0, 0, 0, 0, 1095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 3757, 0, 0, 4071, 0, 59, 512, 0, 0, 4096, 0, 2, 0, 30, 0, 0, 4095, 0, 0, 0, 4091, 0, 0, 4092, 6, 515, 0, 0, 0, 0, 0, 2690, 0, 2816, 1, 0, 2, 0, 0, 0, 1015, 0, 4, 0, 0, 0, 914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 1, 4083, 1, 0, 4095, 0, 0, 2, 0, 0, 3956, 0, 0, 0, 0, 1, 0, 4085, 0, 0, 0, 4094, 0, 0, 3912, 1, 6, 0, 0, 1, 0, 0, 178, 300, 3793, 0, 0, 23, 0, 4, 0, 3925, 85, 0, 0, 0, 0, 215, 0, 0, 2, 1, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 11, 3964, 0, 10, 4077, 0, 0, 0, 0, 0, 4051, 0, 13, 0, 94, 5, 0, 4084, 0, 0, 0, 4076, 0, 0, 3753, 0, 298, 11, 0, 10, 2, 0, 2231, 355, 3618, 9, 2, 0, 13, 0, 0, 1960, 25, 0, 4, 0, 0, 1, 0, 3, 36, 7, 0, 26, 0, 0, 3, 0, 7, 0, 9, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 3774, 0, 0, 4096, 0, 86, 34, 0, 0, 4092, 0, 0, 0, 91, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4093, 0, 1, 0, 0, 0, 0, 0, 2414, 0, 4087, 0, 0, 5, 0, 0, 0, 1122, 0, 0, 0, 0, 0, 681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4038, 0, 0, 109, 7, 0, 0, 0, 0, 0, 0, 0, 0, 1, 13, 0, 0, 4001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3854, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2803, 13, 2592, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1239, 1, 0, 0, 0, 1438, 296, 207, 75, 27, 0, 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, 3691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4154, 4015, 4070, 3889, 5464, 3283, 3797, 4096] avg=4096.0000 max=5464 max_rank=4 min=3283 min_rank=5 imbalance_ratio=1.333984 max_load_reduction=27304 max_load_reduction_pct=0.833252 moved_to_echo=16640 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 0, 4069, 0, 1, 109, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3887, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1678, 91, 2599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2395, 6, 0, 0, 0, 1442, 296, 209, 75, 27, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4186, 4018, 4069, 3890, 4368, 4451, 3691, 4095] avg=4096.0000 max=4451 max_rank=5 min=3691 min_rank=6 imbalance_ratio=1.086670 max_load_reduction=28317 max_load_reduction_pct=0.864166 moved_to_echo=16605 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3757, 0, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 512, 0, 0, 4020, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3884, 6, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2690, 0, 1811, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 0, 4, 0, 0, 1005, 294, 208, 76, 27, 0, 0, 0, 0, 914, 0, 0, 0, 0, 0, 0, 3668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3866, 4593, 4094, 4405, 4504, 2629, 4582, 4095] avg=4096.0000 max=4593 max_rank=1 min=2629 min_rank=5 imbalance_ratio=1.121338 max_load_reduction=28175 max_load_reduction_pct=0.859833 moved_to_echo=15810 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 4091, 0, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3887, 0, 1325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 219, 61, 2488, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1634, 42, 0, 0, 0, 1380, 296, 209, 75, 27, 0, 0, 0, 0, 1095, 0, 0, 0, 0, 0, 0, 3691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4201, 3972, 4069, 5212, 2769, 3663, 4786, 4096] avg=4096.0000 max=5212 max_rank=3 min=2769 min_rank=4 imbalance_ratio=1.272461 max_load_reduction=27556 max_load_reduction_pct=0.840942 moved_to_echo=15314 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3774, 0, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 34, 0, 0, 4017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3885, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2414, 0, 2629, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1122, 0, 0, 0, 0, 1458, 296, 208, 75, 27, 0, 0, 0, 0, 681, 0, 0, 0, 0, 0, 0, 3691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3883, 4137, 4160, 3886, 5048, 3186, 4372, 4096] avg=4096.0000 max=5048 max_rank=4 min=3186 min_rank=5 imbalance_ratio=1.232422 max_load_reduction=27720 max_load_reduction_pct=0.845947 moved_to_echo=16702 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4075, 0, 5, 109, 11, 0, 0, 0, 0, 0, 0, 0, 0, 16, 148, 0, 0, 3999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 14, 2581, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3590, 69, 0, 0, 0, 1432, 296, 209, 75, 27, 0, 0, 0, 0, 188, 0, 0, 0, 0, 2, 1, 3691, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4200, 4163, 4092, 3888, 2751, 5698, 3882, 4094] avg=4096.0000 max=5698 max_rank=5 min=2751 min_rank=4 imbalance_ratio=1.391113 max_load_reduction=27070 max_load_reduction_pct=0.826111 moved_to_echo=16425 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 4083, 1, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3713, 1, 6, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 300, 2440, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3925, 85, 0, 0, 0, 1353, 296, 199, 73, 27, 0, 0, 0, 0, 215, 0, 0, 2, 1, 0, 0, 3690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4194, 3885, 4068, 3721, 2941, 5962, 3908, 4089] avg=4096.0000 max=5962 max_rank=5 min=2941 min_rank=4 imbalance_ratio=1.455566 max_load_reduction=26806 max_load_reduction_pct=0.818054 moved_to_echo=16900 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 11, 3964, 0, 10, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3972, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 5, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3558, 0, 298, 11, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2231, 355, 2324, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 1960, 25, 0, 4, 0, 1294, 297, 195, 79, 29, 0, 0, 0, 0, 1, 0, 3, 36, 7, 0, 26, 3677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 7, 0, 9, 0, 4084, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 3985, 4146, 3877, 4923, 3896, 3750, 4107] avg=4096.0000 max=4923 max_rank=4 min=3750 min_rank=6 imbalance_ratio=1.201904 max_load_reduction=27845 max_load_reduction_pct=0.849762 moved_to_echo=16676 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 0, 16896, 256, 128, 128, 0, 10880, 2432, 1664, 640, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 12416, 896, 19584, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 3328, 0, 128, 128, 128, 128, 128, 29568, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 31872, 128, 128, 896, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 30592, 128, 2304, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 768, 0, 0, 32000, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 128, 0, 0, 0, 128, 0, 32640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 128, 128, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 5, 0, 4070, 1, 0, 287, 1720, 2, 4081, 2, 0, 0, 0, 4078, 3, 0, 475, 0, 0, 0, 0, 106, 0, 2, 1, 2, 3, 0, 0, 0, 3913, 0, 4093, 0, 0, 0, 2, 84, 0, 4017, 0, 0, 0, 0, 1734, 4082, 0, 2, 0, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 4, 0, 0, 4056, 0, 0, 1817, 389, 0, 3947, 0, 0, 1, 0, 4095, 0, 0, 3167, 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 0, 961, 0, 4096, 0, 0, 0, 0, 30, 0, 4033, 0, 0, 0, 0, 2069, 4094, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 965, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 2453, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2792, 0, 4096, 0, 0, 0, 0, 2, 0, 2839, 0, 0, 0, 0, 3236, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4094, 0, 0, 61, 269, 0, 3730, 5, 1, 6, 0, 4087, 0, 0, 9, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 2, 0, 1, 3699, 0, 4092, 0, 0, 0, 0, 928, 0, 4089, 1, 0, 0, 1, 3570, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2, 4, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 341, 0, 0, 4081, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3712, 0, 4096, 0, 0, 0, 0, 293, 0, 3884, 0, 0, 0, 0, 4073, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 4025, 0, 1, 1083, 113, 15, 4035, 0, 7, 0, 0, 4074, 1, 0, 619, 0, 0, 0, 15, 7, 32, 0, 2, 7, 6, 5, 0, 0, 2306, 0, 4083, 4, 0, 0, 8, 1015, 0, 3953, 5, 12, 4, 0, 3230, 4044, 0, 0, 6, 9, 11, 9, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 13, 0, 3, 0, 0, 0, 2, 0, 4088, 0, 0, 265, 0, 0, 4038, 0, 0, 0, 1, 4091, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3819, 1, 4094, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 4, 4096, 0, 0, 1679, 881, 0, 4083, 18, 0, 0, 0, 4094, 0, 0, 360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2902, 0, 4096, 0, 0, 0, 0, 9, 0, 4068, 0, 0, 0, 0, 2379, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3, 0, 0, 3252, 970, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 4056, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 1817, 389, 0, 2810, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3207, 0, 0, 3167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 2043, 1137, 873, 0, 0, 0, 0, 0, 0, 961, 0, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 781, 0, 0, 0, 0, 2069, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4225, 4073, 5017, 6374, 4058, 2044, 2880, 4097] avg=4096.0000 max=6374 max_rank=3 min=2044 min_rank=5 imbalance_ratio=1.556152 max_load_reduction=26394 max_load_reduction_pct=0.805481 moved_to_echo=13079 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 0, 0, 0, 0, 3297, 969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4094, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 61, 269, 0, 2655, 5, 1, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3200, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 2, 0, 2041, 1075, 872, 0, 0, 0, 0, 0, 1, 3699, 0, 1082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 928, 0, 792, 1, 0, 0, 1, 3570, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4269, 4114, 2997, 3209, 4006, 4782, 5292, 4099] avg=4096.0000 max=5292 max_rank=6 min=2997 min_rank=2 imbalance_ratio=1.291992 max_load_reduction=27476 max_load_reduction_pct=0.838501 moved_to_echo=18179 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 2289, 970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 965, 0, 0, 2916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3207, 0, 0, 2453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2043, 1180, 874, 0, 0, 0, 0, 0, 0, 2792, 0, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 550, 0, 0, 0, 0, 3236, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3259, 4108, 3881, 5660, 4098, 3875, 3788, 4099] avg=4096.0000 max=5660 max_rank=3 min=3259 min_rank=0 imbalance_ratio=1.381836 max_load_reduction=27108 max_load_reduction_pct=0.827271 moved_to_echo=15860 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2, 4, 0, 3131, 970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 341, 0, 0, 2905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2043, 1176, 873, 0, 0, 0, 0, 0, 0, 3712, 0, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 0, 753, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4106, 3246, 3203, 4093, 4795, 5119, 4099] avg=4096.0000 max=5119 max_rank=6 min=3203 min_rank=3 imbalance_ratio=1.249756 max_load_reduction=27649 max_load_reduction_pct=0.843781 moved_to_echo=18106 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 13, 0, 3, 3299, 970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4088, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 265, 0, 0, 2874, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3203, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2042, 1164, 873, 0, 0, 0, 0, 0, 0, 3819, 1, 1082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 793, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4285, 4102, 3140, 3273, 4079, 4902, 4888, 4099] avg=4096.0000 max=4902 max_rank=5 min=3140 min_rank=2 imbalance_ratio=1.196777 max_load_reduction=27866 max_load_reduction_pct=0.850403 moved_to_echo=17968 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2, 0, 0, 3239, 969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4070, 1, 0, 12, 0, 0, 0, 0, 0, 0, 0, 287, 1720, 2, 2905, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3193, 3, 0, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 0, 2, 1, 2, 3, 0, 0, 2042, 1176, 870, 0, 0, 0, 0, 0, 0, 3913, 0, 1082, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 778, 0, 0, 0, 0, 1734, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 2, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4210, 4088, 4916, 3671, 4202, 4997, 2596, 4088] avg=4096.0000 max=4997 max_rank=5 min=2596 min_rank=6 imbalance_ratio=1.219971 max_load_reduction=27771 max_load_reduction_pct=0.847504 moved_to_echo=15883 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 3190, 972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 4025, 0, 1, 18, 0, 0, 0, 0, 0, 0, 0, 1083, 113, 15, 2869, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3177, 1, 0, 619, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 7, 32, 0, 2, 7, 6, 5, 0, 2042, 1166, 872, 0, 0, 0, 0, 0, 0, 2306, 0, 1069, 4, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 0, 763, 5, 12, 4, 0, 3230, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 0, 6, 9, 11, 9, 2, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4162, 4064, 4087, 3812, 4139, 3387, 5029, 4088] avg=4096.0000 max=5029 max_rank=6 min=3387 min_rank=5 imbalance_ratio=1.227783 max_load_reduction=27739 max_load_reduction_pct=0.846527 moved_to_echo=16643 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 3280, 970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 4096, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 1679, 881, 0, 2906, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3206, 0, 0, 360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2043, 1177, 873, 0, 0, 0, 0, 0, 1, 2902, 0, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 788, 0, 0, 0, 0, 2379, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4251, 4113, 5484, 3566, 4093, 3986, 3176, 4099] avg=4096.0000 max=5484 max_rank=2 min=3176 min_rank=6 imbalance_ratio=1.338867 max_load_reduction=27284 max_load_reduction_pct=0.832642 moved_to_echo=15354 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[256, 128, 128, 128, 128, 128, 128, 0, 16384, 9344, 7040, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[6528, 3456, 128, 22912, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 0, 128, 128, 128, 25088, 7808, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 24192, 128, 8704, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[2432, 0, 6016, 128, 128, 128, 128, 24448, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[25600, 128, 0, 7168, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 128, 32640, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[32768, 0, 128, 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33792 slack_slots=31744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 3116, 0, 0, 0, 1, 0, 0, 362, 2966, 0, 2, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4085, 4096, 3843, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 31, 0, 0, 0, 0, 3314, 0, 0, 0, 4094, 0, 1043, 0, 0, 0, 4095, 0, 0, 1709, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 1873, 1, 0, 0, 122, 0, 0, 412, 2311, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 4095, 4096, 3920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 2035, 0, 0, 0, 4096, 0, 1836, 0, 0, 0, 4045, 0, 0, 3888, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 1640, 10, 1, 0, 106, 0, 0, 1240, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4091, 4096, 4095, 0, 0, 0, 1, 0, 0, 0, 4, 0, 0, 0, 0, 8, 0, 0, 0, 0, 458, 0, 0, 0, 4095, 0, 2883, 1, 0, 0, 4085, 0, 0, 1859, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 1597, 0, 1, 0, 0, 0, 2, 781, 3786, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 12, 0, 8, 0, 4094, 4076, 4094, 0, 0, 2, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 1870, 0, 2, 0, 4089, 0, 2610, 0, 0, 0, 4064, 0, 0, 1667, 0, 1, 0, 0, 0, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 1971, 0, 0, 0, 58, 0, 0, 274, 3170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4094, 4096, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 2316, 0, 0, 0, 4096, 0, 676, 0, 0, 0, 4032, 0, 0, 3822, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 3646, 16, 3, 0, 88, 13, 0, 4, 3731, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4052, 4087, 4089, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 698, 0, 0, 0, 4096, 0, 58, 1, 0, 0, 3989, 0, 0, 4081, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 1659, 77, 0, 0, 102, 0, 10, 376, 3989, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 10, 0, 3903, 4055, 4084, 0, 0, 2, 29, 0, 6, 0, 4, 0, 0, 0, 0, 4, 0, 4, 16, 9, 1187, 0, 0, 1, 4069, 0, 1529, 24, 0, 0, 4065, 1, 25, 3390, 11, 0, 64, 8, 30, 15] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1, 1682, 11, 0, 0, 7, 0, 0, 71, 3943, 1, 0, 0, 0, 0, 1, 1, 17, 0, 173, 0, 4, 0, 0, 4003, 4096, 4091, 2, 0, 0, 1, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3621, 0, 0, 0, 4094, 1, 808, 0, 0, 0, 3797, 0, 0, 2330, 0, 0, 0, 1, 0, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1640, 10, 1, 0, 106, 0, 0, 1887, 0, 0, 0, 0, 0, 0, 0, 1240, 4094, 0, 0, 0, 0, 0, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4052, 21, 0, 0, 0, 0, 0, 0, 46, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 458, 0, 0, 0, 2130, 44, 0, 0, 0, 0, 0, 0, 0, 0, 2883, 1, 0, 0, 2705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1859, 0, 0, 0, 0, 0, 0, 1223, 28, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3644, 5491, 4074, 4142, 4086, 2632, 5589, 3110] avg=4096.0000 max=5589 max_rank=6 min=2632 min_rank=5 imbalance_ratio=1.364502 max_load_reduction=27179 max_load_reduction_pct=0.829437 moved_to_echo=15417 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3116, 0, 0, 0, 1, 0, 0, 1884, 0, 0, 0, 0, 0, 0, 0, 362, 2966, 0, 2, 0, 0, 0, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4052, 21, 0, 0, 0, 0, 0, 0, 47, 0, 3843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 31, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3314, 0, 0, 0, 2127, 44, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 0, 0, 0, 2712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1709, 0, 1, 0, 0, 0, 0, 1226, 27, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5001, 3487, 4079, 3890, 4108, 5485, 3755, 2963] avg=4096.0000 max=5485 max_rank=5 min=2963 min_rank=7 imbalance_ratio=1.339111 max_load_reduction=27283 max_load_reduction_pct=0.832611 moved_to_echo=16311 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1873, 1, 0, 0, 122, 0, 0, 1889, 0, 0, 0, 0, 0, 0, 0, 412, 2311, 0, 0, 0, 0, 0, 0, 155, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 4052, 21, 0, 0, 0, 0, 0, 0, 46, 0, 3920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2035, 0, 0, 0, 2132, 44, 0, 0, 0, 0, 0, 0, 0, 0, 1836, 0, 0, 0, 2679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3888, 0, 0, 0, 0, 0, 0, 1211, 28, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3885, 2878, 4083, 3966, 4103, 4211, 4515, 5127] avg=4096.0000 max=5127 max_rank=7 min=2878 min_rank=1 imbalance_ratio=1.251709 max_load_reduction=27641 max_load_reduction_pct=0.843536 moved_to_echo=17956 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1971, 0, 0, 0, 58, 0, 0, 1888, 0, 0, 0, 0, 0, 0, 0, 274, 3170, 0, 0, 0, 0, 0, 0, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4052, 22, 0, 0, 0, 0, 0, 0, 46, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2316, 0, 0, 0, 2132, 44, 0, 0, 0, 0, 0, 0, 0, 0, 676, 0, 0, 0, 2670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3822, 0, 0, 0, 0, 0, 0, 1207, 28, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3917, 3599, 4078, 4143, 4136, 4492, 3346, 5057] avg=4096.0000 max=5057 max_rank=7 min=3346 min_rank=6 imbalance_ratio=1.234619 max_load_reduction=27711 max_load_reduction_pct=0.845673 moved_to_echo=17031 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3646, 16, 3, 0, 88, 13, 0, 1869, 0, 0, 0, 0, 0, 0, 0, 4, 3731, 1, 3, 0, 0, 0, 0, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4043, 22, 0, 0, 0, 0, 0, 0, 46, 0, 4089, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 698, 0, 0, 0, 2110, 44, 0, 0, 0, 0, 0, 0, 0, 0, 58, 1, 0, 0, 2642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 1194, 27, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5635, 3892, 4067, 4194, 4125, 2852, 2701, 5302] avg=4096.0000 max=5635 max_rank=0 min=2701 min_rank=6 imbalance_ratio=1.375732 max_load_reduction=27133 max_load_reduction_pct=0.828033 moved_to_echo=14980 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1597, 0, 1, 0, 0, 0, 2, 1888, 0, 0, 0, 0, 0, 0, 0, 781, 3786, 0, 0, 0, 0, 0, 0, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 12, 0, 8, 0, 4032, 22, 0, 0, 0, 0, 0, 0, 46, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 1870, 0, 2, 0, 2132, 44, 0, 0, 0, 0, 0, 0, 0, 0, 2610, 0, 0, 0, 2691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1667, 0, 1, 0, 0, 0, 4, 1217, 28, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3488, 4723, 4075, 4142, 4070, 4052, 5301, 2917] avg=4096.0000 max=5301 max_rank=6 min=2917 min_rank=7 imbalance_ratio=1.294189 max_load_reduction=27467 max_load_reduction_pct=0.838226 moved_to_echo=16340 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 1682, 11, 0, 0, 7, 0, 0, 1846, 0, 0, 0, 0, 0, 0, 0, 71, 3943, 1, 0, 0, 0, 0, 1, 146, 0, 0, 0, 0, 0, 0, 0, 1, 17, 0, 173, 0, 4, 0, 0, 4052, 21, 0, 0, 0, 0, 0, 0, 46, 0, 4091, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3621, 0, 0, 0, 2084, 44, 0, 0, 0, 0, 0, 0, 0, 1, 808, 0, 0, 0, 2514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2330, 0, 0, 0, 1, 0, 5, 1137, 27, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3547, 4162, 4268, 4140, 4079, 5749, 3323, 3500] avg=4096.0000 max=5749 max_rank=5 min=3323 min_rank=6 imbalance_ratio=1.403564 max_load_reduction=27019 max_load_reduction_pct=0.824554 moved_to_echo=16651 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1659, 77, 0, 0, 102, 0, 10, 1803, 0, 0, 0, 0, 0, 0, 0, 376, 3989, 0, 0, 0, 0, 0, 10, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4012, 22, 0, 0, 0, 0, 0, 0, 36, 0, 4084, 0, 0, 2, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4, 0, 0, 0, 0, 4, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 4, 16, 9, 1187, 0, 0, 1, 2035, 43, 0, 0, 0, 0, 0, 0, 0, 0, 1529, 24, 0, 0, 2684, 1, 0, 0, 0, 0, 0, 0, 0, 0, 25, 3390, 11, 0, 64, 8, 30, 15, 1220, 29, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3651, 4536, 4044, 4151, 4061, 3295, 4238, 4792] avg=4096.0000 max=4792 max_rank=7 min=3295 min_rank=5 imbalance_ratio=1.169922 max_load_reduction=27976 max_load_reduction_pct=0.853760 moved_to_echo=16386 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 17280, 128, 128, 0, 512, 128, 128, 14976, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 15616, 0, 128, 128, 16896, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 22784, 128, 128, 128, 128, 128, 128, 9728, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 11520, 128, 0, 0, 21376, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3584, 28032, 128, 128, 0, 0, 0, 128, 1280, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 256, 128, 128, 128, 0, 32384, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 0, 0, 0, 256, 32640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[384, 0, 32384, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 4096, 3609, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 130, 0, 0, 4096, 1612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3432, 3863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3736, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[87, 0, 1, 0, 4091, 0, 4090, 3034, 0, 0, 4095, 1, 0, 1, 0, 0, 0, 81, 0, 9, 4049, 1136, 0, 0, 0, 1, 254, 0, 1, 0, 4, 0, 3670, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 1, 0, 0, 4085, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 4096, 3457, 0, 0, 4096, 0, 63, 0, 0, 0, 0, 0, 0, 0, 4096, 436, 0, 0, 0, 0, 121, 0, 0, 0, 36, 0, 3994, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 4094, 3334, 0, 0, 4096, 0, 1, 0, 3, 1, 1, 0, 0, 0, 4095, 2244, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 2737, 4071, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 3974, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4087, 6, 4091, 3818, 0, 0, 4091, 3, 7, 2, 0, 0, 0, 2, 0, 0, 4095, 33, 0, 0, 0, 0, 0, 0, 9, 2, 0, 0, 4011, 4075, 1, 0, 0, 0, 0, 7, 0, 0, 0, 283, 0, 0, 5, 36, 0, 2, 0, 0, 2, 0, 0, 26, 0, 2, 0, 0, 0, 0, 4072, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3989, 0, 4068, 3801, 0, 0, 4066, 8, 0, 11, 0, 0, 0, 125, 12, 0, 4073, 2833, 0, 0, 123, 0, 25, 0, 0, 6, 0, 9, 1067, 3785, 8, 5, 0, 0, 0, 70, 0, 0, 0, 531, 0, 1, 38, 3, 0, 8, 18, 0, 12, 0, 15, 19, 13, 0, 0, 0, 0, 0, 4026, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 0, 4096, 3901, 0, 0, 4096, 0, 7, 0, 0, 0, 1, 62, 0, 0, 4088, 1791, 0, 0, 0, 0, 0, 0, 4, 0, 23, 0, 2612, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3569, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[11, 0, 0, 0, 4094, 1, 4096, 1552, 0, 0, 4096, 0, 2, 0, 1, 0, 0, 3, 0, 0, 4096, 3254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3371, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 773, 0, 0, 3334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 1, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2382, 2244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3321, 707, 10, 0, 0, 0, 0, 0, 2737, 986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 2378, 1586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 17, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3974, 0, 127, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4092, 4627, 4041, 3723, 3966, 4102, 4110] avg=4096.0000 max=4627 max_rank=2 min=3723 min_rank=4 imbalance_ratio=1.129639 max_load_reduction=28141 max_load_reduction_pct=0.858795 moved_to_echo=15901 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 773, 0, 0, 3609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 0, 2383, 1612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3323, 670, 10, 0, 0, 0, 0, 0, 3432, 937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2256, 1586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3736, 0, 127, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4382, 4089, 4125, 4003, 4369, 3842, 4086, 3872] avg=4096.0000 max=4382 max_rank=0 min=3842 min_rank=5 imbalance_ratio=1.069824 max_load_reduction=28386 max_load_reduction_pct=0.866272 moved_to_echo=16169 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[87, 0, 1, 0, 772, 0, 0, 3034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 9, 2355, 1136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 254, 0, 1, 0, 4, 0, 3319, 707, 10, 0, 0, 0, 0, 0, 3670, 987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2379, 1568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 4085, 0, 126, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3894, 4088, 3581, 4296, 4657, 3948, 4082, 4222] avg=4096.0000 max=4657 max_rank=4 min=3581 min_rank=2 imbalance_ratio=1.136963 max_load_reduction=28111 max_load_reduction_pct=0.857880 moved_to_echo=16909 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 773, 0, 0, 3457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2383, 436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, 0, 0, 0, 36, 0, 3322, 711, 10, 0, 0, 0, 0, 0, 3994, 992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 78, 0, 0, 2391, 1586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 127, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4230, 4150, 2819, 4200, 4986, 4065, 4086, 4232] avg=4096.0000 max=4986 max_rank=4 min=2819 min_rank=2 imbalance_ratio=1.217285 max_load_reduction=27782 max_load_reduction_pct=0.847839 moved_to_echo=17369 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 771, 6, 0, 3818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 3, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2382, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2, 0, 0, 3316, 707, 10, 0, 0, 0, 0, 0, 4011, 988, 1, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 283, 0, 0, 5, 36, 2380, 1586, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 26, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4072, 0, 127, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4595, 4094, 2417, 4044, 5007, 4290, 4111, 4210] avg=4096.0000 max=5007 max_rank=4 min=2417 min_rank=2 imbalance_ratio=1.222412 max_load_reduction=27761 max_load_reduction_pct=0.847198 moved_to_echo=17618 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 772, 0, 0, 3901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 62, 0, 0, 2378, 1791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 23, 0, 3320, 711, 10, 0, 0, 0, 0, 0, 2612, 992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 330, 0, 0, 0, 0, 2393, 1583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3569, 0, 127, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4673, 4094, 4232, 4068, 3604, 4306, 4086, 3705] avg=4096.0000 max=4673 max_rank=0 min=3604 min_rank=4 imbalance_ratio=1.140869 max_load_reduction=28095 max_load_reduction_pct=0.857391 moved_to_echo=15701 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 749, 0, 0, 3801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 8, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 12, 0, 2357, 2833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 0, 25, 0, 0, 6, 0, 9, 3240, 660, 11, 0, 0, 0, 0, 0, 1067, 909, 8, 5, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 0, 1, 38, 3, 2216, 1584, 0, 0, 0, 0, 0, 0, 0, 8, 18, 0, 12, 0, 15, 19, 4057, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 4026, 0, 132, 14, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4550, 4071, 5327, 4074, 2059, 4373, 4129, 4185] avg=4096.0000 max=5327 max_rank=2 min=2059 min_rank=4 imbalance_ratio=1.300537 max_load_reduction=27441 max_load_reduction_pct=0.837433 moved_to_echo=14746 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[11, 0, 0, 0, 773, 1, 0, 1552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2383, 3254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3321, 711, 10, 0, 0, 0, 0, 0, 3371, 992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2392, 1586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 127, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2337, 4090, 5640, 4042, 4363, 3978, 4086, 4232] avg=4096.0000 max=5640 max_rank=2 min=2337 min_rank=0 imbalance_ratio=1.376953 max_load_reduction=27128 max_load_reduction_pct=0.827881 moved_to_echo=16659 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 512, 128, 128, 19072, 13440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 0, 1280, 128, 128, 128, 128, 18816, 12672, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[24960, 7808, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 512, 0, 128, 128, 128, 128, 26496, 5632, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 0, 6272, 128, 0, 26624, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 0, 0, 31744, 0, 1024, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[0, 128, 128, 128, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 0, 32768, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[7, 2646, 0, 3682, 0, 0, 0, 0, 0, 264, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3913, 0, 3047, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 14, 0, 2718, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 3065, 0, 1015, 0, 0, 0, 0, 0, 3445, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3954, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 38, 0, 1, 0, 0, 1, 0, 0, 0, 0, 3483, 1, 2656, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 88, 0, 2724, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 4068, 0, 1306, 0, 0, 0, 0, 0, 3847, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3456, 0, 0, 0, 0, 127, 0, 0, 0, 0, 0, 0, 0, 1588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1860, 0, 4090, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 128, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[42, 3327, 0, 2738, 2, 0, 0, 0, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4078, 3789, 1, 1, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 4, 3, 0, 0, 4, 1, 0, 0, 0, 0, 3660, 0, 249, 0, 0, 4096, 0, 42, 0, 0, 0, 0, 0, 0, 12, 23, 1, 539, 0, 2009, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 1697, 0, 4087, 0, 1, 0, 0, 2, 3686, 0, 0, 0, 0, 0, 2, 0, 4085, 1, 4079, 4092, 0, 1, 2, 0, 21, 0, 2, 0, 0, 0, 8, 0, 18, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2725, 0, 2434, 2, 0, 4086, 0, 0, 0, 0, 0, 1, 3, 0, 3, 4, 0, 0, 0, 1724, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 3024, 0, 1335, 0, 0, 0, 0, 0, 3624, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4087, 4075, 0, 0, 0, 0, 509, 0, 0, 7, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3523, 0, 2895, 0, 0, 4096, 0, 10, 0, 0, 0, 3, 0, 0, 3, 1, 0, 0, 0, 1434, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 3327, 9, 3898, 0, 27, 0, 0, 14, 641, 5, 0, 6, 0, 0, 16, 0, 4052, 0, 3984, 4075, 0, 250, 0, 5, 180, 0, 8, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 15, 0, 0, 3841, 0, 410, 14, 21, 4075, 0, 0, 0, 0, 0, 0, 11, 0, 459, 0, 6, 0, 0, 3411, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 2157, 0, 1360, 0, 0, 0, 0, 0, 3691, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3279, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3047, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 89, 31, 0, 159, 0, 2458, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1745, 0, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3445, 0, 0, 0, 0, 0, 0, 1241, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 231, 3954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 3991, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 1, 0, 0, 1, 0, 3755, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1717, 1, 2656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 88, 0, 2724, 0, 1766, 110, 79, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2760, 4690, 4185, 3995, 3900, 4374, 4092, 4772] avg=4096.0000 max=4772 max_rank=7 min=2760 min_rank=0 imbalance_ratio=1.165039 max_load_reduction=27996 max_load_reduction_pct=0.854370 moved_to_echo=17138 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[7, 1506, 0, 3682, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 264, 0, 0, 0, 0, 0, 0, 1071, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 231, 3958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3991, 0, 0, 0, 0, 0, 0, 0, 0, 224, 0, 0, 0, 0, 0, 0, 3755, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1929, 0, 3047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 14, 0, 2718, 0, 1984, 110, 69, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5195, 1339, 4189, 3991, 4084, 4976, 4092, 4902] avg=4096.0000 max=5195 max_rank=0 min=1339 min_rank=1 imbalance_ratio=1.268311 max_load_reduction=27573 max_load_reduction_pct=0.841461 moved_to_echo=18054 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 2314, 0, 1306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3847, 0, 0, 0, 0, 0, 0, 1648, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 231, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 0, 0, 0, 0, 0, 3991, 0, 0, 0, 0, 0, 0, 0, 0, 1588, 0, 0, 0, 0, 0, 0, 3755, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 917, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 128, 0, 943, 110, 106, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3620, 5499, 3687, 4118, 5448, 5007, 4092, 1297] avg=4096.0000 max=5499 max_rank=1 min=1297 min_rank=7 imbalance_ratio=1.342529 max_load_reduction=27269 max_load_reduction_pct=0.832184 moved_to_echo=15844 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[42, 1894, 0, 2738, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 0, 0, 0, 0, 0, 1347, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 230, 3789, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 3986, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 0, 0, 4, 1, 0, 3738, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1804, 0, 249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 23, 1, 539, 0, 2009, 0, 1856, 110, 86, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4676, 5393, 4021, 4000, 3855, 2053, 4134, 4636] avg=4096.0000 max=5393 max_rank=1 min=2053 min_rank=5 imbalance_ratio=1.316650 max_load_reduction=27375 max_load_reduction_pct=0.835419 moved_to_echo=14678 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1228, 0, 1360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3691, 0, 0, 0, 0, 0, 0, 873, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 231, 3279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 3992, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 3755, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1502, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 31, 0, 159, 0, 2458, 0, 1545, 110, 56, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2588, 4568, 3510, 4002, 3964, 5596, 4092, 4448] avg=4096.0000 max=5596 max_rank=5 min=2588 min_rank=0 imbalance_ratio=1.366211 max_load_reduction=27172 max_load_reduction_pct=0.829224 moved_to_echo=18100 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 966, 0, 4087, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3686, 0, 0, 0, 0, 0, 2, 687, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 230, 4092, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 2, 0, 0, 0, 8, 3981, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 3739, 104, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1343, 0, 2434, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 1724, 0, 1382, 110, 44, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5054, 4381, 4326, 4012, 3861, 3781, 4086, 3267] avg=4096.0000 max=5054 max_rank=0 min=3267 min_rank=7 imbalance_ratio=1.233887 max_load_reduction=27714 max_load_reduction_pct=0.845764 moved_to_echo=14995 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1721, 0, 1335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3624, 0, 0, 0, 0, 0, 0, 1225, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 231, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 0, 0, 7, 0, 0, 0, 3992, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 3746, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1737, 0, 2895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 10, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 1434, 0, 1786, 110, 78, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3056, 4853, 4306, 4508, 3896, 4632, 4105, 3412] avg=4096.0000 max=4853 max_rank=1 min=3056 min_rank=0 imbalance_ratio=1.184814 max_load_reduction=27915 max_load_reduction_pct=0.851898 moved_to_echo=16045 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1885, 9, 3898, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 641, 5, 0, 6, 0, 0, 16, 1352, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 219, 4075, 0, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 180, 0, 8, 0, 0, 0, 0, 3949, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3655, 103, 0, 0, 0, 0, 0, 0, 15, 0, 0, 1889, 0, 410, 14, 21, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 459, 0, 6, 0, 0, 3411, 6, 1952, 110, 90, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5819, 2045, 4544, 4142, 3760, 2349, 4075, 6034] avg=4096.0000 max=6034 max_rank=7 min=2045 min_rank=1 imbalance_ratio=1.473145 max_load_reduction=26734 max_load_reduction_pct=0.815857 moved_to_echo=16218 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 13312, 128, 19456, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 0, 12928, 128, 19968, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 640, 128, 128, 896, 0, 16640, 128, 13312, 896, 640, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 23296, 128, 0, 128, 0, 0, 128, 9472, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 128, 1920, 30720, 128, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 2048, 128, 128, 0, 128, 128, 0, 29952, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 896, 0, 128, 128, 0, 0, 128, 32000, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 0, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 90, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4085, 4092, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4093, 0, 0, 12, 0, 5, 4096, 4007, 0, 0, 0, 0, 4034, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 16, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4086, 0, 0, 4060, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 27, 0, 0, 4094, 0, 0, 0, 11, 159, 4095, 3922, 0, 0, 0, 0, 4094, 0, 2, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 30, 0, 0, 572, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4087, 4085, 1, 1, 0, 0, 0, 4038, 0, 0, 0, 16, 0, 0, 5, 0, 0, 4086, 0, 0, 0, 0, 18, 4092, 3563, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 4096, 4076, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 43, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 12, 4096, 4086, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 4091, 0, 0, 4082, 4096, 3, 3, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 4093, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[9, 5, 0, 0, 0, 16, 122, 4, 0, 0, 0, 0, 33, 19, 0, 0, 0, 0, 0, 0, 0, 14, 0, 4075, 4, 2, 4066, 4050, 0, 0, 16, 14, 27, 4021, 0, 0, 12, 65, 8, 18, 18, 0, 0, 3929, 0, 0, 0, 0, 0, 4078, 4072, 0, 0, 0, 4, 4034, 5, 0, 0, 0, 3, 8, 13, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1, 0, 0, 34, 0, 3, 0, 42, 0, 0, 0, 0, 0, 3, 1, 0, 2, 0, 0, 2, 0, 2, 0, 4093, 3, 0, 4046, 4078, 0, 0, 0, 2, 0, 4083, 0, 0, 1, 21, 0, 2, 0, 0, 0, 4087, 0, 2, 1, 0, 0, 4094, 4091, 0, 4, 4, 1, 4063, 0, 0, 0, 0, 0, 0, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 187, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4087, 4047, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 670, 4096, 3347, 0, 0, 0, 0, 4014, 0, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 90, 58, 3914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 23, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 14, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 12, 0, 9, 5, 0, 0, 0, 0, 0, 0, 5, 0, 4007, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4062, 4102, 4096, 4102, 4092, 4119, 4100, 4095] avg=4096.0000 max=4119 max_rank=5 min=4062 min_rank=0 imbalance_ratio=1.005615 max_load_reduction=28649 max_load_reduction_pct=0.874298 moved_to_echo=16406 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 30, 0, 0, 572, 1, 3956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4071, 23, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 14, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4086, 0, 0, 0, 0, 9, 5, 0, 0, 0, 0, 0, 0, 18, 0, 3563, 0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4559, 4096, 4095, 4103, 4049, 4105, 3670, 4091] avg=4096.0000 max=4559 max_rank=0 min=3670 min_rank=6 imbalance_ratio=1.113037 max_load_reduction=28209 max_load_reduction_pct=0.860870 moved_to_echo=15915 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 3974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 23, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 9, 5, 0, 0, 0, 0, 0, 0, 12, 0, 4086, 0, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3975, 4086, 4102, 4110, 4110, 4102, 4188, 4095] avg=4096.0000 max=4188 max_rank=6 min=3975 min_rank=0 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16495 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 16, 0, 3972, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4082, 23, 1, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 4094, 0, 0, 0, 11, 9, 5, 0, 0, 0, 0, 0, 0, 159, 0, 3922, 0, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3989, 4108, 4093, 4074, 4091, 4146, 4171, 4096] avg=4096.0000 max=4171 max_rank=6 min=3989 min_rank=0 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16504 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 3972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 23, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 14, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 9, 5, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3973, 4106, 4104, 4102, 4090, 4110, 4191, 4092] avg=4096.0000 max=4191 max_rank=6 min=3973 min_rank=0 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16483 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 187, 30, 3895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 22, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 9, 5, 0, 0, 0, 0, 0, 0, 670, 0, 3347, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4057, 4096, 4101, 4091, 4111, 4105, 4095] avg=4096.0000 max=4112 max_rank=0 min=4057 min_rank=1 imbalance_ratio=1.003906 max_load_reduction=28656 max_load_reduction_pct=0.874512 moved_to_echo=16402 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 34, 0, 3, 0, 42, 3942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 4064, 23, 1, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 2, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4046, 14, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 1, 21, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 2, 1, 0, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4, 4, 1, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4093, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4022, 4092, 4099, 4065, 4102, 4104, 4189, 4095] avg=4096.0000 max=4189 max_rank=6 min=4022 min_rank=0 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16490 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[9, 5, 0, 0, 0, 16, 122, 4, 3920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 19, 0, 0, 4041, 26, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 4066, 9, 0, 0, 16, 14, 0, 0, 0, 0, 0, 0, 0, 0, 27, 4013, 0, 0, 12, 65, 8, 18, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 3929, 0, 0, 0, 0, 10, 8, 6, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 4, 78, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 3, 8, 13, 4, 4076, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4121, 4083, 4111, 4143, 3971, 4154, 4109] avg=4096.0000 max=4154 max_rank=6 min=3971 min_rank=5 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16377 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 32640, 0, 0, 128, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 0, 0, 32640, 0, 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[896, 0, 31232, 0, 128, 128, 128, 768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 128, 128, 1024, 256, 31616, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 32640, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[0, 128, 0, 0, 128, 128, 128, 128, 32640, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 0, 0, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 2, 0, 0, 4094, 0, 0, 4096, 0, 1, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 4096, 0, 0, 0, 4095, 9, 0, 0, 0, 0, 0, 32, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 6, 0, 0, 4096, 4, 0, 4091, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4094, 4090, 0, 0, 0, 4069, 1, 0, 0, 0, 0, 0, 31, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 4078, 3, 0, 4077, 0, 28, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 1, 3713, 4092, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 372, 0, 4091, 4, 1, 0, 28, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4075, 10, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 3893, 2, 0, 4084, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4064, 4096, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 247, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 4084, 1, 0, 4093, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 3, 0, 1, 0, 5, 0, 0, 0, 2, 0, 4094, 4093, 0, 0, 2, 3995, 93, 0, 2, 0, 0, 1, 0, 0, 4093, 47, 0, 0, 0, 0, 0, 0, 0, 4054, 2, 2, 3, 2, 2, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 125, 0, 0, 0, 3879, 0, 5, 4062, 0, 54, 0, 0, 0, 4081, 0, 0, 24, 21, 0, 0, 0, 0, 0, 0, 0, 16, 3, 4080, 4072, 0, 0, 25, 3519, 523, 0, 16, 0, 0, 1, 19, 0, 4080, 0, 18, 0, 13, 0, 0, 0, 0, 4075, 4, 16, 2, 5, 11, 0, 0, 12, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3850, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 294, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 2, 0, 0, 4048, 39, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 45, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 83, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4095, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 3967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 0, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4140, 4092, 4140, 4111, 3999, 4104, 4092] avg=4096.0000 max=4140 max_rank=1 min=3999 min_rank=5 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16306 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 6, 0, 0, 4043, 39, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4094, 83, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4069, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4145, 4096, 4179, 4077, 3999, 4098, 4086] avg=4096.0000 max=4179 max_rank=3 min=3999 min_rank=5 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16260 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 4048, 39, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 83, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4075, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4140, 4104, 4179, 4092, 3969, 4104, 4092] avg=4096.0000 max=4179 max_rank=3 min=3969 min_rank=5 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16257 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 4045, 39, 0, 0, 0, 0, 0, 0, 4084, 1, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 2, 0, 4094, 83, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 2, 3995, 93, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3965, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 2, 6, 2, 0, 0, 0, 0, 0, 0, 2, 3, 2, 2, 0, 0, 0, 0, 4086, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4130, 4095, 4184, 4099, 4013, 4064, 4098] avg=4096.0000 max=4184 max_rank=3 min=4013 min_rank=5 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16274 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 4029, 39, 0, 0, 0, 0, 0, 0, 4078, 3, 0, 45, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 1, 3713, 83, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 372, 0, 3963, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 4096, 0, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4085, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4154, 4069, 3812, 4103, 4340, 4132, 4089] avg=4096.0000 max=4340 max_rank=5 min=3812 min_rank=3 imbalance_ratio=1.059570 max_load_reduction=28428 max_load_reduction_pct=0.867554 moved_to_echo=16664 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4036, 39, 0, 0, 0, 0, 0, 0, 3893, 2, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4064, 83, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 0, 3967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 3940, 4093, 4153, 4097, 4214, 4104, 4092] avg=4096.0000 max=4214 max_rank=5 min=3940 min_rank=1 imbalance_ratio=1.028809 max_load_reduction=28554 max_load_reduction_pct=0.871399 moved_to_echo=16507 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4048, 39, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3850, 83, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 294, 0, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4139, 4097, 3933, 4103, 4262, 4055, 4092] avg=4096.0000 max=4262 max_rank=5 min=3933 min_rank=3 imbalance_ratio=1.040527 max_load_reduction=28506 max_load_reduction_pct=0.869934 moved_to_echo=16512 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 125, 0, 0, 0, 4015, 46, 0, 0, 0, 0, 0, 0, 3879, 0, 5, 42, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 24, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 3, 4080, 89, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 25, 3519, 523, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 19, 0, 3934, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 4075, 4, 11, 4, 0, 0, 0, 0, 0, 0, 16, 2, 5, 11, 0, 0, 12, 7, 4069, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4186, 3980, 4122, 4188, 4086, 3972, 4107, 4127] avg=4096.0000 max=4188 max_rank=3 min=3972 min_rank=5 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16292 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 0, 128, 128, 128, 0, 128, 32384, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 128, 1024, 0, 31744, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 0, 0, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 32768, 0, 128, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32384, 128, 128, 384, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 0, 128, 128, 32128, 768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 0, 128, 32128, 640, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 0, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 4095, 89, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 4095, 4094, 0, 23, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 4086, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 12, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4087, 4019, 0, 7, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 4095, 12, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 4049, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 4096, 4096, 0, 45, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 4093, 108, 0, 0, 0, 0, 4, 4090, 0, 5, 2, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 3992, 0, 3, 2, 1, 0, 0, 2, 0, 0, 1, 0, 0, 4093, 3, 0, 1, 0, 0, 0, 4092, 3944, 3, 99, 0, 0, 0, 0, 57, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 4069, 313, 3, 0, 0, 3, 0, 4064, 0, 1, 24, 0, 2, 4066, 0, 0, 0, 0, 0, 2, 8, 3833, 0, 0, 2, 16, 0, 0, 2, 0, 0, 0, 0, 0, 4048, 25, 7, 28, 0, 39, 0, 4072, 3752, 16, 0, 0, 16, 6, 16, 195, 0, 0, 1, 34, 0, 0, 31, 0, 0, 0, 2, 4072, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 4095, 11, 0, 0, 0, 0, 0, 4094, 1, 1, 0, 0, 0, 4095, 0, 0, 0, 2, 3, 0, 0, 3597, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 1, 0, 0, 488, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4095, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 64, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4040, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3982, 0, 82, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 4094, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4129, 4098, 4119, 4096, 4099, 4162, 3968, 4097] avg=4096.0000 max=4162 max_rank=5 min=3968 min_rank=6 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16326 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4015, 0, 82, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 4096, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 3966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3995, 4098, 4152, 4096, 4099, 4187, 4047, 4094] avg=4096.0000 max=4187 max_rank=5 min=3995 min_rank=0 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16427 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4040, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 82, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4052, 4098, 4228, 4096, 4098, 4131, 3969, 4096] avg=4096.0000 max=4228 max_rank=2 min=3969 min_rank=6 imbalance_ratio=1.032227 max_load_reduction=28540 max_load_reduction_pct=0.870972 moved_to_echo=16294 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4040, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 3597, 0, 82, 55, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 4092, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 4084, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 32, 0, 0, 0, 0, 3966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4051, 4099, 3739, 4096, 4060, 4131, 4486, 4106] avg=4096.0000 max=4486 max_rank=6 min=3739 min_rank=2 imbalance_ratio=1.095215 max_load_reduction=28282 max_load_reduction_pct=0.863098 moved_to_echo=16783 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4096, 0, 82, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 4019, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 3958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4031, 4096, 4245, 4098, 4096, 4071, 4035, 4096] avg=4096.0000 max=4245 max_rank=2 min=4031 min_rank=0 imbalance_ratio=1.036377 max_load_reduction=28523 max_load_reduction_pct=0.870453 moved_to_echo=16298 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4028, 0, 82, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 9, 0, 0, 0, 0, 3966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4038, 4096, 4165, 4098, 4098, 4140, 4039, 4094] avg=4096.0000 max=4165 max_rank=2 min=4038 min_rank=0 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16371 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4038, 108, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3992, 0, 82, 55, 0, 0, 0, 0, 0, 0, 3, 2, 1, 0, 0, 2, 0, 0, 4090, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4093, 3, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 3944, 3, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 5, 0, 0, 0, 3963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4150, 4090, 4129, 4100, 4100, 4091, 4025, 4083] avg=4096.0000 max=4150 max_rank=0 min=4025 min_rank=6 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16299 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4003, 313, 3, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 24, 0, 2, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8, 3833, 0, 89, 59, 0, 0, 0, 0, 0, 0, 0, 2, 16, 0, 0, 2, 0, 0, 4064, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 25, 7, 28, 0, 3, 7, 0, 0, 0, 0, 0, 0, 39, 0, 32, 3752, 16, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 6, 16, 195, 0, 0, 1, 34, 0, 3947, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 2, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4322, 4093, 3991, 4088, 4118, 3855, 4199, 4102] avg=4096.0000 max=4322 max_rank=0 min=3855 min_rank=5 imbalance_ratio=1.055176 max_load_reduction=28446 max_load_reduction_pct=0.868103 moved_to_echo=16274 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 384, 32256, 128, 256, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 0, 0, 32768, 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 0, 128, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 0, 128, 128, 128, 128, 31744, 0, 768, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 32256, 640, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 896, 256, 128, 128, 128, 0, 31744, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True + [2026-09-21 08:24:26] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 297.0 | throughput per GPU (TFLOP/s/GPU): 108.5 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.724556E+00 | loss scale: 1.0 | grad norm: 4.139 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[115, 248, 1195, 3, 1, 702, 813, 687, 14, 927, 33, 0, 15, 9, 191, 107, 9, 794, 522, 1847, 18, 119, 241, 6, 109, 10, 126, 404, 945, 430, 819, 1473, 641, 474, 528, 1330, 210, 1417, 691, 715, 1511, 545, 1570, 70, 764, 1013, 26, 20, 665, 742, 449, 50, 839, 1550, 1858, 12, 1110, 40, 311, 13, 47, 24, 586, 15] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[102, 115, 1422, 0, 0, 641, 818, 363, 0, 1060, 31, 6, 9, 4, 500, 28, 1, 855, 1187, 1834, 27, 50, 250, 2, 56, 19, 54, 205, 751, 758, 856, 1570, 938, 793, 462, 804, 204, 1094, 456, 779, 1645, 541, 2152, 63, 493, 727, 14, 70, 684, 571, 493, 56, 387, 1614, 1968, 30, 1293, 33, 231, 2, 19, 27, 541, 10] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[46, 271, 919, 2, 0, 960, 747, 262, 0, 959, 34, 1, 14, 53, 528, 44, 3, 810, 917, 1683, 53, 142, 256, 4, 93, 27, 92, 233, 1346, 376, 1082, 1796, 650, 324, 389, 1186, 253, 846, 813, 1043, 1792, 321, 1806, 51, 653, 879, 17, 28, 738, 740, 732, 21, 666, 1236, 1565, 32, 1342, 54, 303, 2, 7, 43, 471, 12] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[132, 242, 889, 1, 1, 486, 779, 426, 1, 570, 34, 2, 59, 3, 507, 85, 2, 1179, 576, 1957, 30, 270, 198, 1, 35, 21, 22, 417, 849, 1114, 1169, 1652, 778, 501, 485, 1135, 136, 1261, 423, 692, 1584, 817, 1357, 73, 678, 773, 22, 41, 1166, 548, 684, 46, 668, 1423, 1598, 44, 1088, 29, 418, 2, 14, 12, 543, 20] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[226, 399, 609, 0, 0, 844, 936, 460, 1, 581, 21, 0, 7, 65, 250, 134, 1, 718, 707, 1720, 22, 374, 235, 29, 102, 7, 47, 316, 1277, 459, 993, 1579, 536, 430, 709, 1179, 85, 1226, 708, 1032, 1485, 657, 1569, 30, 1279, 947, 17, 55, 945, 530, 243, 27, 529, 1433, 1611, 58, 1108, 36, 313, 7, 28, 15, 801, 21] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[190, 336, 846, 0, 0, 636, 813, 519, 2, 723, 16, 1, 20, 105, 238, 100, 3, 771, 832, 1936, 24, 102, 191, 3, 93, 7, 67, 290, 820, 373, 1572, 1396, 525, 406, 559, 1543, 125, 1119, 901, 899, 1139, 603, 1403, 58, 1203, 1025, 10, 14, 913, 546, 626, 32, 928, 860, 1444, 71, 1394, 11, 250, 9, 3, 5, 1088, 31] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[135, 102, 1127, 0, 0, 668, 744, 458, 3, 466, 45, 1, 7, 33, 208, 118, 3, 1014, 746, 2188, 12, 231, 249, 12, 56, 2, 79, 355, 1066, 610, 1049, 1989, 1261, 395, 561, 926, 148, 718, 527, 1068, 1356, 473, 1696, 73, 796, 589, 6, 27, 1218, 561, 650, 31, 783, 1030, 1653, 22, 978, 12, 186, 3, 6, 8, 1224, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[82, 593, 1197, 1, 1, 862, 1280, 374, 0, 332, 126, 7, 14, 204, 661, 195, 30, 740, 1018, 1517, 55, 236, 278, 31, 143, 128, 97, 231, 939, 542, 810, 1046, 1107, 401, 438, 1092, 205, 1261, 443, 514, 1345, 851, 1482, 168, 730, 848, 28, 90, 876, 634, 769, 129, 908, 601, 1645, 30, 1258, 48, 322, 27, 25, 14, 692, 17] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[102, 115, 1422, 0, 0, 641, 818, 363, 335, 276, 0, 0, 0, 0, 0, 0, 0, 1060, 31, 6, 9, 4, 500, 28, 1968, 1471, 0, 0, 0, 0, 0, 0, 1, 855, 1187, 1834, 27, 50, 250, 2, 97, 16, 0, 0, 0, 0, 0, 0, 56, 19, 54, 205, 751, 758, 856, 983, 0, 0, 0, 0, 0, 0, 0, 0, 938, 793, 462, 0, 204, 662, 456, 779, 0, 0, 0, 0, 0, 0, 0, 0, 1645, 541, 271, 63, 493, 727, 14, 70, 0, 0, 0, 0, 0, 0, 0, 0, 684, 571, 493, 56, 387, 1598, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 1293, 33, 231, 2, 19, 27, 541, 10, 804, 410, 311, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 5077, 4319, 3682, 4294, 3824, 3819, 3681] avg=4096.0000 max=5077 max_rank=1 min=3681 min_rank=7 imbalance_ratio=1.239502 max_load_reduction=27691 max_load_reduction_pct=0.845062 moved_to_echo=15618 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[132, 242, 889, 1, 1, 486, 779, 426, 386, 291, 0, 0, 0, 0, 0, 0, 1, 570, 34, 2, 59, 3, 507, 85, 1598, 927, 0, 0, 0, 0, 0, 0, 2, 1179, 576, 1957, 30, 270, 198, 1, 112, 14, 0, 0, 0, 0, 0, 0, 35, 21, 22, 417, 849, 1114, 1169, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 778, 501, 485, 0, 136, 763, 423, 692, 0, 0, 0, 0, 0, 0, 0, 0, 1584, 817, 172, 73, 678, 773, 22, 41, 0, 0, 0, 0, 0, 0, 0, 0, 1166, 548, 684, 46, 668, 1409, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 1088, 29, 418, 2, 14, 12, 543, 20, 1135, 258, 327, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3633, 3786, 4339, 4661, 3778, 4160, 4565, 3846] avg=4096.0000 max=4661 max_rank=3 min=3633 min_rank=0 imbalance_ratio=1.137939 max_load_reduction=28107 max_load_reduction_pct=0.857758 moved_to_echo=16349 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[115, 248, 1195, 3, 1, 702, 813, 687, 434, 259, 0, 0, 0, 0, 0, 0, 14, 927, 33, 0, 15, 9, 191, 107, 1858, 1073, 0, 0, 0, 0, 0, 0, 9, 794, 522, 1847, 18, 119, 241, 6, 126, 16, 0, 0, 0, 0, 0, 0, 109, 10, 126, 404, 945, 430, 819, 923, 0, 0, 0, 0, 0, 0, 0, 0, 641, 474, 528, 0, 210, 857, 691, 715, 0, 0, 0, 0, 0, 0, 0, 0, 1511, 545, 198, 70, 764, 1013, 26, 20, 0, 0, 0, 0, 0, 0, 0, 0, 665, 742, 449, 50, 839, 1534, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 1110, 40, 311, 13, 47, 24, 586, 15, 1330, 299, 291, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4457, 4227, 3698, 3766, 4116, 4147, 4291, 4066] avg=4096.0000 max=4457 max_rank=0 min=3698 min_rank=2 imbalance_ratio=1.088135 max_load_reduction=28311 max_load_reduction_pct=0.863983 moved_to_echo=16620 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[190, 336, 846, 0, 0, 636, 813, 519, 342, 246, 0, 0, 0, 0, 0, 0, 2, 723, 16, 1, 20, 105, 238, 100, 1444, 959, 0, 0, 0, 0, 0, 0, 3, 771, 832, 1936, 24, 102, 191, 3, 99, 8, 0, 0, 0, 0, 0, 0, 93, 7, 67, 290, 820, 373, 1572, 874, 0, 0, 0, 0, 0, 0, 0, 0, 525, 406, 559, 0, 125, 678, 901, 899, 0, 0, 0, 0, 0, 0, 0, 0, 1139, 603, 177, 58, 1203, 1025, 10, 14, 0, 0, 0, 0, 0, 0, 0, 0, 913, 546, 626, 32, 928, 852, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 1394, 11, 250, 9, 3, 5, 1088, 31, 1543, 267, 276, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3928, 3608, 3969, 4096, 4093, 4229, 3968, 4877] avg=4096.0000 max=4877 max_rank=7 min=3608 min_rank=1 imbalance_ratio=1.190674 max_load_reduction=27891 max_load_reduction_pct=0.851166 moved_to_echo=17167 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[46, 271, 919, 2, 0, 960, 747, 262, 259, 316, 0, 0, 0, 0, 0, 0, 0, 959, 34, 1, 14, 53, 528, 44, 1565, 1235, 0, 0, 0, 0, 0, 0, 3, 810, 917, 1683, 53, 142, 256, 4, 75, 12, 0, 0, 0, 0, 0, 0, 93, 27, 92, 233, 1346, 376, 1082, 1124, 0, 0, 0, 0, 0, 0, 0, 0, 650, 324, 389, 0, 253, 512, 813, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1792, 321, 227, 51, 653, 879, 17, 28, 0, 0, 0, 0, 0, 0, 0, 0, 738, 740, 732, 21, 666, 1224, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 1342, 54, 303, 2, 7, 43, 471, 12, 1186, 344, 356, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3782, 4433, 3955, 4373, 3984, 3968, 4153, 4120] avg=4096.0000 max=4433 max_rank=1 min=3782 min_rank=0 imbalance_ratio=1.082275 max_load_reduction=28335 max_load_reduction_pct=0.864716 moved_to_echo=16225 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[82, 593, 1197, 1, 1, 862, 1280, 374, 386, 184, 0, 0, 0, 0, 0, 0, 0, 332, 126, 7, 14, 204, 661, 195, 1645, 1013, 0, 0, 0, 0, 0, 0, 30, 740, 1018, 1517, 55, 236, 278, 31, 112, 6, 0, 0, 0, 0, 0, 0, 143, 128, 97, 231, 939, 542, 810, 655, 0, 0, 0, 0, 0, 0, 0, 0, 1107, 401, 438, 0, 205, 763, 443, 514, 0, 0, 0, 0, 0, 0, 0, 0, 1345, 851, 187, 168, 730, 848, 28, 90, 0, 0, 0, 0, 0, 0, 0, 0, 876, 634, 769, 129, 908, 595, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 1258, 48, 322, 27, 25, 14, 692, 17, 1092, 282, 207, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4960, 4197, 4023, 3545, 3871, 4247, 3941, 3984] avg=4096.0000 max=4960 max_rank=0 min=3545 min_rank=3 imbalance_ratio=1.210938 max_load_reduction=27808 max_load_reduction_pct=0.848633 moved_to_echo=16043 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[135, 102, 1127, 0, 0, 668, 744, 458, 219, 350, 0, 0, 0, 0, 0, 0, 3, 466, 45, 1, 7, 33, 208, 118, 1653, 1159, 0, 0, 0, 0, 0, 0, 3, 1014, 746, 2188, 12, 231, 249, 12, 63, 10, 0, 0, 0, 0, 0, 0, 56, 2, 79, 355, 1066, 610, 1049, 1245, 0, 0, 0, 0, 0, 0, 0, 0, 1261, 395, 561, 0, 148, 436, 527, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 1356, 473, 214, 73, 796, 589, 6, 27, 0, 0, 0, 0, 0, 0, 0, 0, 1218, 561, 650, 31, 783, 1020, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 978, 12, 186, 3, 6, 8, 1224, 7, 926, 323, 394, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3803, 3693, 4528, 4462, 4396, 3534, 4285, 4067] avg=4096.0000 max=4528 max_rank=2 min=3534 min_rank=5 imbalance_ratio=1.105469 max_load_reduction=28240 max_load_reduction_pct=0.861816 moved_to_echo=16282 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[226, 399, 609, 0, 0, 844, 936, 460, 378, 281, 0, 0, 0, 0, 0, 0, 1, 581, 21, 0, 7, 65, 250, 134, 1611, 1077, 0, 0, 0, 0, 0, 0, 1, 718, 707, 1720, 22, 374, 235, 29, 112, 19, 0, 0, 0, 0, 0, 0, 102, 7, 47, 316, 1277, 459, 993, 982, 0, 0, 0, 0, 0, 0, 0, 0, 536, 430, 709, 0, 85, 736, 708, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1485, 657, 189, 30, 1279, 947, 17, 55, 0, 0, 0, 0, 0, 0, 0, 0, 945, 530, 243, 27, 529, 1414, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 1108, 36, 313, 7, 28, 15, 801, 21, 1179, 303, 316, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4133, 3747, 3937, 4183, 4236, 4659, 3746, 4127] avg=4096.0000 max=4659 max_rank=5 min=3746 min_rank=6 imbalance_ratio=1.137451 max_load_reduction=28109 max_load_reduction_pct=0.857819 moved_to_echo=16768 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[7296, 4992, 4736, 512, 5760, 9728, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[1152, 2432, 8320, 128, 128, 5888, 7040, 3584, 2816, 2304, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[768, 256, 640, 2560, 8064, 4736, 8448, 7936, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[6528, 3840, 4224, 0, 1408, 5504, 4992, 6784, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 6912, 6528, 14720, 256, 1536, 1920, 128, 896, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[11904, 4864, 1664, 640, 6656, 6912, 256, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[9600, 384, 2432, 128, 256, 256, 6016, 256, 9216, 2560, 2560, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 5632, 384, 128, 256, 512, 3200, 896, 13440, 8960, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33792 slack_slots=31744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[456, 0, 0, 0, 0, 1702, 0, 715, 6, 0, 3492, 1187, 1, 0, 610, 0, 0, 0, 329, 11, 1216, 200, 1536, 0, 0, 2237, 1, 0, 289, 2623, 999, 0, 0, 0, 3, 1, 1993, 2322, 3115, 1, 680, 1, 224, 0, 1, 0, 160, 1, 4, 0, 291, 1925, 701, 0, 8, 0, 114, 2, 1339, 1650, 204, 0, 418, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1777, 0, 19, 0, 0, 1867, 0, 1651, 0, 9, 3561, 205, 0, 0, 937, 0, 0, 0, 137, 11, 107, 72, 88, 7, 0, 576, 0, 2, 1, 3839, 1689, 0, 0, 0, 0, 0, 1618, 2431, 3341, 0, 355, 0, 383, 0, 0, 0, 232, 0, 1, 10, 207, 3995, 850, 0, 2, 3, 752, 0, 930, 795, 235, 0, 73, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[633, 0, 0, 0, 1, 1894, 2, 937, 10, 121, 3196, 1858, 1, 1, 222, 2, 0, 0, 72, 359, 2, 67, 57, 0, 0, 2387, 0, 1, 1, 298, 1816, 1, 0, 0, 0, 0, 1216, 3938, 3511, 2, 19, 0, 1552, 0, 1, 0, 2571, 0, 85, 0, 33, 1785, 482, 0, 19, 4, 1623, 10, 284, 965, 531, 0, 196, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1205, 0, 4, 0, 0, 1059, 3, 1500, 314, 3, 2696, 2281, 5, 0, 1567, 0, 0, 0, 90, 46, 618, 18, 828, 0, 0, 2212, 0, 0, 5, 612, 779, 29, 0, 0, 0, 0, 2737, 2236, 1057, 0, 144, 6, 23, 0, 0, 0, 327, 1, 2, 4, 956, 499, 1247, 0, 4, 0, 596, 5, 2458, 1390, 422, 0, 2780, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[960, 0, 1, 0, 0, 6, 1, 28, 0, 0, 3120, 843, 0, 0, 2199, 0, 0, 0, 157, 78, 521, 20, 23, 0, 0, 3163, 0, 0, 6, 3575, 252, 0, 0, 0, 0, 0, 2025, 3941, 1290, 0, 754, 0, 174, 0, 0, 0, 79, 0, 0, 1, 81, 334, 1225, 0, 0, 0, 1141, 1, 1827, 690, 2659, 0, 1593, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1137, 0, 0, 0, 0, 1524, 35, 727, 24, 18, 2123, 1097, 1, 0, 366, 0, 0, 0, 113, 2, 242, 77, 43, 0, 5, 523, 0, 0, 82, 1435, 2662, 0, 2, 1, 0, 0, 1734, 3988, 2914, 0, 876, 1, 1371, 0, 0, 0, 2, 0, 0, 0, 129, 2903, 1208, 0, 11, 1, 749, 8, 734, 440, 1581, 0, 1879, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1464, 0, 0, 0, 0, 504, 1, 942, 20, 63, 3372, 616, 6, 0, 366, 0, 0, 2, 57, 0, 1670, 129, 35, 0, 1, 1361, 2, 0, 3, 1225, 2590, 0, 0, 10, 0, 0, 1598, 3876, 3606, 1, 77, 0, 420, 3, 0, 0, 133, 0, 1, 0, 721, 620, 1821, 0, 0, 0, 360, 0, 1186, 862, 515, 0, 2529, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[136, 0, 0, 0, 0, 3662, 1, 953, 12, 28, 3694, 67, 0, 0, 157, 0, 0, 0, 1, 58, 2091, 3, 15, 0, 0, 2351, 0, 2, 4, 2409, 2810, 0, 2, 7, 22, 0, 1746, 1526, 984, 2, 2002, 0, 171, 0, 0, 0, 186, 0, 0, 0, 2009, 240, 218, 0, 10, 0, 1094, 0, 443, 1523, 2002, 1, 126, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[633, 0, 0, 0, 1, 1894, 2, 937, 441, 313, 0, 0, 0, 0, 0, 0, 10, 121, 2222, 1858, 1, 1, 222, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 359, 2, 67, 57, 0, 3500, 0, 0, 0, 0, 0, 0, 0, 0, 2387, 0, 1, 1, 74, 1816, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1216, 0, 3198, 2, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 1552, 0, 1, 0, 2571, 0, 224, 103, 438, 0, 0, 0, 0, 0, 85, 0, 33, 1785, 482, 0, 19, 4, 533, 79, 0, 0, 0, 0, 0, 0, 1623, 10, 284, 965, 531, 0, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4221, 4437, 4057, 4280, 4416, 4908, 3020, 3429] avg=4096.0000 max=4908 max_rank=5 min=3020 min_rank=6 imbalance_ratio=1.198242 max_load_reduction=27860 max_load_reduction_pct=0.850220 moved_to_echo=15773 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[960, 0, 1, 0, 0, 6, 1, 28, 435, 119, 0, 0, 0, 0, 0, 0, 0, 0, 2160, 843, 0, 0, 2199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 78, 521, 20, 23, 0, 3503, 0, 0, 0, 0, 0, 0, 0, 0, 3163, 0, 0, 6, 872, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2025, 0, 1171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 754, 0, 174, 0, 0, 0, 79, 0, 2703, 840, 438, 0, 0, 0, 0, 0, 0, 1, 81, 334, 1225, 0, 0, 0, 525, 652, 0, 0, 0, 0, 0, 0, 1141, 1, 1827, 690, 2659, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1550, 5202, 4302, 4293, 3196, 4988, 2818, 6419] avg=4096.0000 max=6419 max_rank=7 min=1550 min_rank=0 imbalance_ratio=1.567139 max_load_reduction=26349 max_load_reduction_pct=0.804108 moved_to_echo=17421 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[136, 0, 0, 0, 0, 3662, 1, 953, 510, 87, 0, 0, 0, 0, 0, 0, 12, 28, 2567, 67, 0, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 58, 2091, 3, 15, 0, 1357, 0, 0, 0, 0, 0, 0, 0, 0, 2351, 0, 2, 4, 591, 2810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7, 22, 0, 1746, 0, 897, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2002, 0, 171, 0, 0, 0, 186, 0, 1818, 66, 169, 0, 0, 0, 0, 0, 0, 0, 2009, 240, 218, 0, 10, 0, 617, 51, 0, 0, 0, 0, 0, 0, 1094, 0, 443, 1523, 2002, 1, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5349, 2831, 3525, 5758, 2676, 4412, 3145, 5072] avg=4096.0000 max=5758 max_rank=3 min=2676 min_rank=4 imbalance_ratio=1.405762 max_load_reduction=27010 max_load_reduction_pct=0.824280 moved_to_echo=15305 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1137, 0, 0, 0, 0, 1524, 35, 727, 293, 260, 0, 0, 0, 0, 0, 0, 24, 18, 1476, 1097, 1, 0, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 2, 242, 77, 43, 0, 3544, 0, 0, 0, 0, 0, 0, 0, 5, 523, 0, 0, 82, 352, 2662, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 1734, 0, 2654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 876, 1, 1371, 0, 0, 0, 2, 0, 1083, 989, 444, 0, 0, 0, 0, 0, 0, 0, 129, 2903, 1208, 0, 11, 1, 354, 763, 0, 0, 0, 0, 0, 0, 749, 8, 734, 440, 1581, 0, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3976, 2982, 4021, 3624, 4391, 4766, 5369, 3639] avg=4096.0000 max=5369 max_rank=6 min=2982 min_rank=1 imbalance_ratio=1.310791 max_load_reduction=27399 max_load_reduction_pct=0.836151 moved_to_echo=18165 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[456, 0, 0, 0, 0, 1702, 0, 715, 482, 278, 0, 0, 0, 0, 0, 0, 6, 0, 2427, 1187, 1, 0, 610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 329, 11, 1216, 200, 1536, 0, 2063, 0, 0, 0, 0, 0, 0, 0, 0, 2237, 1, 0, 289, 643, 999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 1993, 0, 2837, 1, 0, 0, 0, 0, 0, 0, 0, 0, 680, 1, 224, 0, 1, 0, 160, 1, 1980, 220, 259, 0, 0, 0, 0, 0, 4, 0, 291, 1925, 701, 0, 8, 0, 583, 169, 0, 0, 0, 0, 0, 0, 114, 2, 1339, 1650, 204, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3633, 4231, 5355, 4169, 4835, 3526, 3681, 3338] avg=4096.0000 max=5355 max_rank=2 min=3338 min_rank=7 imbalance_ratio=1.307373 max_load_reduction=27413 max_load_reduction_pct=0.836578 moved_to_echo=15380 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1205, 0, 4, 0, 0, 1059, 3, 1500, 372, 94, 0, 0, 0, 0, 0, 0, 314, 3, 1874, 2281, 5, 0, 1567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 46, 618, 18, 828, 0, 1988, 0, 0, 0, 0, 0, 0, 0, 0, 2212, 0, 0, 5, 151, 779, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2737, 0, 963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 6, 23, 0, 0, 0, 327, 1, 461, 1463, 248, 0, 0, 0, 0, 0, 2, 4, 956, 499, 1247, 0, 4, 0, 450, 1129, 0, 0, 0, 0, 0, 0, 596, 5, 2458, 1390, 422, 0, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4237, 6044, 3588, 3176, 3700, 2673, 4291, 5059] avg=4096.0000 max=6044 max_rank=1 min=2673 min_rank=5 imbalance_ratio=1.475586 max_load_reduction=26724 max_load_reduction_pct=0.815552 moved_to_echo=15723 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1777, 0, 19, 0, 0, 1867, 0, 1651, 492, 298, 0, 0, 0, 0, 0, 0, 0, 9, 2475, 205, 0, 0, 937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 11, 107, 72, 88, 7, 2160, 0, 0, 0, 0, 0, 0, 0, 0, 576, 0, 2, 1, 942, 1689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1618, 0, 3043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 355, 0, 383, 0, 0, 0, 232, 0, 2897, 38, 271, 0, 0, 0, 0, 0, 1, 10, 207, 3995, 850, 0, 2, 3, 594, 29, 0, 0, 0, 0, 0, 0, 752, 0, 930, 795, 235, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6104, 3626, 2582, 3210, 4661, 4176, 5691, 2718] avg=4096.0000 max=6104 max_rank=0 min=2582 min_rank=2 imbalance_ratio=1.490234 max_load_reduction=26664 max_load_reduction_pct=0.813721 moved_to_echo=17246 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1464, 0, 0, 0, 0, 504, 1, 942, 465, 322, 0, 0, 0, 0, 0, 0, 20, 63, 2344, 616, 6, 0, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 57, 0, 1670, 129, 35, 0, 3445, 0, 0, 0, 0, 0, 0, 0, 1, 1361, 2, 0, 3, 301, 2590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 1598, 0, 3284, 1, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 420, 3, 0, 0, 133, 0, 924, 1331, 431, 0, 0, 0, 0, 0, 1, 0, 721, 620, 1821, 0, 0, 0, 563, 1027, 0, 0, 0, 0, 0, 0, 360, 0, 1186, 862, 515, 0, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3698, 3415, 5338, 4258, 4893, 3319, 4753, 3094] avg=4096.0000 max=5338 max_rank=2 min=3094 min_rank=7 imbalance_ratio=1.303223 max_load_reduction=27430 max_load_reduction_pct=0.837097 moved_to_echo=16059 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[7808, 0, 128, 0, 128, 12288, 128, 7552, 3584, 1792, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[6528, 128, 9216, 8320, 8192, 128, 768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4992, 128, 4352, 128, 128, 0, 3712, 128, 12160, 5120, 2816, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[512, 256, 17664, 8192, 128, 128, 6528, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 14720, 0, 18048, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 4480, 12416, 7808, 0, 128, 128, 4224, 3968, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 14848, 128, 128, 512, 3968, 13696, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 1024, 640, 6528, 640, 2688, 128, 21632, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[2, 6, 0, 0, 2480, 0, 53, 1, 1149, 0, 0, 123, 3, 1507, 1, 273, 377, 0, 3950, 0, 2, 2, 0, 1, 0, 5, 0, 519, 0, 1, 0, 0, 0, 3796, 13, 0, 0, 141, 1, 0, 2, 1, 1, 0, 3906, 0, 27, 0, 44, 11, 3960, 1436, 2, 1571, 4019, 0, 0, 48, 0, 0, 1956, 842, 536, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 6, 0, 19, 3108, 0, 1770, 0, 560, 0, 23, 1779, 2, 1966, 0, 49, 1584, 0, 2730, 15, 0, 0, 0, 0, 0, 0, 0, 24, 0, 1, 0, 0, 11, 1204, 115, 0, 2, 2, 0, 0, 0, 0, 0, 4, 3144, 0, 2, 0, 43, 169, 4082, 57, 188, 3505, 3933, 0, 0, 0, 0, 0, 305, 68, 2298, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3304, 0, 231, 0, 667, 0, 0, 34, 3, 3314, 0, 412, 709, 4, 3603, 109, 92, 0, 0, 0, 1, 0, 0, 168, 4, 11, 0, 0, 0, 2907, 0, 6, 0, 137, 17, 0, 0, 0, 0, 0, 3170, 2, 0, 0, 57, 0, 3903, 2669, 0, 2858, 653, 0, 0, 770, 0, 0, 52, 0, 2901, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2320, 0, 1272, 0, 1812, 0, 0, 25, 0, 2456, 0, 193, 3038, 31, 3823, 0, 2, 0, 0, 0, 2, 0, 0, 83, 0, 0, 0, 3, 19, 834, 1, 0, 364, 1713, 75, 0, 0, 0, 0, 5, 2269, 0, 0, 0, 512, 0, 3986, 3071, 1, 62, 3814, 0, 0, 207, 0, 0, 136, 84, 553, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 478, 0, 694, 0, 3005, 0, 0, 2, 0, 2997, 0, 1036, 3141, 9, 3693, 0, 8, 0, 0, 0, 0, 0, 0, 580, 7, 1, 0, 0, 0, 1205, 1, 0, 0, 3, 4, 0, 0, 0, 0, 1, 3960, 0, 1, 0, 22, 0, 4019, 1651, 0, 734, 3898, 0, 0, 1, 0, 0, 515, 70, 1031, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 6, 0, 2, 2332, 0, 1663, 6, 2406, 0, 4, 1698, 0, 727, 2, 296, 1131, 5, 3938, 87, 0, 0, 0, 0, 0, 2, 0, 900, 0, 0, 0, 5, 7, 1496, 0, 0, 0, 314, 73, 0, 0, 0, 4, 30, 3704, 0, 0, 0, 45, 41, 4094, 2446, 0, 441, 3789, 0, 0, 22, 0, 0, 335, 28, 688, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 4, 1, 3436, 0, 1368, 0, 54, 0, 0, 193, 0, 1687, 0, 145, 2001, 27, 4022, 0, 0, 0, 0, 1, 0, 0, 0, 40, 95, 10, 0, 1, 0, 3560, 0, 0, 0, 121, 54, 1, 0, 0, 0, 15, 3657, 0, 0, 0, 214, 11, 4024, 1076, 0, 730, 3989, 1, 0, 9, 1, 0, 190, 8, 2022, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3651, 0, 190, 0, 2889, 1, 0, 1, 0, 3013, 0, 39, 480, 0, 2176, 222, 0, 0, 0, 0, 0, 0, 0, 298, 2, 1, 0, 13, 2, 1475, 2, 4, 1, 1493, 544, 0, 0, 0, 0, 0, 2410, 0, 0, 0, 656, 18, 4060, 2173, 0, 1849, 3889, 4, 0, 5, 2, 0, 12, 0, 1193, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 6, 0, 0, 2480, 0, 53, 1, 322, 82, 0, 0, 0, 0, 0, 0, 1149, 0, 0, 123, 3, 1185, 1, 273, 0, 0, 0, 0, 0, 0, 0, 0, 377, 0, 2786, 0, 2, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 519, 0, 1, 0, 0, 3696, 0, 0, 0, 0, 0, 0, 0, 0, 3796, 13, 0, 0, 141, 1, 0, 1413, 170, 0, 0, 0, 0, 0, 0, 2, 1, 1, 0, 3906, 0, 27, 0, 912, 0, 0, 0, 0, 0, 0, 0, 44, 11, 0, 1436, 2, 1571, 633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 1956, 842, 536, 0, 1973, 264, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2946, 2734, 3168, 4221, 5534, 4849, 3697, 5619] avg=4096.0000 max=5619 max_rank=7 min=2734 min_rank=1 imbalance_ratio=1.371826 max_load_reduction=27149 max_load_reduction_pct=0.828522 moved_to_echo=19699 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3304, 0, 231, 0, 708, 75, 0, 0, 0, 0, 0, 0, 667, 0, 0, 34, 3, 2606, 0, 412, 0, 0, 0, 0, 0, 0, 0, 0, 709, 4, 2541, 109, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 168, 4, 11, 0, 0, 3644, 0, 0, 0, 0, 0, 0, 0, 0, 2907, 0, 6, 0, 137, 17, 0, 229, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3170, 2, 0, 0, 832, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 2669, 0, 2858, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 770, 0, 0, 52, 0, 2901, 0, 320, 259, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4318, 3722, 3455, 3828, 3451, 4004, 5688, 4302] avg=4096.0000 max=5688 max_rank=6 min=3451 min_rank=4 imbalance_ratio=1.388672 max_load_reduction=27080 max_load_reduction_pct=0.826416 moved_to_echo=17445 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 0, 19, 3108, 0, 1770, 0, 420, 57, 0, 0, 0, 0, 0, 0, 560, 0, 23, 1779, 2, 1546, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 1584, 0, 1926, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 1, 0, 0, 3811, 0, 0, 0, 0, 0, 0, 0, 11, 1204, 115, 0, 2, 2, 0, 0, 1383, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3144, 0, 2, 0, 630, 0, 0, 0, 0, 0, 0, 0, 43, 169, 0, 57, 188, 3505, 619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 305, 68, 2298, 0, 1931, 271, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5380, 3959, 3525, 3836, 2834, 3780, 4581, 4873] avg=4096.0000 max=5380 max_rank=0 min=2834 min_rank=4 imbalance_ratio=1.313477 max_load_reduction=27388 max_load_reduction_pct=0.835815 moved_to_echo=16068 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2320, 0, 1272, 0, 525, 80, 0, 0, 0, 0, 0, 0, 1812, 0, 0, 25, 0, 1931, 0, 193, 0, 0, 0, 0, 0, 0, 0, 0, 3038, 31, 2695, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 83, 0, 0, 0, 3, 3720, 0, 0, 0, 0, 0, 0, 0, 19, 834, 1, 0, 364, 1713, 75, 0, 1341, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2269, 0, 0, 0, 883, 0, 0, 0, 0, 0, 0, 0, 512, 0, 0, 3071, 1, 62, 601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 0, 0, 136, 84, 553, 2, 1872, 266, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4197, 3961, 5766, 3808, 4512, 3157, 4247, 3120] avg=4096.0000 max=5766 max_rank=2 min=3120 min_rank=7 imbalance_ratio=1.407715 max_load_reduction=27002 max_load_reduction_pct=0.824036 moved_to_echo=15036 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 478, 0, 694, 0, 644, 81, 0, 0, 0, 0, 0, 0, 3005, 0, 0, 2, 0, 2353, 0, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 3141, 9, 2593, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 580, 7, 1, 0, 0, 3752, 0, 0, 0, 0, 0, 0, 0, 0, 1205, 1, 0, 0, 3, 4, 0, 1375, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3960, 0, 1, 0, 856, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 1651, 0, 734, 606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 515, 70, 1031, 0, 1917, 267, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1898, 6396, 5751, 4340, 2751, 4818, 3013, 3801] avg=4096.0000 max=6396 max_rank=1 min=1898 min_rank=0 imbalance_ratio=1.561523 max_load_reduction=26372 max_load_reduction_pct=0.804810 moved_to_echo=14383 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 0, 2, 2332, 0, 1663, 6, 155, 82, 0, 0, 0, 0, 0, 0, 2406, 0, 4, 1698, 0, 572, 2, 296, 0, 0, 0, 0, 0, 0, 0, 0, 1131, 5, 2777, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 900, 0, 0, 0, 5, 3822, 0, 0, 0, 0, 0, 0, 0, 7, 1496, 0, 0, 0, 314, 73, 0, 1332, 170, 0, 0, 0, 0, 0, 0, 0, 0, 4, 30, 3704, 0, 0, 0, 909, 0, 0, 0, 0, 0, 0, 0, 45, 41, 0, 2446, 0, 441, 597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 335, 28, 688, 1, 1860, 272, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4246, 4978, 4000, 4729, 3392, 4647, 3570, 3206] avg=4096.0000 max=4978 max_rank=1 min=3206 min_rank=7 imbalance_ratio=1.215332 max_load_reduction=27790 max_load_reduction_pct=0.848083 moved_to_echo=14815 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3651, 0, 190, 0, 644, 45, 0, 0, 0, 0, 0, 0, 2889, 1, 0, 1, 0, 2369, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 480, 0, 1536, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, 2, 1, 0, 13, 3790, 0, 0, 0, 0, 0, 0, 0, 2, 1475, 2, 4, 1, 1493, 544, 0, 1368, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2410, 0, 0, 0, 502, 0, 0, 0, 0, 0, 0, 0, 656, 18, 0, 2173, 0, 1849, 612, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 0, 12, 0, 1193, 0, 1909, 270, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4530, 5299, 2238, 4104, 4982, 2912, 5312, 3391] avg=4096.0000 max=5312 max_rank=6 min=2238 min_rank=2 imbalance_ratio=1.296875 max_load_reduction=27456 max_load_reduction_pct=0.837891 moved_to_echo=16597 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 1, 3436, 0, 1368, 0, 360, 84, 0, 0, 0, 0, 0, 0, 54, 0, 0, 193, 0, 1327, 0, 145, 0, 0, 0, 0, 0, 0, 0, 0, 2001, 27, 2836, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 95, 10, 0, 1, 3756, 0, 0, 0, 0, 0, 0, 0, 0, 3560, 0, 0, 0, 121, 54, 1, 1403, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 3657, 0, 0, 0, 929, 0, 0, 0, 0, 0, 0, 0, 214, 11, 0, 1076, 0, 730, 628, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 0, 190, 8, 2022, 0, 1958, 268, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5253, 1719, 4865, 3902, 5312, 4601, 2660, 4456] avg=4096.0000 max=5312 max_rank=4 min=1719 min_rank=1 imbalance_ratio=1.296875 max_load_reduction=27456 max_load_reduction_pct=0.837891 moved_to_echo=17029 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 16512, 256, 128, 384, 3968, 768, 128, 9856, 1280, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[12544, 128, 19712, 512, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[12544, 128, 128, 3968, 128, 13952, 128, 2560, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1664, 256, 0, 14592, 256, 11776, 4480, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 21120, 0, 7296, 128, 3840, 640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 1152, 128, 0, 3584, 1152, 11264, 128, 13824, 2176, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 26240, 128, 128, 0, 6528, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 0, 2688, 128, 128, 0, 128, 30080, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 3041, 0, 2, 0, 14, 2984, 0, 0, 2380, 0, 69, 0, 0, 3664, 3499, 0, 21, 0, 0, 0, 0, 0, 0, 7, 98, 0, 4095, 0, 0, 3801, 0, 0, 0, 0, 0, 0, 0, 3652, 78, 1, 0, 1, 0, 0, 895, 0, 0, 3596, 0, 5, 0, 0, 0, 0, 0, 851, 0, 0, 13] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3931, 0, 0, 0, 735, 713, 0, 0, 3177, 0, 7, 0, 0, 3083, 3619, 6, 0, 0, 0, 0, 0, 8, 13, 1, 231, 0, 4094, 0, 0, 3628, 0, 7, 0, 0, 0, 0, 0, 4005, 77, 0, 0, 0, 0, 0, 1100, 0, 0, 4074, 0, 110, 0, 0, 0, 0, 0, 146, 2, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1, 1, 4, 0, 0, 3631, 0, 23, 9, 107, 1972, 0, 0, 123, 0, 80, 0, 0, 2283, 3831, 0, 18, 0, 0, 0, 0, 0, 0, 0, 189, 0, 4088, 0, 0, 3758, 0, 1, 0, 7, 0, 39, 0, 4088, 13, 0, 0, 0, 0, 15, 4039, 0, 0, 4096, 0, 288, 0, 5, 2, 0, 0, 54, 0, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3377, 1, 4, 0, 1826, 2562, 0, 0, 3477, 0, 51, 0, 16, 607, 3578, 0, 0, 0, 0, 0, 0, 0, 0, 16, 231, 0, 2718, 0, 11, 3904, 0, 0, 0, 0, 0, 274, 0, 861, 0, 0, 0, 0, 1, 0, 4084, 0, 0, 4033, 0, 15, 0, 0, 12, 0, 2, 1107, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 1905, 1602, 0, 0, 143, 0, 6, 0, 0, 3181, 4066, 0, 1, 0, 0, 0, 0, 0, 0, 0, 486, 0, 4085, 0, 0, 3042, 0, 0, 0, 0, 0, 0, 0, 2062, 6, 0, 0, 0, 0, 0, 3404, 0, 0, 4091, 0, 519, 0, 0, 0, 0, 0, 73, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3602, 0, 27, 0, 1577, 3706, 0, 0, 2452, 0, 0, 0, 1, 2281, 3112, 0, 1, 0, 0, 0, 0, 0, 4, 0, 293, 0, 4073, 0, 0, 2915, 0, 0, 0, 0, 0, 1, 0, 2492, 0, 0, 0, 0, 0, 0, 1816, 1, 0, 4095, 0, 70, 0, 0, 0, 0, 0, 249, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 8, 0, 0, 2775, 4, 0, 0, 1330, 2134, 0, 6, 34, 0, 64, 19, 1, 3993, 4091, 3, 0, 1, 1, 0, 12, 5, 2, 0, 950, 0, 4089, 0, 0, 2747, 0, 0, 11, 0, 0, 19, 0, 3054, 0, 22, 0, 0, 2, 0, 3242, 18, 0, 4050, 4, 62, 0, 0, 3, 3, 0, 3, 0, 6, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[12, 0, 0, 0, 0, 1063, 1, 0, 0, 6, 3616, 0, 0, 3086, 0, 21, 0, 0, 3935, 3889, 0, 1, 0, 0, 0, 0, 0, 0, 3, 10, 0, 4083, 0, 0, 3836, 0, 0, 0, 0, 0, 2, 0, 3832, 0, 0, 0, 0, 0, 0, 1063, 3, 0, 4081, 0, 78, 1, 0, 0, 0, 2, 144, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3931, 0, 0, 193, 236, 0, 0, 0, 0, 0, 0, 0, 735, 373, 0, 0, 3177, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3083, 1179, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 13, 1, 231, 0, 3949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3628, 0, 7, 0, 0, 0, 147, 145, 0, 0, 0, 0, 0, 0, 0, 0, 4005, 77, 0, 0, 0, 0, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1100, 0, 0, 1515, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 2, 0, 1, 2559, 1207, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4360, 4292, 4268, 4202, 3927, 5079, 2725, 3915] avg=4096.0000 max=5079 max_rank=5 min=2725 min_rank=6 imbalance_ratio=1.239990 max_load_reduction=27689 max_load_reduction_pct=0.845001 moved_to_echo=15646 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 3041, 0, 2, 807, 228, 0, 0, 0, 0, 0, 0, 0, 14, 1561, 0, 0, 2380, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3664, 1140, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 98, 0, 3950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3801, 0, 0, 0, 0, 0, 616, 145, 0, 0, 0, 0, 0, 0, 0, 0, 3652, 78, 1, 0, 1, 0, 964, 0, 0, 0, 0, 0, 0, 0, 0, 895, 0, 0, 1337, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 851, 0, 0, 13, 2259, 1167, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4024, 4825, 4055, 4562, 4696, 2237, 4290] avg=4096.0000 max=4825 max_rank=2 min=2237 min_rank=6 imbalance_ratio=1.177979 max_load_reduction=27943 max_load_reduction_pct=0.852753 moved_to_echo=15785 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 4, 0, 0, 3631, 0, 23, 533, 250, 0, 0, 0, 0, 0, 0, 9, 107, 1032, 0, 0, 123, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2283, 1247, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 0, 3943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3758, 0, 1, 0, 7, 0, 407, 145, 0, 0, 0, 0, 0, 0, 39, 0, 4088, 13, 0, 0, 0, 0, 1056, 0, 0, 0, 0, 0, 0, 0, 15, 4039, 0, 0, 1523, 0, 288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 0, 0, 54, 0, 3, 0, 2573, 1278, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4443, 1351, 3548, 4132, 4318, 5196, 5865, 3915] avg=4096.0000 max=5865 max_rank=6 min=1351 min_rank=1 imbalance_ratio=1.431885 max_load_reduction=26903 max_load_reduction_pct=0.821014 moved_to_echo=19294 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3377, 1, 4, 693, 233, 0, 0, 0, 0, 0, 0, 0, 1826, 1340, 0, 0, 3477, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 607, 1165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 231, 0, 2622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 3904, 0, 0, 0, 0, 0, 529, 96, 0, 0, 0, 0, 0, 0, 274, 0, 861, 0, 0, 0, 0, 1, 986, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 1499, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 2, 1107, 0, 0, 0, 2534, 1194, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4308, 6694, 1788, 2869, 4540, 2122, 5598, 4849] avg=4096.0000 max=6694 max_rank=1 min=1788 min_rank=2 imbalance_ratio=1.634277 max_load_reduction=26074 max_load_reduction_pct=0.795715 moved_to_echo=17109 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3602, 0, 27, 1003, 203, 0, 0, 0, 0, 0, 0, 0, 1577, 1938, 0, 0, 2452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2281, 1013, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 293, 0, 3928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2915, 0, 0, 0, 0, 0, 765, 145, 0, 0, 0, 0, 0, 0, 1, 0, 2492, 0, 0, 0, 0, 0, 858, 0, 0, 0, 0, 0, 0, 0, 0, 1816, 1, 0, 1522, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 249, 0, 0, 0, 2573, 1038, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4835, 5967, 3296, 4225, 3825, 3351, 3409, 3860] avg=4096.0000 max=5967 max_rank=1 min=3296 min_rank=2 imbalance_ratio=1.456787 max_load_reduction=26801 max_load_reduction_pct=0.817902 moved_to_echo=14445 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 438, 270, 0, 0, 0, 0, 0, 0, 0, 1905, 830, 0, 0, 143, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3181, 1310, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 0, 3935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3042, 0, 0, 0, 0, 0, 334, 150, 0, 0, 0, 0, 0, 0, 0, 0, 2062, 6, 0, 0, 0, 0, 1125, 0, 0, 0, 0, 0, 0, 0, 0, 3404, 0, 0, 1517, 0, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 2574, 1361, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4804, 2884, 4492, 4421, 3526, 3193, 5440, 4008] avg=4096.0000 max=5440 max_rank=6 min=2884 min_rank=1 imbalance_ratio=1.328125 max_load_reduction=27328 max_load_reduction_pct=0.833984 moved_to_echo=16167 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8, 0, 0, 2775, 4, 0, 577, 267, 0, 0, 0, 0, 0, 0, 0, 1330, 1117, 0, 6, 34, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 19, 1, 3993, 1331, 3, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 5, 2, 0, 950, 0, 3944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2747, 0, 0, 11, 0, 0, 440, 145, 0, 0, 0, 0, 0, 0, 19, 0, 3054, 0, 22, 0, 0, 2, 1128, 0, 0, 0, 0, 0, 0, 0, 0, 3242, 18, 0, 1506, 4, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 3, 0, 6, 0, 2544, 1365, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3631, 2551, 5349, 4913, 3343, 4225, 4832, 3924] avg=4096.0000 max=5349 max_rank=2 min=2551 min_rank=1 imbalance_ratio=1.305908 max_load_reduction=27419 max_load_reduction_pct=0.836761 moved_to_echo=16324 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 0, 0, 0, 1063, 1, 0, 978, 254, 0, 0, 0, 0, 0, 0, 0, 6, 1892, 0, 0, 3086, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3935, 1266, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 10, 0, 3938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3836, 0, 0, 0, 0, 0, 746, 145, 0, 0, 0, 0, 0, 0, 2, 0, 3832, 0, 0, 0, 0, 0, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 3, 0, 1517, 0, 78, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 144, 0, 0, 0, 2564, 1297, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2308, 5005, 5202, 3951, 4727, 4906, 2662, 4007] avg=4096.0000 max=5202 max_rank=2 min=2308 min_rank=0 imbalance_ratio=1.270020 max_load_reduction=27566 max_load_reduction_pct=0.841248 moved_to_echo=16302 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 2688, 128, 128, 128, 20224, 9984, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 7552, 10112, 0, 128, 14976, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 19712, 128, 0, 12032, 128, 1152, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 23040, 9728, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 27648, 0, 128, 128, 128, 0, 4096, 1152, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[384, 0, 24064, 256, 128, 0, 128, 128, 8192, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 0, 25600, 128, 128, 5248, 2048, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 128, 2560, 0, 30336, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33792 slack_slots=31744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 50, 0, 21, 2956, 0, 0, 1757, 4085, 0, 0, 0, 0, 0, 0, 0, 13, 4, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2331, 0, 1, 0, 0, 0, 0, 0, 0, 2, 19, 3636, 4094, 0, 3958, 3137, 0, 0, 1, 0, 0, 753, 0, 1399, 0, 476, 0, 0, 0, 0, 0, 0, 4073] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[39, 1001, 0, 1, 15, 0, 0, 3075, 4082, 0, 27, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 14, 0, 540, 2, 0, 0, 0, 519, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 4064, 3939, 1, 2706, 3560, 0, 0, 0, 0, 0, 807, 16, 328, 0, 2479, 0, 0, 0, 0, 0, 0, 1396] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 274, 0, 1, 2252, 1, 0, 2853, 4005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 81, 0, 0, 1369, 0, 0, 3, 0, 0, 0, 0, 0, 0, 848, 3934, 4087, 0, 3943, 1290, 0, 6, 0, 0, 9, 2, 16, 679, 0, 3062, 0, 0, 0, 0, 0, 0, 4039] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[4, 0, 0, 0, 1953, 0, 0, 2510, 4039, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 3, 0, 0, 1977, 0, 0, 11, 0, 0, 0, 0, 1, 9, 0, 4007, 3987, 0, 4090, 2011, 0, 0, 0, 0, 0, 168, 29, 310, 0, 3527, 0, 0, 0, 0, 0, 0, 4055] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 38, 0, 0, 3716, 1, 0, 1770, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 2479, 0, 0, 0, 0, 0, 0, 3, 0, 0, 25, 4071, 4096, 0, 4041, 870, 0, 0, 0, 0, 0, 213, 0, 1, 0, 3281, 0, 0, 0, 0, 0, 0, 4062] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[4, 2, 0, 0, 3854, 0, 0, 443, 4060, 18, 0, 6, 0, 3, 2, 0, 4, 0, 0, 10, 7, 0, 2, 175, 0, 5, 0, 0, 0, 10, 0, 2946, 0, 1, 0, 12, 0, 0, 1, 0, 0, 48, 4088, 4081, 0, 4082, 301, 0, 2, 0, 0, 0, 926, 0, 581, 0, 3225, 2, 0, 0, 7, 0, 0, 3860] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[3, 224, 0, 2, 2556, 0, 0, 2599, 4093, 7, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 1, 0, 119, 0, 0, 0, 38, 131, 13, 3, 2870, 0, 0, 5, 0, 0, 0, 0, 4, 0, 5, 4071, 3974, 12, 4079, 26, 0, 0, 0, 1, 0, 25, 5, 113, 0, 3693, 0, 0, 0, 0, 0, 0, 4091] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 18, 0, 3, 3686, 0, 0, 1584, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 47, 0, 1, 0, 0, 15, 4, 0, 1310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 227, 4091, 4093, 0, 3962, 1248, 0, 4, 0, 0, 0, 85, 0, 421, 0, 3979, 0, 0, 0, 0, 0, 0, 3891] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 0, 0, 1349, 0, 0, 2510, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 0, 14, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 3967, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1977, 592, 823, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 1, 3987, 40, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 2536, 2011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 29, 310, 0, 2799, 731, 0, 0, 0, 0, 0, 0, 3527, 0, 0, 0, 0, 0, 0, 1245, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3863, 4065, 4041, 3395, 4039, 4556, 4037, 4772] avg=4096.0000 max=4772 max_rank=7 min=3395 min_rank=3 imbalance_ratio=1.165039 max_load_reduction=27996 max_load_reduction_pct=0.854370 moved_to_echo=17404 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 50, 0, 21, 2041, 0, 0, 1757, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 13, 4, 0, 0, 0, 0, 0, 2, 3600, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2331, 896, 796, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4094, 36, 0, 0, 0, 0, 0, 0, 2, 19, 0, 0, 0, 2454, 3137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 753, 0, 1399, 0, 2812, 708, 0, 0, 0, 0, 0, 0, 476, 0, 0, 0, 0, 0, 0, 1250, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3869, 4104, 3630, 4023, 4131, 5612, 5673, 1726] avg=4096.0000 max=5673 max_rank=6 min=1726 min_rank=7 imbalance_ratio=1.385010 max_load_reduction=27095 max_load_reduction_pct=0.826874 moved_to_echo=17142 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[39, 1001, 0, 1, 11, 0, 0, 3075, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 14, 0, 540, 4025, 3, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 519, 0, 0, 4088, 4, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 3939, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1678, 3560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 807, 16, 328, 0, 963, 484, 0, 0, 0, 0, 0, 0, 2479, 0, 0, 0, 0, 0, 0, 430, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4127, 4109, 4588, 5157, 4041, 5239, 2598, 2909] avg=4096.0000 max=5239 max_rank=5 min=2598 min_rank=6 imbalance_ratio=1.279053 max_load_reduction=27529 max_load_reduction_pct=0.840118 moved_to_echo=14787 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4, 2, 0, 0, 2662, 0, 0, 443, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 18, 0, 6, 0, 3, 2, 0, 24, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 10, 7, 0, 2, 175, 4048, 11, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 10, 0, 2946, 1168, 821, 0, 0, 0, 0, 0, 0, 0, 1, 0, 12, 0, 0, 1, 0, 4081, 40, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 2531, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 926, 0, 581, 0, 2665, 730, 0, 0, 0, 0, 0, 0, 3225, 2, 0, 0, 7, 0, 0, 1184, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3111, 4113, 4257, 4950, 4135, 2880, 4904, 4418] avg=4096.0000 max=4950 max_rank=3 min=2880 min_rank=5 imbalance_ratio=1.208496 max_load_reduction=27818 max_load_reduction_pct=0.848938 moved_to_echo=16337 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 274, 0, 1, 1556, 1, 0, 2853, 0, 0, 0, 0, 0, 0, 0, 0, 4005, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 3895, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 1369, 682, 793, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4087, 39, 0, 0, 0, 0, 0, 0, 0, 848, 0, 0, 0, 2445, 1290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 9, 2, 16, 679, 0, 2788, 705, 0, 0, 0, 0, 0, 0, 3062, 0, 0, 0, 0, 0, 0, 1240, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4685, 4019, 3920, 2925, 4129, 4583, 4205, 4302] avg=4096.0000 max=4685 max_rank=0 min=2925 min_rank=3 imbalance_ratio=1.143799 max_load_reduction=28083 max_load_reduction_pct=0.857025 moved_to_echo=17219 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 38, 0, 0, 2559, 1, 0, 1770, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4032, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2479, 1130, 816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 39, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 2498, 870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 213, 0, 1, 0, 2809, 727, 0, 0, 0, 0, 0, 0, 3281, 0, 0, 0, 0, 0, 0, 1238, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4368, 4123, 4052, 4425, 4138, 3393, 3750, 4519] avg=4096.0000 max=4519 max_rank=7 min=3393 min_rank=5 imbalance_ratio=1.103271 max_load_reduction=28249 max_load_reduction_pct=0.862091 moved_to_echo=15800 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3, 224, 0, 2, 1766, 0, 0, 2599, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 7, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 1, 0, 119, 4032, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 131, 13, 3, 2870, 774, 820, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4, 3974, 39, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 12, 2530, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 25, 5, 113, 0, 2824, 729, 0, 0, 0, 0, 0, 0, 3693, 0, 0, 0, 0, 0, 0, 1256, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4594, 4116, 4168, 4649, 4022, 2573, 3697, 4949] avg=4096.0000 max=4949 max_rank=7 min=2573 min_rank=5 imbalance_ratio=1.208252 max_load_reduction=27819 max_load_reduction_pct=0.848969 moved_to_echo=15241 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 18, 0, 3, 2546, 0, 0, 1584, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 47, 4051, 11, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 15, 4, 0, 1310, 1117, 797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 40, 0, 0, 0, 0, 0, 0, 0, 227, 0, 0, 0, 2457, 1248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 85, 0, 421, 0, 2686, 708, 0, 0, 0, 0, 0, 0, 3979, 0, 0, 0, 0, 0, 0, 1194, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4151, 4119, 4112, 3244, 4133, 3932, 3904, 5173] avg=4096.0000 max=5173 max_rank=7 min=3244 min_rank=3 imbalance_ratio=1.262939 max_load_reduction=27595 max_load_reduction_pct=0.842133 moved_to_echo=17142 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 3072, 128, 3840, 0, 20352, 5632, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[23808, 128, 0, 0, 128, 0, 0, 9088, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 1664, 0, 128, 14592, 128, 0, 16640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 768, 128, 128, 19456, 6400, 6272, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 1280, 0, 0, 128, 19200, 12544, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 0, 128, 128, 128, 128, 1024, 31744, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 0, 0, 128, 128, 32384, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[32640, 128, 128, 128, 128, 128, 128, 0, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 206, 0, 449, 0, 0, 2, 0, 0, 3806, 0, 17, 13, 0, 1, 773, 3079, 1, 0, 0, 0, 0, 800, 51, 0, 1, 0, 58, 0, 0, 4079, 0, 0, 0, 0, 1, 122, 4, 0, 0, 0, 0, 3929, 843, 0, 2297, 0, 0, 4095, 1, 0, 0, 0, 0, 4096, 4, 0, 4040, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 1, 0, 4055, 1, 68, 3, 0, 0, 0, 0, 3759, 0, 383, 0, 0, 0, 456, 9, 229, 0, 0, 0, 12, 3923, 1, 0, 0, 0, 158, 0, 0, 4090, 0, 0, 0, 0, 8, 0, 0, 1, 0, 0, 2, 3772, 48, 0, 3477, 0, 0, 4096, 0, 0, 0, 0, 0, 124, 0, 0, 4092, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1, 0, 0, 3704, 0, 241, 0, 0, 0, 0, 0, 4050, 0, 87, 3, 0, 0, 2496, 34, 25, 0, 0, 0, 0, 339, 26, 0, 0, 0, 117, 0, 0, 4019, 0, 0, 0, 0, 10, 241, 0, 0, 6, 0, 1, 4091, 84, 0, 3641, 0, 0, 4077, 0, 0, 0, 0, 0, 4051, 0, 0, 1424, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[8, 0, 0, 208, 0, 5, 0, 0, 0, 12, 0, 4086, 0, 1238, 0, 0, 7, 1884, 2667, 0, 4, 0, 416, 0, 262, 0, 0, 0, 0, 602, 4, 7, 4078, 9, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4084, 12, 11, 2397, 0, 1, 4088, 0, 3, 0, 1, 8, 3047, 0, 0, 3607, 0, 7, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 721, 0, 110, 0, 0, 0, 0, 0, 4089, 0, 164, 0, 0, 0, 10, 2830, 9, 0, 0, 0, 0, 100, 341, 0, 0, 0, 712, 0, 0, 3764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1384, 0, 2682, 0, 0, 4096, 0, 0, 0, 0, 0, 4061, 0, 0, 3599, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 1, 6, 3170, 0, 1, 8, 25, 0, 0, 0, 3966, 0, 1770, 0, 0, 1, 56, 749, 0, 0, 3, 241, 0, 1717, 571, 28, 0, 0, 2, 0, 0, 4005, 0, 0, 0, 15, 0, 14, 0, 0, 4, 3, 0, 4065, 548, 0, 1753, 0, 0, 4024, 0, 0, 0, 2, 0, 3701, 0, 0, 2319, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 3949, 0, 85, 0, 0, 0, 0, 0, 4033, 0, 0, 0, 0, 0, 738, 700, 52, 0, 0, 2, 0, 292, 0, 20, 0, 0, 1358, 0, 0, 4074, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3712, 523, 0, 3528, 0, 0, 4067, 0, 0, 0, 0, 0, 2995, 0, 0, 2639, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 934, 0, 20, 0, 0, 0, 0, 0, 3949, 2, 63, 2, 1, 0, 1685, 196, 14, 0, 0, 11, 0, 3028, 6, 0, 0, 0, 542, 1, 0, 4081, 0, 0, 0, 0, 2, 51, 0, 0, 0, 0, 0, 4076, 24, 0, 4036, 0, 1, 4087, 0, 0, 0, 1, 1, 2684, 0, 0, 3269, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 3704, 0, 241, 0, 0, 1113, 349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3703, 0, 87, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2496, 34, 25, 0, 0, 0, 0, 619, 320, 0, 0, 0, 0, 0, 0, 339, 26, 0, 0, 0, 117, 0, 0, 2275, 0, 0, 0, 0, 0, 0, 0, 4019, 0, 0, 0, 0, 10, 241, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1, 703, 84, 0, 3641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 4051, 0, 0, 456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5408, 3793, 3494, 2757, 4282, 4435, 4092, 4507] avg=4096.0000 max=5408 max_rank=0 min=2757 min_rank=3 imbalance_ratio=1.320312 max_load_reduction=27360 max_load_reduction_pct=0.834961 moved_to_echo=17316 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 0, 208, 0, 5, 0, 0, 1111, 884, 0, 0, 0, 0, 0, 0, 0, 12, 0, 3736, 0, 1238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1884, 2667, 0, 4, 0, 416, 0, 1568, 323, 0, 0, 0, 0, 0, 0, 262, 0, 0, 0, 0, 602, 4, 7, 2271, 0, 0, 0, 0, 0, 0, 0, 4078, 9, 0, 0, 0, 0, 5, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 702, 12, 11, 2397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4088, 0, 3, 0, 1, 8, 15, 0, 0, 0, 0, 0, 0, 0, 3047, 0, 0, 1155, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2216, 4986, 6869, 3146, 4104, 3122, 4116, 4209] avg=4096.0000 max=6869 max_rank=2 min=2216 min_rank=0 imbalance_ratio=1.677002 max_load_reduction=25899 max_load_reduction_pct=0.790375 moved_to_echo=15551 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 206, 0, 449, 0, 0, 1069, 990, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3480, 0, 17, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 773, 3079, 1, 0, 0, 0, 0, 1756, 301, 0, 0, 0, 0, 0, 0, 800, 51, 0, 1, 0, 58, 0, 0, 2185, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 1, 122, 4, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 675, 843, 0, 2297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 1294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2714, 3512, 5911, 3095, 4217, 3815, 4110, 5394] avg=4096.0000 max=5911 max_rank=2 min=2714 min_rank=0 imbalance_ratio=1.443115 max_load_reduction=26857 max_load_reduction_pct=0.819611 moved_to_echo=17536 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 4055, 1, 68, 3, 0, 1026, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3437, 0, 383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 456, 9, 229, 0, 0, 0, 12, 1779, 297, 0, 0, 0, 0, 0, 0, 3923, 1, 0, 0, 0, 158, 0, 0, 2097, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 8, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 649, 48, 0, 3477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 1310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6157, 3820, 2782, 6179, 4109, 4177, 4110, 1434] avg=4096.0000 max=6179 max_rank=3 min=1434 min_rank=7 imbalance_ratio=1.508545 max_load_reduction=26589 max_load_reduction_pct=0.811432 moved_to_echo=13830 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3949, 0, 85, 0, 0, 1010, 647, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 738, 700, 52, 0, 0, 2, 0, 1147, 319, 0, 0, 0, 0, 0, 0, 292, 0, 20, 0, 0, 1358, 0, 0, 2064, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 1, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 638, 523, 0, 3528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 2995, 0, 0, 845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5691, 3687, 2958, 3734, 4087, 4689, 4082, 3840] avg=4096.0000 max=5691 max_rank=0 min=2958 min_rank=2 imbalance_ratio=1.389404 max_load_reduction=27077 max_load_reduction_pct=0.826324 moved_to_echo=16698 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 721, 0, 110, 0, 0, 1117, 885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3729, 0, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 2830, 9, 0, 0, 0, 0, 1567, 327, 0, 0, 0, 0, 0, 0, 100, 341, 0, 0, 0, 712, 0, 0, 2282, 0, 0, 0, 0, 0, 0, 0, 3764, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 697, 1384, 0, 2682, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 1147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2833, 3893, 4743, 3435, 3779, 4763, 4114, 5208] avg=4096.0000 max=5208 max_rank=7 min=2833 min_rank=0 imbalance_ratio=1.271484 max_load_reduction=27560 max_load_reduction_pct=0.841064 moved_to_echo=17864 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 6, 3170, 0, 1, 8, 25, 1106, 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3627, 0, 1770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 56, 749, 0, 0, 3, 241, 0, 1008, 314, 0, 0, 0, 0, 0, 0, 1717, 571, 28, 0, 0, 2, 0, 0, 2260, 0, 0, 0, 0, 0, 0, 0, 4005, 0, 0, 0, 15, 0, 14, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 0, 699, 548, 0, 1753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 0, 0, 0, 2, 0, 14, 0, 0, 0, 0, 0, 0, 0, 3701, 0, 0, 743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4885, 5397, 2372, 4578, 4045, 3007, 4040, 4444] avg=4096.0000 max=5397 max_rank=1 min=2372 min_rank=2 imbalance_ratio=1.317627 max_load_reduction=27371 max_load_reduction_pct=0.835297 moved_to_echo=15536 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 934, 0, 20, 0, 0, 1109, 801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3612, 2, 63, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1685, 196, 14, 0, 0, 11, 0, 1421, 312, 0, 0, 0, 0, 0, 0, 3028, 6, 0, 0, 0, 542, 1, 0, 2267, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 2, 51, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 700, 24, 0, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4087, 0, 0, 0, 1, 1, 14, 0, 0, 0, 0, 0, 0, 0, 2684, 0, 0, 1047, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2864, 3680, 3639, 5844, 4145, 4760, 4104, 3732] avg=4096.0000 max=5844 max_rank=3 min=2864 min_rank=0 imbalance_ratio=1.426758 max_load_reduction=26924 max_load_reduction_pct=0.821655 moved_to_echo=16741 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 17024, 128, 1024, 128, 128, 8704, 6144, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 8192, 10368, 384, 128, 128, 768, 128, 10880, 2560, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[10496, 1024, 128, 128, 0, 3584, 128, 128, 17792, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 5504, 3584, 128, 23936, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[24832, 128, 0, 8064, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 0, 29056, 128, 3840, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[32256, 128, 0, 0, 128, 128, 512, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 32640, 128, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 464, 0, 2810, 319, 0, 0, 0, 0, 0, 0, 103, 1088, 0, 2, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 4096, 0, 3851, 2, 4076, 3, 0, 0, 0, 4095, 0, 434, 0, 0, 0, 1, 0, 0, 0, 39, 38, 0, 0, 92, 0, 0, 2267, 3240, 0, 0, 0, 0, 0, 0, 1664, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 245, 0, 1823, 620, 0, 0, 0, 0, 0, 0, 1102, 172, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 3781, 4096, 1, 4082, 0, 4086, 0, 0, 0, 0, 4096, 0, 2743, 0, 0, 0, 7, 0, 0, 0, 0, 1, 0, 0, 28, 0, 0, 2200, 3670, 0, 0, 0, 0, 0, 0, 9, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[5, 0, 13, 15, 3296, 1650, 0, 0, 12, 0, 3, 0, 212, 602, 0, 29, 0, 2, 0, 0, 0, 1, 2, 0, 0, 0, 3949, 4078, 0, 2427, 0, 3648, 30, 0, 10, 0, 4083, 3, 2171, 0, 0, 0, 0, 0, 36, 0, 22, 0, 0, 0, 0, 0, 13, 2315, 4083, 6, 0, 0, 8, 4, 0, 17, 9, 14] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[6, 0, 0, 0, 1479, 2143, 0, 0, 0, 0, 0, 0, 3711, 3467, 0, 1, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 4091, 4081, 0, 3369, 0, 4033, 0, 0, 0, 0, 4069, 0, 99, 0, 0, 4, 0, 0, 0, 0, 49, 41, 0, 0, 0, 0, 0, 48, 2035, 0, 27, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 11, 0, 2720, 583, 0, 0, 0, 0, 0, 0, 3168, 668, 0, 0, 0, 0, 0, 6, 0, 0, 0, 671, 0, 0, 3696, 4093, 2, 1568, 0, 4004, 7, 0, 0, 0, 4096, 0, 121, 3, 2, 1, 149, 0, 0, 0, 31, 12, 0, 0, 0, 0, 0, 3208, 3797, 0, 0, 0, 0, 0, 0, 151, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[9, 3, 20, 0, 2925, 2040, 0, 0, 0, 0, 0, 0, 683, 216, 0, 0, 14, 0, 4, 0, 0, 0, 1, 0, 0, 0, 3385, 4086, 0, 2530, 0, 3217, 0, 0, 0, 0, 4096, 0, 3634, 0, 0, 0, 0, 0, 0, 0, 0, 29, 1, 0, 11, 0, 0, 2080, 3767, 0, 0, 0, 0, 0, 0, 17, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2252, 12, 0, 0, 0, 0, 0, 0, 3847, 1209, 0, 0, 5, 0, 0, 0, 0, 0, 0, 215, 0, 7, 4094, 4095, 0, 3460, 0, 3703, 0, 0, 0, 0, 4096, 0, 22, 0, 0, 4, 0, 0, 0, 0, 1, 4, 0, 0, 0, 0, 0, 1661, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 1, 0, 2865, 2062, 0, 1, 0, 0, 0, 0, 2298, 1906, 0, 0, 0, 0, 0, 2, 0, 0, 0, 10, 0, 0, 4096, 4093, 2, 593, 0, 3644, 0, 0, 0, 0, 4095, 1, 380, 0, 0, 0, 444, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 2193, 4075, 1, 0, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 464, 0, 2810, 319, 0, 0, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 1088, 0, 2, 906, 115, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 4044, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3851, 2, 1458, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 2892, 0, 434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 39, 38, 3988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 2267, 1877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1664, 0, 0, 2618, 1248, 30, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3890, 2214, 4162, 5311, 3329, 4066, 4236, 5560] avg=4096.0000 max=5560 max_rank=7 min=2214 min_rank=1 imbalance_ratio=1.357422 max_load_reduction=27208 max_load_reduction_pct=0.830322 moved_to_echo=17191 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 13, 15, 3296, 1650, 0, 0, 296, 0, 0, 0, 0, 0, 0, 0, 12, 0, 3, 0, 212, 602, 0, 29, 903, 145, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 2, 0, 3919, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2427, 0, 1305, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 10, 0, 2884, 3, 2171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 22, 0, 3970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 2315, 2365, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4, 0, 17, 9, 14, 2343, 1573, 30, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5275, 1906, 4032, 3732, 5098, 4028, 4699, 3998] avg=4096.0000 max=5275 max_rank=0 min=1906 min_rank=1 imbalance_ratio=1.287842 max_load_reduction=27493 max_load_reduction_pct=0.839020 moved_to_echo=17823 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 0, 0, 1479, 2143, 0, 0, 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3711, 3467, 0, 1, 900, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4061, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3369, 0, 1443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2874, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 49, 41, 3972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 1179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 2590, 784, 30, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3923, 8151, 4185, 4812, 2973, 4066, 1227, 3431] avg=4096.0000 max=8151 max_rank=1 min=1227 min_rank=6 imbalance_ratio=1.989990 max_load_reduction=24617 max_load_reduction_pct=0.751251 moved_to_echo=11697 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 11, 0, 2720, 583, 0, 0, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3168, 668, 0, 0, 906, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 671, 3668, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1568, 0, 1432, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 2893, 0, 121, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 149, 0, 0, 0, 31, 12, 3985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3208, 2199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 0, 0, 2572, 1463, 28, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3611, 4877, 4453, 3002, 3024, 4180, 5407, 4214] avg=4096.0000 max=5407 max_rank=6 min=3002 min_rank=3 imbalance_ratio=1.320068 max_load_reduction=27361 max_load_reduction_pct=0.834991 moved_to_echo=16825 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2252, 12, 0, 0, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3847, 1209, 0, 0, 906, 145, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 215, 4064, 109, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 3460, 0, 1325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2893, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 1, 4, 3986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1661, 2364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2378, 1572, 30, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2561, 6107, 4393, 4792, 2915, 3995, 4025, 3980] avg=4096.0000 max=6107 max_rank=1 min=2561 min_rank=0 imbalance_ratio=1.490967 max_load_reduction=26661 max_load_reduction_pct=0.813629 moved_to_echo=14915 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 245, 0, 1823, 620, 0, 0, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1102, 172, 0, 0, 911, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 3754, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4082, 0, 1457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2886, 0, 2743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 1, 3987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 2200, 2117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 2629, 1419, 27, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2987, 2319, 3869, 5540, 5629, 3995, 4345, 4084] avg=4096.0000 max=5629 max_rank=4 min=2319 min_rank=1 imbalance_ratio=1.374268 max_load_reduction=27139 max_load_reduction_pct=0.828217 moved_to_echo=18053 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[9, 3, 20, 0, 2925, 2040, 0, 0, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 683, 216, 0, 0, 906, 134, 0, 0, 0, 0, 0, 0, 14, 0, 4, 0, 0, 0, 1, 0, 3361, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2530, 0, 1151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2893, 0, 3634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 3977, 0, 0, 0, 0, 0, 0, 0, 1, 0, 11, 0, 0, 2080, 2182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 2066, 1451, 24, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5294, 1939, 3489, 3681, 6527, 4006, 4274, 3558] avg=4096.0000 max=6527 max_rank=4 min=1939 min_rank=1 imbalance_ratio=1.593506 max_load_reduction=26241 max_load_reduction_pct=0.800812 moved_to_echo=18365 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 0, 2865, 2062, 0, 1, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2298, 1906, 0, 0, 906, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 10, 4065, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 593, 0, 1303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2892, 1, 380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 444, 0, 0, 0, 3, 0, 3985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2193, 2360, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2341, 1570, 31, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5227, 5255, 4185, 1898, 3273, 4432, 4555, 3943] avg=4096.0000 max=5255 max_rank=1 min=1898 min_rank=3 imbalance_ratio=1.282959 max_load_reduction=27513 max_load_reduction_pct=0.839630 moved_to_echo=16203 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 15232, 9344, 0, 128, 7296, 1152, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 0, 1920, 128, 128, 19584, 11136, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 0, 256, 128, 128, 16000, 16768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 128, 21888, 128, 10880, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 768, 128, 20224, 9472, 0, 128, 2432, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 23168, 128, 9728, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 0, 128, 128, 1024, 30976, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 640, 0, 128, 0, 256, 128, 31872, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 9, 0, 0, 0, 31, 0, 0, 1, 0, 0, 0, 4002, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 16, 56, 0, 0, 0, 0, 0, 0, 4049, 4085, 3971, 1, 0, 0, 0, 1015, 0, 0, 0, 3292, 2960, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 632, 490] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 0, 0, 0, 13, 2, 0, 1, 0, 0, 0, 3963, 0, 0, 0, 0, 0, 7, 0, 0, 4095, 0, 0, 0, 15, 34, 0, 0, 0, 3, 0, 0, 4078, 4090, 4089, 74, 0, 0, 0, 1626, 0, 0, 0, 3887, 193, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 0, 0, 2482, 9] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 16, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3734, 0, 0, 0, 0, 0, 71, 0, 0, 3972, 0, 0, 2, 0, 236, 0, 0, 0, 0, 0, 0, 3541, 4008, 3325, 379, 0, 0, 0, 1124, 0, 0, 0, 3999, 3900, 3, 0, 0, 0, 0, 0, 5, 1, 0, 0, 4088, 0, 0, 0, 48, 314] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[3, 5, 0, 1, 0, 0, 1, 0, 0, 8, 0, 6, 0, 19, 3435, 0, 0, 11, 19, 0, 0, 0, 0, 4084, 0, 1, 0, 0, 0, 0, 0, 8, 0, 0, 0, 4086, 4060, 4071, 0, 0, 13, 0, 3289, 11, 0, 2, 3544, 1995, 0, 0, 1, 9, 0, 0, 0, 0, 0, 4, 4081, 0, 0, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3780, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 4094, 3242, 0, 0, 0, 0, 3711, 0, 0, 0, 3951, 1037, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4093, 0, 0, 0, 521, 87] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2223, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 9, 93, 0, 0, 0, 0, 0, 0, 4096, 4096, 3741, 7, 0, 0, 0, 256, 0, 0, 0, 4092, 3307, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2596, 34] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4, 0, 1, 0, 0, 0, 0, 0, 0, 3999, 0, 0, 0, 0, 0, 73, 0, 0, 4040, 0, 0, 0, 9, 1, 0, 0, 0, 0, 0, 0, 3961, 4073, 3690, 34, 38, 0, 27, 417, 0, 3, 0, 3316, 2104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 0, 0, 0, 2596, 360] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 21, 0, 0, 12, 0, 0, 0, 1, 0, 0, 0, 2326, 0, 0, 0, 0, 2, 1, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4094, 4046, 4087, 18, 1, 0, 0, 2675, 0, 1, 1, 3865, 2083, 0, 28, 1, 0, 0, 0, 1, 0, 0, 1, 4094, 0, 1, 2, 1309, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 9, 0, 0, 0, 31, 3778, 297, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4002, 0, 659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 56, 0, 0, 0, 2882, 759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 0, 3971, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 0, 0, 0, 113, 2960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 4085, 17, 0, 0, 0, 0, 0, 0, 0, 0, 2665, 0, 0, 0, 632, 490, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4662, 4072, 3713, 4226, 4088, 4105, 3787] avg=4096.0000 max=4662 max_rank=1 min=3713 min_rank=3 imbalance_ratio=1.138184 max_load_reduction=28106 max_load_reduction_pct=0.857727 moved_to_echo=16206 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 16, 0, 0, 0, 2, 3304, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3734, 0, 659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 3972, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 236, 0, 0, 0, 3501, 759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 222, 0, 3325, 379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1124, 0, 0, 0, 137, 3900, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 5, 1, 4008, 15, 0, 0, 0, 0, 0, 0, 0, 0, 2665, 0, 0, 0, 48, 314, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3683, 4393, 4048, 4498, 3926, 5161, 4032, 3027] avg=4096.0000 max=5161 max_rank=5 min=3027 min_rank=7 imbalance_ratio=1.260010 max_load_reduction=27607 max_load_reduction_pct=0.842499 moved_to_echo=16146 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 0, 0, 0, 13, 3805, 351, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 3963, 0, 660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4095, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 34, 0, 0, 0, 3403, 761, 0, 0, 0, 0, 0, 0, 3, 0, 0, 256, 0, 4089, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1626, 0, 0, 0, 133, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4090, 17, 0, 0, 0, 0, 0, 0, 0, 0, 2670, 0, 0, 0, 2482, 9, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4175, 4626, 4107, 4213, 4422, 1952, 4112, 5161] avg=4096.0000 max=5161 max_rank=7 min=1952 min_rank=5 imbalance_ratio=1.260010 max_load_reduction=27607 max_load_reduction_pct=0.842499 moved_to_echo=15647 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3, 5, 0, 1, 0, 0, 1, 0, 3812, 320, 0, 0, 0, 0, 0, 0, 0, 8, 0, 6, 0, 19, 3435, 0, 658, 0, 0, 0, 0, 0, 0, 0, 0, 11, 19, 0, 0, 0, 0, 4084, 5, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8, 3102, 758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 257, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 3289, 11, 0, 2, 122, 1995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 9, 0, 0, 0, 0, 4060, 17, 0, 0, 0, 0, 0, 0, 0, 4, 2660, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4142, 4126, 4119, 3869, 4328, 5432, 4087, 2665] avg=4096.0000 max=5432 max_rank=5 min=2665 min_rank=7 imbalance_ratio=1.326172 max_load_reduction=27336 max_load_reduction_pct=0.834229 moved_to_echo=16512 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 0, 0, 0, 2, 3822, 370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2223, 0, 660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 93, 0, 0, 0, 3582, 761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 257, 0, 3741, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 140, 3307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 4096, 17, 0, 0, 0, 0, 0, 0, 0, 0, 2670, 0, 0, 0, 2596, 34, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4198, 2883, 4101, 4445, 4005, 3703, 4133, 5300] avg=4096.0000 max=5300 max_rank=7 min=2883 min_rank=1 imbalance_ratio=1.293945 max_load_reduction=27468 max_load_reduction_pct=0.838257 moved_to_echo=17141 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 110, 0, 0, 0, 0, 3780, 362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3780, 0, 664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3462, 763, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 0, 3242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3711, 0, 0, 0, 127, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 21, 0, 0, 0, 0, 0, 0, 0, 0, 2660, 0, 0, 0, 521, 87, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4252, 4444, 4100, 4225, 3488, 4875, 4116, 3268] avg=4096.0000 max=4875 max_rank=5 min=3268 min_rank=7 imbalance_ratio=1.190186 max_load_reduction=27893 max_load_reduction_pct=0.851227 moved_to_echo=15747 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4, 0, 1, 3696, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3999, 0, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 4040, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 0, 0, 0, 2903, 747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 248, 0, 3690, 34, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 417, 0, 3, 0, 113, 2104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 17, 0, 0, 0, 0, 0, 0, 0, 0, 2622, 0, 0, 0, 2596, 360, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4001, 4647, 4118, 3660, 4010, 2664, 4090, 5578] avg=4096.0000 max=5578 max_rank=7 min=2664 min_rank=5 imbalance_ratio=1.361816 max_load_reduction=27190 max_load_reduction_pct=0.829773 moved_to_echo=16342 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 21, 0, 0, 12, 0, 3820, 349, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2326, 0, 660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 4095, 5, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 3383, 760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 257, 0, 4087, 18, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2675, 0, 1, 1, 133, 2083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 1, 0, 0, 0, 1, 0, 4046, 17, 0, 0, 0, 0, 0, 0, 0, 1, 2669, 0, 1, 2, 1309, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4202, 2987, 4103, 4145, 4363, 4893, 4093, 3982] avg=4096.0000 max=4893 max_rank=5 min=2987 min_rank=1 imbalance_ratio=1.194580 max_load_reduction=27875 max_load_reduction_pct=0.850677 moved_to_echo=17331 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 21376, 0, 128, 128, 10240, 1408, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 0, 128, 27520, 0, 5376, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 14208, 128, 128, 128, 1024, 17664, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 512, 0, 0, 128, 26240, 6144, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 0, 2048, 0, 30336, 640, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 0, 256, 0, 128, 128, 128, 29824, 2816, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 256, 0, 0, 32640, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 128, 0, 128, 128, 32640, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33792 slack_slots=31744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 1, 0, 4077, 0, 0, 4096, 11, 3697, 41, 0, 0, 3889, 0, 0, 0, 12, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 3556, 0, 839, 0, 0, 0, 0, 0, 0, 2499, 294, 0, 0, 42, 0, 0, 0, 1521, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 439, 0, 1582, 0, 0, 4096, 0, 551, 1445, 0, 0, 4080, 0, 0, 0, 393, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 3673, 0, 0, 0, 0, 0, 0, 210, 3300, 0, 0, 14, 0, 0, 0, 619, 16, 0, 0, 0, 0, 0, 5, 0, 0, 0, 5, 52, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 1, 0, 3708, 0, 0, 4096, 0, 2104, 97, 0, 0, 4057, 0, 0, 0, 20, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 3965, 0, 19, 0, 0, 0, 0, 0, 0, 3547, 2889, 0, 0, 34, 0, 0, 0, 25, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 2622, 0, 0, 4095, 2, 3528, 361, 0, 0, 3377, 0, 0, 0, 221, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4090, 0, 725, 0, 0, 0, 0, 0, 0, 1458, 3288, 0, 0, 65, 0, 0, 0, 744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 3997, 0, 0, 4096, 8, 573, 0, 0, 0, 4086, 0, 0, 1, 0, 0, 0, 4093, 1, 0, 0, 4090, 0, 0, 4077, 0, 1621, 0, 0, 0, 0, 1, 0, 3173, 2371, 0, 0, 256, 2, 0, 0, 121, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 25, 4, 3931, 12, 0, 4070, 0, 3115, 4, 0, 0, 2747, 0, 13, 0, 7, 22, 0, 4084, 0, 0, 3, 4030, 0, 0, 4073, 0, 573, 24, 0, 10, 0, 0, 3, 1842, 1359, 22, 6, 349, 0, 0, 0, 1243, 1160, 0, 8, 0, 0, 0, 0, 0, 10, 8, 0, 0, 2, 0, 0, 0, 0, 0, 9, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 27, 0, 3129, 0, 0, 4094, 8, 3008, 110, 0, 0, 3666, 0, 0, 0, 100, 0, 0, 4091, 0, 0, 0, 4095, 0, 0, 4088, 0, 2620, 0, 0, 0, 0, 0, 2, 3476, 99, 0, 0, 2, 0, 0, 0, 62, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 22, 0, 4094, 1, 0, 4094, 0, 2967, 57, 0, 0, 3693, 2, 0, 1, 0, 0, 0, 4094, 0, 0, 1, 4095, 0, 0, 4080, 0, 1815, 0, 0, 0, 1, 0, 0, 884, 1557, 0, 0, 588, 0, 0, 0, 711, 6, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 4077, 0, 0, 635, 11, 0, 0, 0, 0, 0, 0, 0, 0, 3697, 41, 0, 0, 1459, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 4008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2316, 0, 839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2373, 294, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1521, 0, 0, 0, 0, 1773, 1240, 126, 88, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3460, 657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4724, 5197, 4020, 3155, 2709, 4748, 4118, 4097] avg=4096.0000 max=5197 max_rank=1 min=2709 min_rank=4 imbalance_ratio=1.268799 max_load_reduction=27571 max_load_reduction_pct=0.841400 moved_to_echo=15672 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 439, 0, 1582, 0, 0, 635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 1445, 0, 0, 1530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 393, 0, 0, 0, 0, 0, 0, 4008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2667, 0, 3673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 3300, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 619, 16, 0, 0, 0, 1860, 1429, 10, 88, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 5, 52, 3460, 690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2656, 3526, 4401, 6340, 3514, 4022, 4212, 4097] avg=4096.0000 max=6340 max_rank=3 min=2656 min_rank=0 imbalance_ratio=1.547852 max_load_reduction=26428 max_load_reduction_pct=0.806519 moved_to_echo=15845 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3997, 0, 0, 635, 8, 0, 0, 0, 0, 0, 0, 0, 0, 573, 0, 0, 0, 1532, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2655, 0, 1621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3013, 2371, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 121, 188, 0, 0, 0, 1863, 1422, 160, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 3460, 691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4640, 2106, 4003, 4276, 5641, 3844, 4164, 4094] avg=4096.0000 max=5641 max_rank=4 min=2106 min_rank=1 imbalance_ratio=1.377197 max_load_reduction=27127 max_load_reduction_pct=0.827850 moved_to_echo=17743 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 25, 4, 3931, 12, 0, 631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3115, 4, 0, 0, 1031, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 22, 0, 0, 0, 0, 3, 3943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2652, 0, 573, 24, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1749, 1359, 22, 6, 349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1243, 1160, 0, 8, 0, 1252, 1421, 93, 87, 0, 0, 0, 0, 0, 0, 0, 0, 10, 8, 0, 0, 3438, 464, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 9, 0, 4084, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4603, 4163, 3975, 3259, 3488, 5264, 3920, 4096] avg=4096.0000 max=5264 max_rank=5 min=3259 min_rank=3 imbalance_ratio=1.285156 max_load_reduction=27504 max_load_reduction_pct=0.839355 moved_to_echo=16768 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2622, 0, 0, 635, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3528, 361, 0, 0, 1267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 221, 0, 0, 0, 0, 0, 0, 4008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2663, 0, 725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1385, 3288, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 744, 0, 0, 0, 0, 1539, 1427, 73, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3459, 571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3259, 5156, 4229, 3388, 4738, 3871, 4030, 4097] avg=4096.0000 max=5156 max_rank=1 min=3259 min_rank=0 imbalance_ratio=1.258789 max_load_reduction=27612 max_load_reduction_pct=0.842651 moved_to_echo=16736 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 3708, 0, 0, 626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2104, 97, 0, 0, 1514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 4005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2577, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3365, 2889, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 14, 0, 0, 0, 1854, 1388, 182, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3466, 689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4335, 3715, 4025, 2596, 6288, 3554, 4155, 4100] avg=4096.0000 max=6288 max_rank=4 min=2596 min_rank=3 imbalance_ratio=1.535156 max_load_reduction=26480 max_load_reduction_pct=0.808105 moved_to_echo=18097 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 27, 0, 3129, 0, 0, 635, 8, 0, 0, 0, 0, 0, 0, 0, 0, 3008, 110, 0, 0, 1375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 4007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2662, 0, 2620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3301, 99, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 87, 0, 0, 0, 1671, 1426, 175, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3458, 620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3799, 4493, 4107, 5282, 3404, 3509, 4082, 4092] avg=4096.0000 max=5282 max_rank=3 min=3404 min_rank=4 imbalance_ratio=1.289551 max_load_reduction=27486 max_load_reduction_pct=0.838806 moved_to_echo=15087 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 22, 0, 4094, 1, 0, 635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2967, 57, 0, 0, 1385, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2657, 0, 1815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 840, 1557, 0, 0, 588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 711, 6, 0, 0, 0, 1684, 1423, 44, 88, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 3458, 624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4752, 4412, 4008, 4472, 2986, 3956, 4087, 4095] avg=4096.0000 max=4752 max_rank=0 min=2986 min_rank=4 imbalance_ratio=1.160156 max_load_reduction=28016 max_load_reduction_pct=0.854980 moved_to_echo=15124 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 0, 5120, 1536, 0, 128, 0, 13568, 11264, 896, 768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 20864, 0, 11904, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 16256, 15232, 128, 128, 1408, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 640, 128, 27264, 128, 0, 5120, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 128, 128, 128, 128, 128, 27776, 5120, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 0, 0, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[768, 128, 0, 0, 128, 0, 128, 32000, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33792 slack_slots=31744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[19584, 2176, 0, 0, 11136, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 1, 8, 4096, 0, 0, 1296, 1850, 0, 3990, 2, 0, 0, 0, 4082, 0, 0, 1301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 0, 4096, 0, 0, 0, 0, 141, 0, 4072, 0, 0, 0, 0, 3600, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 23, 9, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 3373, 302, 0, 3888, 11, 0, 0, 19, 4096, 0, 0, 83, 0, 0, 0, 0, 281, 0, 0, 0, 0, 0, 0, 0, 9, 9, 0, 4095, 0, 0, 0, 0, 287, 0, 4015, 0, 0, 0, 0, 4082, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 441, 0, 4069, 1, 0, 0, 0, 4096, 0, 0, 3043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4, 0, 4096, 0, 0, 0, 0, 786, 0, 4096, 0, 0, 0, 0, 3950, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 50, 1, 0, 0, 0, 0, 0, 64, 0, 3745, 0, 1, 94, 1169, 15, 3742, 0, 0, 0, 0, 4068, 0, 0, 2948, 5, 0, 0, 0, 0, 11, 0, 0, 11, 0, 0, 1, 0, 70, 0, 4044, 0, 22, 20, 1, 1489, 0, 3943, 48, 20, 5, 0, 3088, 4034, 0, 0, 0, 8, 17, 14, 20] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 2, 0, 3904, 0, 0, 699, 1792, 0, 3907, 0, 0, 0, 1, 4080, 0, 0, 1278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 132, 0, 4096, 0, 0, 0, 0, 703, 0, 4092, 0, 0, 0, 0, 3979, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 4078, 0, 0, 51, 3222, 0, 4067, 0, 0, 0, 0, 4096, 0, 0, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 0, 4096, 0, 0, 0, 0, 711, 0, 4089, 0, 0, 0, 0, 3027, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 1190, 616, 0, 4086, 6, 0, 0, 0, 4095, 0, 0, 3039, 0, 0, 0, 3, 0, 4, 0, 8, 0, 0, 0, 0, 0, 90, 0, 4095, 0, 0, 0, 0, 1281, 0, 4094, 0, 1, 1, 0, 1984, 4091, 0, 0, 0, 1, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 26, 4085, 0, 0, 708, 2902, 0, 2988, 0, 0, 0, 2, 4060, 0, 0, 2081, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 224, 0, 4096, 0, 0, 0, 0, 293, 0, 3966, 0, 0, 0, 0, 3208, 4095, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2, 1, 0, 4061, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8, 4096, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 1296, 1850, 0, 1689, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2345, 0, 0, 1301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2255, 1737, 146, 0, 0, 0, 0, 0, 0, 134, 0, 3889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 0, 11, 0, 0, 0, 0, 3600, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4166, 4837, 3646, 4138, 4023, 3752, 4096] avg=4096.0000 max=4837 max_rank=2 min=3646 min_rank=3 imbalance_ratio=1.180908 max_load_reduction=27931 max_load_reduction_pct=0.852386 moved_to_echo=16009 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 50, 1, 0, 0, 3932, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 3745, 0, 1, 60, 0, 0, 0, 0, 0, 0, 0, 94, 1169, 15, 1585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2337, 0, 0, 2948, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 11, 0, 0, 1, 2114, 1731, 145, 0, 0, 0, 0, 0, 0, 70, 0, 3839, 0, 22, 20, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1489, 0, 11, 48, 20, 5, 0, 3088, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 0, 0, 8, 17, 14, 20, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4026, 3870, 2863, 5290, 4013, 3952, 4661, 4093] avg=4096.0000 max=5290 max_rank=3 min=2863 min_rank=2 imbalance_ratio=1.291504 max_load_reduction=27478 max_load_reduction_pct=0.838562 moved_to_echo=16719 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 5, 0, 0, 4081, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3904, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 699, 1792, 0, 1654, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2344, 0, 0, 1278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2208, 1736, 146, 0, 0, 0, 0, 0, 2, 132, 0, 3889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 703, 0, 11, 0, 0, 0, 0, 3979, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4131, 3967, 4146, 3622, 4090, 4023, 4693, 4096] avg=4096.0000 max=4693 max_rank=6 min=3622 min_rank=3 imbalance_ratio=1.145752 max_load_reduction=28075 max_load_reduction_pct=0.856781 moved_to_echo=16902 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4085, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 441, 0, 1723, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2353, 0, 0, 3043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2299, 1743, 146, 0, 0, 0, 0, 0, 10, 4, 0, 3889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 786, 0, 11, 0, 0, 0, 0, 3950, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4132, 4141, 2165, 5396, 4188, 3903, 4747, 4096] avg=4096.0000 max=5396 max_rank=3 min=2165 min_rank=2 imbalance_ratio=1.317383 max_load_reduction=27372 max_load_reduction_pct=0.835327 moved_to_echo=16934 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 23, 9, 0, 4004, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 3373, 302, 0, 1646, 11, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 2353, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 281, 0, 0, 0, 0, 0, 0, 0, 2197, 1743, 146, 0, 0, 0, 0, 0, 9, 9, 0, 3888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 287, 0, 11, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4151, 5351, 2436, 4367, 3906, 4380, 4096] avg=4096.0000 max=5351 max_rank=2 min=2436 min_rank=3 imbalance_ratio=1.306396 max_load_reduction=27417 max_load_reduction_pct=0.836700 moved_to_echo=16749 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4082, 52, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 4078, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 51, 3222, 0, 1713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2349, 0, 0, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2302, 1747, 152, 0, 0, 0, 0, 0, 0, 1000, 0, 3870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 711, 0, 7, 0, 0, 0, 0, 3027, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4134, 4160, 4986, 2569, 4201, 4870, 3745, 4103] avg=4096.0000 max=4986 max_rank=2 min=2569 min_rank=3 imbalance_ratio=1.217285 max_load_reduction=27782 max_load_reduction_pct=0.847839 moved_to_echo=16919 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 33, 0, 3955, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 4085, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 708, 2902, 0, 1266, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2333, 0, 0, 2081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1688, 1727, 146, 0, 0, 0, 0, 0, 0, 224, 0, 3889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 0, 11, 0, 0, 0, 0, 3208, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4022, 4172, 4878, 4414, 3562, 4113, 3512, 4095] avg=4096.0000 max=4878 max_rank=2 min=3512 min_rank=6 imbalance_ratio=1.190918 max_load_reduction=27890 max_load_reduction_pct=0.851135 moved_to_echo=15282 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 1, 0, 0, 4083, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 1190, 616, 0, 1730, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2353, 0, 0, 3039, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8, 0, 0, 0, 0, 2309, 1742, 146, 0, 0, 0, 0, 0, 0, 90, 0, 3888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1281, 0, 11, 0, 1, 1, 0, 1984, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4132, 4141, 3542, 5395, 4209, 3978, 3278, 4093] avg=4096.0000 max=5395 max_rank=3 min=3278 min_rank=6 imbalance_ratio=1.317139 max_load_reduction=27373 max_load_reduction_pct=0.835358 moved_to_echo=15558 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[384, 128, 0, 128, 128, 0, 0, 128, 17408, 13952, 1280, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[7424, 12416, 128, 13056, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[18816, 0, 0, 14080, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[5760, 0, 128, 128, 128, 128, 0, 27008, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 128, 128, 128, 0, 32384, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 1664, 0, 31104, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[32768, 0, 0, 0, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 128, 128, 128, 32256, 0, 128, 512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 365, 0, 0, 0, 791, 0, 0, 4011, 28, 0, 4, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 4065, 4096, 4087, 0, 0, 0, 0, 39, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2990, 0, 0, 0, 4093, 0, 79, 0, 0, 0, 4063, 0, 0, 4017, 0, 5, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 88, 4, 0, 0, 84, 0, 14, 4071, 241, 0, 0, 1, 0, 0, 0, 0, 0, 0, 23, 43, 0, 0, 7, 4077, 4000, 4070, 0, 0, 5, 42, 148, 6, 0, 0, 0, 0, 0, 0, 31, 0, 24, 22, 18, 3940, 0, 0, 5, 4071, 0, 0, 0, 2, 0, 4048, 4, 20, 3382, 5, 0, 153, 9, 110, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 3597, 8, 0, 0, 0, 22, 0, 3977, 543, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 4085, 4096, 822, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 250, 0, 0, 0, 0, 3902, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3845, 0, 0, 3504, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 1425, 1, 0, 0, 247, 0, 0, 3919, 2, 0, 18, 1, 0, 0, 0, 0, 20, 0, 2, 0, 0, 0, 0, 4094, 4096, 3610, 0, 0, 0, 0, 959, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 2596, 0, 0, 0, 4096, 0, 48, 0, 0, 0, 3787, 0, 0, 3830, 0, 4, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 310, 0, 0, 0, 141, 0, 0, 4079, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 352, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3493, 0, 0, 0, 4096, 0, 10, 0, 0, 0, 4096, 0, 0, 3841, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 2878, 25, 0, 0, 726, 10, 0, 3749, 31, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4008, 4095, 3749, 0, 0, 0, 0, 364, 0, 0, 61, 0, 0, 0, 0, 128, 0, 0, 0, 0, 1885, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3939, 0, 0, 3005, 0, 13, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 1547, 0, 0, 0, 77, 0, 0, 3991, 141, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 3, 0, 0, 4091, 4093, 3872, 0, 0, 0, 0, 72, 0, 0, 0, 2, 0, 0, 0, 46, 0, 0, 0, 0, 4050, 0, 0, 0, 4096, 0, 29, 0, 0, 0, 3282, 0, 0, 3358, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 3355, 0, 0, 0, 322, 0, 1, 3893, 120, 0, 4, 0, 3, 0, 0, 0, 0, 1, 7, 0, 0, 6, 0, 4095, 4082, 3977, 0, 0, 0, 0, 98, 1, 0, 1, 0, 0, 0, 0, 9, 0, 0, 1, 1, 1387, 0, 0, 1, 4094, 0, 106, 0, 0, 0, 3833, 1, 0, 3366, 0, 0, 0, 1, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 365, 0, 0, 0, 791, 0, 0, 2085, 0, 0, 0, 0, 0, 0, 0, 3996, 28, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 302, 0, 4087, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3893, 124, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2990, 0, 0, 0, 1053, 0, 0, 0, 0, 0, 0, 0, 200, 0, 79, 0, 0, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4017, 0, 5, 0, 0, 0, 0, 501, 15, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3241, 4028, 4121, 4428, 4027, 4043, 4342, 4538] avg=4096.0000 max=4538 max_rank=7 min=3241 min_rank=0 imbalance_ratio=1.107910 max_load_reduction=28230 max_load_reduction_pct=0.861511 moved_to_echo=16950 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 88, 4, 0, 0, 84, 0, 14, 2091, 0, 0, 0, 0, 0, 0, 0, 4056, 241, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 43, 0, 0, 7, 3992, 0, 0, 0, 0, 0, 0, 0, 303, 0, 4070, 0, 0, 5, 42, 148, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 31, 3872, 125, 8, 0, 0, 0, 0, 0, 0, 24, 22, 18, 3940, 0, 0, 5, 1056, 0, 0, 0, 0, 0, 0, 0, 199, 0, 0, 0, 2, 0, 4048, 4, 0, 0, 0, 0, 0, 0, 0, 0, 20, 3382, 5, 0, 153, 9, 110, 0, 502, 15, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2281, 4298, 4065, 4568, 4042, 5065, 4253, 4196] avg=4096.0000 max=5065 max_rank=5 min=2281 min_rank=0 imbalance_ratio=1.236572 max_load_reduction=27703 max_load_reduction_pct=0.845428 moved_to_echo=17556 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3597, 8, 0, 0, 0, 22, 0, 2095, 0, 0, 0, 0, 0, 0, 0, 3962, 543, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 304, 0, 822, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 250, 3896, 125, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3902, 0, 0, 0, 1058, 0, 0, 0, 0, 0, 0, 0, 200, 0, 0, 0, 0, 0, 3845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3504, 0, 0, 0, 0, 0, 0, 503, 15, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5722, 4506, 4105, 1128, 4280, 4960, 4045, 4022] avg=4096.0000 max=5722 max_rank=0 min=1128 min_rank=3 imbalance_ratio=1.396973 max_load_reduction=27046 max_load_reduction_pct=0.825378 moved_to_echo=17307 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1425, 1, 0, 0, 247, 0, 0, 2100, 0, 0, 0, 0, 0, 0, 0, 3905, 2, 0, 18, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 2, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 304, 0, 3610, 0, 0, 0, 0, 959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 3896, 125, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2596, 0, 0, 0, 1061, 0, 0, 0, 0, 0, 0, 0, 200, 0, 48, 0, 0, 0, 3787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3830, 0, 4, 0, 0, 0, 0, 504, 14, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3773, 3926, 4108, 4873, 4044, 3657, 4035, 4352] avg=4096.0000 max=4873 max_rank=3 min=3657 min_rank=5 imbalance_ratio=1.189697 max_load_reduction=27895 max_load_reduction_pct=0.851288 moved_to_echo=16088 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1547, 0, 0, 0, 77, 0, 0, 2098, 0, 0, 0, 0, 0, 0, 0, 3976, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 3, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 304, 0, 3872, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 46, 3896, 125, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 0, 1060, 0, 0, 0, 0, 0, 0, 0, 200, 0, 29, 0, 0, 0, 3282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3358, 0, 0, 0, 0, 0, 0, 504, 15, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3722, 4117, 4105, 4248, 4078, 5110, 3511, 3877] avg=4096.0000 max=5110 max_rank=5 min=3511 min_rank=6 imbalance_ratio=1.247559 max_load_reduction=27658 max_load_reduction_pct=0.844055 moved_to_echo=16576 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 310, 0, 0, 0, 141, 0, 0, 2105, 0, 0, 0, 0, 0, 0, 0, 4060, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 286, 0, 4096, 0, 0, 0, 0, 352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3899, 130, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3493, 0, 0, 0, 1066, 0, 0, 0, 0, 0, 0, 0, 197, 0, 10, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3841, 0, 0, 0, 0, 0, 0, 509, 19, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2556, 4119, 4087, 4734, 4041, 4559, 4303, 4369] avg=4096.0000 max=4734 max_rank=3 min=2556 min_rank=0 imbalance_ratio=1.155762 max_load_reduction=28034 max_load_reduction_pct=0.855530 moved_to_echo=17272 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 2878, 25, 0, 0, 726, 10, 0, 2056, 0, 0, 0, 0, 0, 0, 0, 3735, 31, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 297, 0, 3749, 0, 0, 0, 0, 364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 128, 3896, 123, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1885, 0, 0, 0, 1038, 0, 0, 0, 0, 0, 0, 0, 200, 0, 0, 0, 0, 0, 3939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3005, 0, 13, 0, 0, 0, 0, 494, 14, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5695, 3768, 4090, 4410, 4217, 2923, 4139, 3526] avg=4096.0000 max=5695 max_rank=0 min=2923 min_rank=5 imbalance_ratio=1.390381 max_load_reduction=27073 max_load_reduction_pct=0.826202 moved_to_echo=14805 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3355, 0, 0, 0, 322, 0, 1, 2100, 0, 0, 0, 0, 0, 0, 0, 3879, 120, 0, 4, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 0, 0, 6, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 304, 0, 3977, 0, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 9, 3894, 125, 9, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1387, 0, 0, 1, 1061, 0, 0, 0, 0, 0, 0, 0, 200, 0, 106, 0, 0, 0, 3833, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3366, 0, 0, 0, 1, 0, 2, 505, 14, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5778, 4006, 4087, 4379, 4039, 2451, 4140, 3888] avg=4096.0000 max=5778 max_rank=0 min=2451 min_rank=5 imbalance_ratio=1.410645 max_load_reduction=26990 max_load_reduction_pct=0.823669 moved_to_echo=14518 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 13568, 128, 0, 0, 2432, 128, 128, 16768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 24320, 0, 0, 128, 8576, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 28416, 128, 128, 256, 128, 128, 128, 4096, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[2432, 0, 28288, 0, 0, 128, 128, 2048, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 0, 0, 0, 512, 31232, 1024, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[31616, 1280, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 128, 128, 128, 128, 32640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1664, 0, 384, 0, 128, 0, 30976, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 4096, 3557, 0, 0, 4096, 0, 3070, 0, 6, 0, 0, 130, 0, 0, 4096, 1570, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 2415, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 436, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[16, 0, 0, 0, 3827, 0, 4078, 1593, 0, 0, 4096, 0, 2203, 0, 6, 0, 0, 2610, 0, 2, 3979, 24, 0, 0, 163, 0, 0, 0, 3, 0, 4, 0, 4039, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 532, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[4, 0, 0, 0, 4073, 0, 4096, 4074, 0, 0, 4096, 0, 614, 0, 7, 0, 0, 266, 0, 0, 4080, 893, 0, 0, 0, 0, 0, 0, 2, 0, 194, 0, 3682, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 4093, 0, 4089, 3786, 0, 0, 4096, 0, 988, 0, 2, 0, 0, 92, 0, 0, 4082, 1119, 0, 0, 16, 0, 25, 0, 0, 0, 15, 0, 2367, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3765, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 9, 0, 0, 0, 0, 0, 0, 148, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 32, 3951, 0, 4034, 3220, 0, 0, 4044, 11, 2802, 16, 0, 0, 0, 470, 9, 0, 4082, 1219, 0, 5, 76, 0, 234, 0, 258, 15, 1, 12, 159, 3426, 7, 0, 5, 0, 0, 115, 0, 0, 0, 4075, 0, 0, 20, 328, 0, 12, 25, 0, 19, 0, 4, 78, 1, 2, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 4096, 3101, 0, 0, 4096, 0, 351, 0, 0, 0, 0, 122, 0, 0, 4096, 2249, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1463, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3867, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1165, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 4095, 3498, 0, 0, 4096, 0, 720, 0, 0, 0, 0, 157, 0, 0, 4080, 3215, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2384, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4089, 0, 4088, 3872, 0, 1, 4094, 0, 1110, 1, 0, 0, 3, 20, 1, 0, 4094, 508, 0, 0, 19, 0, 1, 0, 0, 1, 565, 0, 1940, 4091, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4073, 0, 0, 2, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 191, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[16, 0, 0, 0, 713, 0, 0, 1593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2613, 0, 2203, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2610, 0, 2, 2208, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 163, 0, 0, 0, 3, 0, 4, 0, 3114, 612, 8, 0, 0, 0, 0, 0, 4039, 1816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1494, 0, 0, 0, 0, 1161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 0, 1771, 1668, 322, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2322, 4822, 4844, 3904, 5855, 2655, 4073, 4293] avg=4096.0000 max=5855 max_rank=4 min=2322 min_rank=0 imbalance_ratio=1.429443 max_load_reduction=26913 max_load_reduction_pct=0.821320 moved_to_echo=16876 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 762, 0, 0, 3786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2613, 0, 988, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 2266, 1119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 25, 0, 0, 0, 15, 0, 3331, 608, 7, 0, 0, 0, 0, 0, 2367, 1806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3765, 0, 0, 0, 0, 1161, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 9, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 0, 1816, 1658, 322, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4550, 3603, 3477, 4002, 4173, 4926, 4093, 3944] avg=4096.0000 max=4926 max_rank=5 min=3477 min_rank=2 imbalance_ratio=1.202637 max_load_reduction=27842 max_load_reduction_pct=0.849670 moved_to_echo=17136 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 32, 736, 0, 0, 3220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2580, 11, 2802, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 9, 0, 2266, 1219, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 234, 0, 258, 15, 1, 12, 3215, 512, 7, 0, 0, 0, 0, 0, 159, 1519, 7, 0, 5, 0, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 20, 328, 1146, 0, 0, 0, 0, 0, 0, 0, 0, 12, 25, 0, 19, 0, 4, 78, 4027, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 1, 0, 0, 0, 0, 1816, 1395, 318, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3988, 5409, 3969, 4330, 1805, 5569, 4165, 3533] avg=4096.0000 max=5569 max_rank=5 min=1805 min_rank=4 imbalance_ratio=1.359619 max_load_reduction=27199 max_load_reduction_pct=0.830048 moved_to_echo=15072 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 763, 0, 0, 3498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2613, 0, 720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 0, 0, 2265, 3215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3333, 612, 7, 0, 0, 0, 0, 0, 2384, 1816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2302, 0, 0, 0, 0, 1161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 1815, 1668, 322, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4261, 3333, 5637, 3953, 4200, 3463, 4088, 3833] avg=4096.0000 max=5637 max_rank=2 min=3333 min_rank=1 imbalance_ratio=1.376221 max_load_reduction=27131 max_load_reduction_pct=0.827972 moved_to_echo=15584 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 0, 0, 758, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2613, 0, 614, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, 0, 0, 2265, 893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 194, 0, 3315, 611, 8, 0, 0, 0, 0, 0, 3682, 1815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2455, 0, 0, 0, 0, 1161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 0, 1815, 1667, 322, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4836, 3234, 3424, 4130, 5497, 3616, 4088, 3943] avg=4096.0000 max=5497 max_rank=4 min=3234 min_rank=1 imbalance_ratio=1.342041 max_load_reduction=27271 max_load_reduction_pct=0.832245 moved_to_echo=17144 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 757, 0, 0, 3101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2604, 0, 351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 0, 0, 2269, 2249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3337, 612, 8, 0, 0, 0, 0, 0, 1463, 1793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3867, 0, 0, 0, 1, 1166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1165, 0, 1827, 1659, 326, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3858, 2955, 4640, 3960, 3256, 5034, 4088, 4977] avg=4096.0000 max=5034 max_rank=5 min=2955 min_rank=1 imbalance_ratio=1.229004 max_load_reduction=27734 max_load_reduction_pct=0.846375 moved_to_echo=17355 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 763, 0, 0, 3557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2613, 0, 3070, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 0, 2273, 1570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 3333, 610, 7, 0, 0, 0, 0, 0, 2415, 1812, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1108, 0, 0, 0, 0, 1161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 436, 0, 1823, 1663, 322, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4320, 5689, 3973, 3957, 4227, 2269, 4089, 4244] avg=4096.0000 max=5689 max_rank=1 min=2269 min_rank=5 imbalance_ratio=1.388916 max_load_reduction=27079 max_load_reduction_pct=0.826385 moved_to_echo=14829 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 761, 0, 0, 3872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2611, 0, 1110, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 20, 1, 0, 2272, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 1, 0, 0, 1, 565, 0, 3328, 611, 7, 0, 0, 0, 0, 0, 1940, 1814, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 2, 0, 1161, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 191, 0, 1822, 1666, 322, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4633, 3723, 2804, 4532, 3755, 5236, 4084, 4001] avg=4096.0000 max=5236 max_rank=5 min=2804 min_rank=2 imbalance_ratio=1.278320 max_load_reduction=27532 max_load_reduction_pct=0.840210 moved_to_echo=17076 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 0, 0, 2688, 0, 14592, 13056, 2688, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 20864, 128, 11904, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 3968, 128, 128, 18176, 10880, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 23168, 0, 0, 128, 384, 9344, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[18560, 14208, 128, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 128, 6016, 0, 0, 26752, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[384, 0, 384, 0, 384, 128, 896, 128, 26368, 4864, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 128, 128, 128, 128, 32640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[32, 934, 0, 795, 0, 0, 0, 0, 0, 117, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3983, 3972, 0, 0, 0, 0, 590, 0, 0, 0, 0, 1, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3826, 0, 2373, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 4060, 6, 0, 3636, 0, 149, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 452, 11, 2747, 0, 10, 0, 0, 18, 33, 6, 0, 10, 0, 0, 16, 0, 4053, 0, 3430, 4078, 0, 314, 0, 13, 322, 0, 20, 0, 0, 0, 0, 2, 1, 0, 0, 0, 1, 0, 0, 28, 0, 0, 3740, 0, 3784, 32, 26, 4025, 0, 0, 0, 0, 0, 0, 5, 0, 4072, 0, 0, 1102, 0, 409, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[7, 1583, 0, 960, 0, 0, 0, 0, 0, 144, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4082, 3945, 0, 0, 0, 0, 7, 0, 0, 0, 17, 0, 0, 0, 1436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 2906, 0, 0, 4084, 0, 25, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 1091, 0, 238, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[312, 1803, 0, 0, 0, 0, 0, 0, 0, 343, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3220, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2975, 0, 748, 0, 0, 4096, 0, 10, 0, 0, 7, 0, 0, 0, 3999, 0, 0, 4080, 0, 2979, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[4, 4044, 0, 1672, 0, 0, 0, 0, 0, 248, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4065, 3827, 0, 0, 0, 0, 364, 0, 0, 0, 0, 0, 0, 0, 166, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4035, 0, 579, 0, 0, 4096, 0, 7, 0, 1, 2, 0, 0, 0, 4062, 0, 0, 1496, 0, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[12, 313, 0, 244, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 3952, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 3406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3994, 0, 261, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 3929, 0, 227, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 1795, 0, 315, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4057, 4084, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 1934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 1032, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 2696, 0, 471, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 2195, 1, 467, 0, 0, 0, 0, 0, 94, 0, 0, 0, 0, 1, 1, 0, 4085, 0, 4043, 4092, 0, 0, 0, 0, 2132, 0, 1, 0, 0, 0, 0, 0, 2366, 0, 0, 0, 0, 0, 0, 1, 0, 0, 680, 0, 2461, 1, 0, 4089, 4, 0, 0, 1, 0, 0, 1, 0, 4095, 0, 2, 1067, 0, 887, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[32, 934, 0, 795, 0, 0, 0, 0, 1234, 401, 0, 0, 0, 0, 0, 0, 0, 117, 0, 0, 0, 0, 0, 0, 3971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 3972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 590, 0, 0, 0, 0, 1, 0, 3528, 125, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 2622, 293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2589, 0, 2373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 6, 0, 3636, 0, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3396, 4088, 4134, 4244, 3015, 4962, 4101, 4828] avg=4096.0000 max=4962 max_rank=5 min=3015 min_rank=4 imbalance_ratio=1.211426 max_load_reduction=27806 max_load_reduction_pct=0.848572 moved_to_echo=16906 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[312, 1803, 0, 0, 0, 0, 0, 0, 959, 395, 0, 0, 0, 0, 0, 0, 0, 343, 0, 0, 0, 0, 0, 0, 3972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 166, 3220, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3628, 124, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 2583, 302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2014, 0, 748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 7, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 0, 0, 4080, 0, 2979, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3469, 4315, 3387, 3752, 2888, 2762, 4115, 8080] avg=4096.0000 max=8080 max_rank=7 min=2762 min_rank=5 imbalance_ratio=1.972656 max_load_reduction=24688 max_load_reduction_pct=0.753418 moved_to_echo=17845 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 452, 11, 2747, 0, 10, 0, 0, 1206, 403, 0, 0, 0, 0, 0, 0, 18, 33, 6, 0, 10, 0, 0, 16, 3929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 4078, 0, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 322, 0, 20, 0, 0, 0, 0, 3038, 124, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 1, 0, 0, 2630, 252, 0, 0, 0, 0, 0, 0, 28, 0, 0, 2531, 0, 3784, 32, 26, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 0, 0, 0, 0, 0, 0, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 0, 0, 1102, 0, 409, 8, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4829, 4012, 4532, 3517, 2886, 6401, 4033, 2558] avg=4096.0000 max=6401 max_rank=5 min=2558 min_rank=7 imbalance_ratio=1.562744 max_load_reduction=26367 max_load_reduction_pct=0.804657 moved_to_echo=15878 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4, 4044, 0, 1672, 0, 0, 0, 0, 1301, 402, 0, 0, 0, 0, 0, 0, 0, 248, 0, 0, 0, 0, 0, 0, 3971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 3827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 364, 0, 0, 0, 0, 0, 0, 3601, 125, 0, 0, 0, 0, 0, 0, 0, 166, 0, 0, 0, 1, 0, 0, 2624, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2731, 0, 579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 0, 1, 2, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 0, 0, 1496, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7423, 4219, 3992, 4090, 3090, 3310, 4109, 2535] avg=4096.0000 max=7423 max_rank=0 min=2535 min_rank=7 imbalance_ratio=1.812256 max_load_reduction=25345 max_load_reduction_pct=0.773468 moved_to_echo=13044 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[12, 313, 0, 244, 0, 0, 0, 0, 1288, 405, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 3971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 3952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 3628, 125, 0, 0, 0, 0, 0, 0, 0, 3406, 0, 0, 0, 0, 0, 0, 2645, 302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2703, 0, 261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 0, 0, 3929, 0, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2262, 3989, 4117, 3781, 6353, 2964, 4100, 5202] avg=4096.0000 max=6353 max_rank=4 min=2262 min_rank=0 imbalance_ratio=1.551025 max_load_reduction=26415 max_load_reduction_pct=0.806122 moved_to_echo=18619 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[7, 1583, 0, 960, 0, 0, 0, 0, 1317, 402, 0, 0, 0, 0, 0, 0, 0, 144, 0, 0, 0, 0, 0, 0, 3971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 3945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 17, 0, 0, 3616, 125, 0, 0, 0, 0, 0, 0, 0, 1436, 0, 0, 0, 0, 0, 0, 2625, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2763, 0, 2906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 25, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 0, 0, 1091, 0, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4269, 4115, 4110, 3765, 4362, 5669, 4112, 2366] avg=4096.0000 max=5669 max_rank=5 min=2366 min_rank=7 imbalance_ratio=1.384033 max_load_reduction=27099 max_load_reduction_pct=0.826996 moved_to_echo=16509 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1795, 0, 315, 0, 0, 0, 0, 1317, 406, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 3599, 124, 0, 0, 0, 0, 0, 0, 0, 1934, 0, 0, 0, 0, 0, 0, 2629, 302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2744, 0, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 0, 0, 2696, 0, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3833, 3974, 4240, 3780, 4865, 3776, 4103, 4197] avg=4096.0000 max=4865 max_rank=4 min=3776 min_rank=5 imbalance_ratio=1.187744 max_load_reduction=27903 max_load_reduction_pct=0.851532 moved_to_echo=16941 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 2195, 1, 467, 0, 0, 0, 0, 219, 405, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 0, 0, 1, 1, 3960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2132, 0, 1, 0, 0, 0, 0, 3581, 125, 0, 0, 0, 0, 0, 0, 0, 2366, 0, 0, 0, 0, 0, 0, 2645, 298, 0, 0, 0, 0, 0, 0, 1, 0, 0, 461, 0, 2461, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 0, 2, 1067, 0, 887, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3287, 4056, 4256, 5839, 5309, 2924, 4095, 3002] avg=4096.0000 max=5839 max_rank=3 min=2924 min_rank=5 imbalance_ratio=1.425537 max_load_reduction=26929 max_load_reduction_pct=0.821808 moved_to_echo=15330 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[384, 13184, 128, 7296, 0, 128, 0, 0, 8960, 3328, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 8320, 128, 128, 19200, 0, 5376, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 0, 18560, 0, 14208, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 9472, 0, 0, 0, 128, 0, 0, 21120, 2432, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 3584, 0, 128, 0, 128, 128, 0, 28288, 1024, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 1408, 31232, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 1024, 128, 0, 128, 0, 128, 128, 31744, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 0, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 225, 0, 4096, 0, 0, 4025, 3862, 1, 1, 0, 0, 0, 3975, 0, 0, 0, 19, 0, 0, 0, 0, 0, 792, 0, 0, 19, 0, 2270, 4096, 1225, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[14, 7, 0, 119, 0, 28, 4079, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 10, 3, 1050, 0, 4068, 136, 7, 4007, 3775, 0, 0, 21, 19, 55, 2513, 0, 19, 0, 629, 22, 17, 0, 0, 0, 2039, 0, 0, 109, 0, 2150, 4070, 516, 0, 0, 0, 0, 3241, 0, 0, 0, 0, 0, 0, 0, 21] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 0, 4096, 0, 0, 3131, 3617, 4, 0, 0, 0, 0, 2968, 0, 0, 0, 6, 0, 0, 2, 52, 0, 4022, 0, 0, 0, 0, 225, 4094, 2565, 0, 0, 0, 0, 3729, 0, 0, 0, 0, 0, 0, 0, 17] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 269, 0, 4096, 0, 0, 4087, 4096, 0, 0, 0, 0, 0, 3707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 43, 0, 2017, 4096, 2408, 0, 0, 0, 0, 3988, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3823, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 861, 0, 4096, 0, 0, 3624, 4045, 1, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 182, 28, 0, 696, 0, 0, 234, 0, 4, 4096, 3642, 0, 0, 0, 0, 3211, 0, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 4096, 0, 0, 4095, 4095, 0, 0, 0, 0, 0, 2582, 0, 0, 0, 50, 0, 0, 1, 0, 0, 1305, 0, 0, 3, 0, 1421, 4096, 2839, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2031, 0, 4083, 0, 0, 3949, 4096, 5, 0, 0, 0, 0, 3078, 0, 0, 0, 1, 0, 0, 43, 4, 0, 1160, 0, 0, 32, 0, 990, 4096, 1114, 0, 0, 0, 0, 3988, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 4057, 19, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 228, 0, 4095, 0, 1, 4017, 4094, 0, 0, 1, 1, 1, 1670, 0, 0, 0, 22, 0, 1, 3, 4, 0, 1697, 0, 0, 144, 0, 889, 4090, 3734, 2, 0, 0, 0, 3951, 0, 0, 0, 0, 0, 0, 0, 42] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4072, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 225, 0, 3491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 201, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3975, 0, 0, 0, 19, 0, 0, 574, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 792, 0, 0, 19, 0, 2714, 0, 0, 0, 0, 0, 0, 0, 2270, 0, 1225, 0, 0, 0, 0, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3661, 357, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4096, 3716, 4228, 4943, 3525, 4139, 4018] avg=4096.0000 max=4943 max_rank=4 min=3525 min_rank=5 imbalance_ratio=1.206787 max_load_reduction=27825 max_load_reduction_pct=0.849152 moved_to_echo=16625 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[14, 7, 0, 119, 0, 28, 4079, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3, 1050, 0, 3468, 0, 0, 0, 0, 0, 0, 0, 0, 136, 7, 4007, 196, 0, 0, 21, 19, 0, 0, 0, 0, 0, 0, 0, 0, 55, 2513, 0, 19, 0, 629, 22, 17, 570, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2039, 0, 0, 109, 0, 2150, 0, 0, 0, 0, 0, 0, 0, 2150, 0, 516, 0, 0, 0, 0, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 3579, 283, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4277, 4094, 4531, 4386, 4122, 4298, 3177, 3883] avg=4096.0000 max=4531 max_rank=2 min=3177 min_rank=6 imbalance_ratio=1.106201 max_load_reduction=28237 max_load_reduction_pct=0.861725 moved_to_echo=15480 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4062, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 0, 3491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3131, 188, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2968, 0, 0, 0, 6, 0, 0, 574, 342, 0, 0, 0, 0, 0, 0, 2, 52, 0, 4022, 0, 0, 0, 0, 2474, 0, 0, 0, 0, 0, 0, 0, 225, 0, 2565, 0, 0, 0, 0, 587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 3429, 326, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4094, 3669, 3323, 3890, 6550, 3377, 3772] avg=4096.0000 max=6550 max_rank=5 min=3323 min_rank=3 imbalance_ratio=1.599121 max_load_reduction=26218 max_load_reduction_pct=0.800110 moved_to_echo=17589 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3903, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 269, 0, 3491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3707, 0, 0, 0, 0, 0, 0, 574, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 43, 0, 2646, 0, 0, 0, 0, 0, 0, 0, 2017, 0, 2408, 0, 0, 0, 0, 628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3883, 348, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3934, 4097, 3760, 4299, 4647, 2747, 5053, 4231] avg=4096.0000 max=5053 max_rank=6 min=2747 min_rank=5 imbalance_ratio=1.233643 max_load_reduction=27715 max_load_reduction_pct=0.845795 moved_to_echo=16678 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 0, 4072, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 3483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2582, 0, 0, 0, 50, 0, 0, 580, 378, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1305, 0, 0, 3, 0, 2715, 0, 0, 0, 0, 0, 0, 0, 1421, 0, 2839, 0, 0, 0, 0, 629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3886, 362, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4100, 3508, 4300, 3590, 4024, 4889, 4249] avg=4096.0000 max=4889 max_rank=6 min=3508 min_rank=2 imbalance_ratio=1.193604 max_load_reduction=27879 max_load_reduction_pct=0.850800 moved_to_echo=16752 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2031, 0, 3479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3949, 212, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3078, 0, 0, 0, 1, 0, 0, 573, 366, 0, 0, 0, 0, 0, 0, 43, 4, 0, 1160, 0, 0, 32, 0, 2646, 0, 0, 0, 0, 0, 0, 0, 990, 0, 1114, 0, 0, 0, 0, 628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3883, 348, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4127, 4097, 5512, 4166, 4018, 3885, 2732, 4231] avg=4096.0000 max=5512 max_rank=2 min=2732 min_rank=6 imbalance_ratio=1.345703 max_load_reduction=27256 max_load_reduction_pct=0.831787 moved_to_echo=14866 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 4057, 19, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 4090, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 228, 0, 3490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4017, 212, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1670, 0, 0, 0, 22, 0, 1, 574, 362, 0, 0, 0, 0, 0, 0, 3, 4, 0, 1697, 0, 0, 144, 0, 2621, 0, 0, 0, 0, 0, 0, 0, 889, 0, 3734, 2, 0, 0, 0, 623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 3881, 345, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4093, 3720, 4232, 2630, 4469, 5248, 4268] avg=4096.0000 max=5248 max_rank=6 min=2630 min_rank=4 imbalance_ratio=1.281250 max_load_reduction=27520 max_load_reduction_pct=0.839844 moved_to_echo=16615 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3823, 164, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 861, 0, 3491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3624, 209, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 574, 294, 0, 0, 0, 0, 0, 0, 182, 28, 0, 696, 0, 0, 234, 0, 2130, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3642, 0, 0, 0, 0, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3835, 280, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4018, 4097, 4352, 3834, 4928, 3270, 4153, 4116] avg=4096.0000 max=4928 max_rank=4 min=3270 min_rank=5 imbalance_ratio=1.203125 max_load_reduction=27840 max_load_reduction_pct=0.849609 moved_to_echo=16467 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[256, 128, 0, 11776, 0, 0, 640, 0, 20096, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[9984, 0, 18048, 128, 0, 0, 0, 4864, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 24576, 0, 128, 0, 768, 128, 128, 4608, 2816, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 128, 4992, 0, 27904, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 128, 30080, 2688, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[256, 128, 30976, 1664, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 0, 128, 32256, 256, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 588, 0, 0, 28, 2385, 0, 10, 3640, 0, 153, 0, 0, 0, 4071, 0, 2, 177, 16, 0, 0, 0, 0, 0, 0, 0, 27, 3, 4078, 4072, 0, 0, 7, 3440, 1425, 0, 18, 0, 0, 7, 313, 0, 4071, 192, 21, 0, 37, 0, 0, 0, 0, 3917, 3, 25, 0, 0, 16, 0, 0, 22, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 10, 0, 0, 3235, 6, 0, 4019, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 4094, 4089, 0, 0, 0, 3892, 0, 0, 0, 0, 0, 0, 1136, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 47, 0, 0, 3985, 31, 0, 3853, 0, 0, 0, 0, 0, 4065, 0, 2, 0, 0, 0, 0, 4, 0, 19, 0, 0, 0, 0, 4034, 4073, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 374, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 17, 0, 0, 3231, 5, 0, 4095, 0, 0, 0, 0, 0, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4087, 0, 0, 0, 4077, 0, 0, 0, 7, 0, 0, 880, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 31, 0, 0, 3978, 26, 0, 4037, 0, 18, 0, 0, 0, 4084, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 130, 0, 4096, 45, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 66, 0, 0, 3745, 0, 1, 4079, 0, 0, 0, 0, 0, 4095, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 4091, 0, 0, 1, 4044, 3, 0, 1, 3, 0, 1, 365, 0, 4094, 8, 0, 0, 0, 1, 0, 0, 0, 4043, 0, 1, 0, 1, 0, 1, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 3959, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4077, 4096, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 159, 0, 4096, 12, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4, 0, 0, 4095, 27, 0, 4034, 0, 0, 0, 0, 0, 4070, 0, 45, 3, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 2065, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2041, 0, 4093, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 10, 0, 0, 3529, 500, 0, 0, 0, 0, 0, 0, 3235, 6, 0, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 4094, 208, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3831, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1136, 0, 3388, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 25, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4039, 3731, 4072, 4368, 3831, 4530, 4108, 4089] avg=4096.0000 max=4530 max_rank=5 min=3731 min_rank=1 imbalance_ratio=1.105957 max_load_reduction=28238 max_load_reduction_pct=0.861755 moved_to_echo=16558 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 588, 0, 0, 28, 3196, 497, 0, 0, 0, 0, 0, 0, 2385, 0, 10, 444, 0, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 0, 2, 177, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 3, 4078, 207, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3386, 1425, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 313, 0, 3367, 192, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 3917, 3, 25, 7, 0, 0, 0, 0, 0, 0, 25, 0, 0, 16, 0, 0, 22, 4, 4072, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4309, 2992, 4241, 4362, 4836, 3900, 3989, 4139] avg=4096.0000 max=4836 max_rank=4 min=2992 min_rank=1 imbalance_ratio=1.180664 max_load_reduction=27932 max_load_reduction_pct=0.852417 moved_to_echo=16864 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 17, 0, 0, 3596, 500, 0, 0, 0, 0, 0, 0, 3231, 5, 0, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 208, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 880, 0, 3385, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 25, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 3735, 4070, 4347, 4013, 4273, 4130, 4087] avg=4096.0000 max=4347 max_rank=3 min=3735 min_rank=1 imbalance_ratio=1.061279 max_load_reduction=28421 max_load_reduction_pct=0.867340 moved_to_echo=16503 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 31, 0, 0, 3545, 500, 0, 0, 0, 0, 0, 0, 3978, 26, 0, 492, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 208, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 3388, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 25, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4514, 4081, 4331, 4031, 3563, 4076, 4096] avg=4096.0000 max=4514 max_rank=1 min=3563 min_rank=5 imbalance_ratio=1.102051 max_load_reduction=28254 max_load_reduction_pct=0.862244 moved_to_echo=15766 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 47, 0, 0, 3383, 500, 0, 0, 0, 0, 0, 0, 3985, 31, 0, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4040, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 19, 0, 0, 0, 0, 4034, 208, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 374, 0, 3388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 25, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3930, 4486, 4042, 4321, 4030, 3762, 4124, 4073] avg=4096.0000 max=4486 max_rank=1 min=3762 min_rank=5 imbalance_ratio=1.095215 max_load_reduction=28282 max_load_reduction_pct=0.863098 moved_to_echo=15989 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 3601, 504, 1, 0, 0, 0, 0, 0, 3959, 0, 0, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4077, 212, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 159, 0, 3380, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 26, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4453, 4070, 4350, 4011, 3551, 4132, 4095] avg=4096.0000 max=4453 max_rank=1 min=3551 min_rank=5 imbalance_ratio=1.087158 max_load_reduction=28315 max_load_reduction_pct=0.864105 moved_to_echo=15789 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 66, 0, 0, 3582, 500, 0, 0, 0, 0, 0, 0, 3745, 0, 1, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 208, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3980, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 365, 0, 3386, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4043, 0, 25, 9, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4148, 4243, 4098, 4359, 3985, 3763, 4078, 4094] avg=4096.0000 max=4359 max_rank=3 min=3763 min_rank=5 imbalance_ratio=1.064209 max_load_reduction=28409 max_load_reduction_pct=0.866974 moved_to_echo=15920 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4, 0, 0, 3542, 500, 1, 0, 0, 0, 0, 0, 4095, 27, 0, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 45, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2065, 208, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2041, 0, 3385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 25, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4047, 4614, 4094, 2330, 4031, 5426, 4131, 4095] avg=4096.0000 max=5426 max_rank=5 min=2330 min_rank=3 imbalance_ratio=1.324707 max_load_reduction=27342 max_load_reduction_pct=0.834412 moved_to_echo=17683 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 640, 256, 0, 128, 28032, 4096, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[28672, 128, 128, 3968, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 128, 5504, 0, 27136, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 0, 0, 0, 32512, 128, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 31360, 1536, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 0, 128, 0, 0, 128, 128, 30720, 1792, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 32512, 0, 128, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 3051, 3788, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 7, 0, 0, 4046, 4082, 0, 37, 0, 0, 0, 0, 1079, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 4029, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4095, 3951, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 1, 0, 0, 184, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 4066, 4082, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 4096, 2572, 0, 10, 0, 0, 0, 0, 8, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 3386, 4072, 0, 0, 0, 0, 0, 4058, 0, 11, 28, 0, 13, 4009, 0, 0, 0, 0, 0, 29, 9, 1918, 0, 0, 24, 21, 0, 0, 1, 0, 0, 0, 0, 0, 4052, 31, 4, 39, 0, 0, 0, 4065, 2605, 22, 0, 0, 23, 1, 0, 199, 0, 21, 0, 32, 0, 0, 52, 0, 0, 0, 0, 4043, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 4092, 4038, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 3574, 0, 23, 0, 0, 0, 0, 39, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 3745, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 4050, 3992, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 27, 0, 0, 4088, 3608, 0, 0, 0, 0, 0, 0, 83, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[2, 4067, 4091, 0, 0, 0, 1, 2, 4093, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 50, 0, 2, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 4094, 0, 0, 1, 0, 0, 0, 4095, 3929, 1, 10, 0, 0, 0, 0, 135, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 4095, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4066, 236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1514, 0, 3460, 34, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 574, 2572, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 22, 0, 0, 0, 0, 3841, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4302, 4101, 5016, 4096, 4090, 3156, 3933, 4074] avg=4096.0000 max=5016 max_rank=2 min=3156 min_rank=5 imbalance_ratio=1.224609 max_load_reduction=27752 max_load_reduction_pct=0.846924 moved_to_echo=15253 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3051, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 3417, 34, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 568, 4082, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1079, 268, 0, 0, 0, 0, 3564, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3271, 4101, 3463, 4083, 4095, 4687, 4972, 4096] avg=4096.0000 max=4972 max_rank=6 min=3271 min_rank=0 imbalance_ratio=1.213867 max_load_reduction=27796 max_load_reduction_pct=0.848267 moved_to_echo=17850 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 233, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3459, 34, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 574, 3951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 3791, 62, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 184, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4331, 4101, 3504, 4096, 4086, 4525, 3878, 4247] avg=4096.0000 max=4525 max_rank=5 min=3504 min_rank=2 imbalance_ratio=1.104736 max_load_reduction=28243 max_load_reduction_pct=0.861908 moved_to_echo=16736 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4092, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 0, 3460, 34, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 574, 3574, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 14, 0, 0, 0, 0, 3800, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4325, 4101, 3996, 4096, 4088, 4171, 3915, 4076] avg=4096.0000 max=4325 max_rank=0 min=3915 min_rank=6 imbalance_ratio=1.055908 max_load_reduction=28443 max_load_reduction_pct=0.868011 moved_to_echo=16250 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 314, 0, 3463, 40, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 567, 3745, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3858, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4327, 4098, 3826, 4101, 4087, 4313, 3927, 4089] avg=4096.0000 max=4327 max_rank=0 min=3826 min_rank=2 imbalance_ratio=1.056396 max_load_reduction=28441 max_load_reduction_pct=0.867950 moved_to_echo=16416 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4050, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 0, 3453, 34, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 573, 3608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 49, 0, 0, 0, 0, 3756, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4282, 4101, 3982, 4095, 4115, 4181, 3950, 4062] avg=4096.0000 max=4282 max_rank=0 min=3950 min_rank=6 imbalance_ratio=1.045410 max_load_reduction=28486 max_load_reduction_pct=0.869324 moved_to_echo=16308 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2, 4067, 237, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 3459, 34, 8, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 1, 0, 0, 4088, 5, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4086, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 574, 3929, 1, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 0, 0, 0, 1, 0, 3849, 62, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4309, 4099, 3551, 4097, 4088, 4514, 4047, 4063] avg=4096.0000 max=4514 max_rank=5 min=3551 min_rank=2 imbalance_ratio=1.102051 max_load_reduction=28254 max_load_reduction_pct=0.862244 moved_to_echo=16712 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3386, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 11, 28, 0, 13, 4009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 9, 1918, 0, 3433, 34, 7, 0, 0, 0, 0, 0, 0, 24, 21, 0, 0, 1, 0, 0, 4053, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 31, 4, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 571, 2605, 22, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 199, 0, 21, 0, 32, 0, 3832, 61, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 4009, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3621, 4066, 5430, 4104, 4119, 3221, 4146, 4061] avg=4096.0000 max=5430 max_rank=2 min=3221 min_rank=5 imbalance_ratio=1.325684 max_load_reduction=27338 max_load_reduction_pct=0.834290 moved_to_echo=15547 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 128, 128, 4864, 0, 27648, 384, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 30976, 1920, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 0, 0, 384, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 4608, 28160, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 0, 128, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 0, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 1664, 384, 128, 0, 128, 0, 30336, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True + [2026-09-21 08:24:26] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 295.6 | throughput per GPU (TFLOP/s/GPU): 109.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.660037E+00 | loss scale: 1.0 | grad norm: 3.612 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[156, 163, 1008, 1, 0, 646, 1134, 349, 0, 464, 17, 3, 21, 9, 277, 191, 2, 891, 765, 1474, 68, 135, 295, 12, 69, 41, 265, 233, 1007, 846, 1145, 1433, 799, 630, 524, 1268, 206, 1460, 515, 665, 1512, 415, 1593, 73, 733, 1087, 24, 28, 885, 664, 689, 43, 754, 1070, 1864, 61, 1171, 17, 322, 5, 32, 30, 481, 28] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[96, 303, 1435, 0, 0, 465, 679, 444, 0, 470, 118, 1, 8, 8, 736, 118, 1, 769, 1092, 1715, 26, 185, 223, 0, 74, 16, 58, 364, 878, 695, 1173, 1338, 983, 364, 456, 967, 204, 1026, 644, 856, 1820, 641, 1456, 95, 807, 380, 8, 47, 902, 937, 398, 36, 501, 1569, 1626, 28, 1190, 12, 371, 3, 5, 20, 894, 34] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[101, 210, 1033, 0, 0, 601, 699, 406, 14, 393, 20, 3, 4, 39, 364, 136, 8, 1206, 976, 1914, 30, 241, 199, 12, 61, 4, 127, 421, 841, 396, 1246, 1766, 1101, 504, 382, 932, 116, 633, 666, 689, 1345, 518, 1451, 172, 812, 767, 39, 94, 963, 503, 724, 20, 905, 1723, 1470, 112, 1079, 15, 238, 7, 15, 28, 1231, 43] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[186, 286, 1033, 0, 2, 550, 943, 214, 2, 478, 25, 4, 14, 44, 438, 254, 19, 1162, 794, 1861, 21, 297, 224, 6, 66, 40, 99, 440, 1202, 708, 818, 1513, 788, 239, 428, 1495, 149, 821, 616, 744, 1273, 400, 1066, 178, 811, 864, 13, 25, 773, 853, 610, 85, 956, 1150, 1681, 83, 1508, 21, 267, 7, 29, 25, 1021, 46] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[236, 135, 1614, 0, 0, 829, 635, 351, 0, 697, 68, 0, 14, 6, 477, 111, 15, 705, 913, 1546, 48, 94, 311, 0, 68, 26, 90, 231, 768, 870, 966, 1723, 1556, 485, 451, 1042, 153, 864, 211, 607, 2106, 326, 2233, 56, 618, 577, 5, 23, 900, 541, 638, 41, 617, 1272, 1834, 11, 1104, 7, 266, 1, 4, 30, 625, 17] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[156, 240, 1316, 1, 2, 527, 705, 486, 1, 530, 31, 0, 30, 14, 492, 97, 8, 736, 705, 1451, 17, 184, 297, 0, 52, 10, 103, 412, 1033, 754, 1213, 1992, 911, 590, 718, 906, 192, 1052, 417, 787, 1156, 656, 1760, 214, 923, 633, 17, 97, 925, 542, 617, 58, 666, 1072, 1167, 23, 1398, 18, 337, 12, 9, 23, 1265, 12] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[79, 158, 1407, 3, 11, 664, 682, 644, 2, 567, 28, 5, 3, 9, 423, 120, 6, 919, 949, 1461, 26, 125, 287, 1, 83, 16, 226, 317, 760, 367, 997, 1600, 1232, 527, 636, 1140, 99, 840, 364, 548, 2093, 350, 1858, 59, 897, 1265, 9, 97, 1031, 542, 602, 72, 603, 1226, 1613, 26, 1074, 12, 269, 7, 23, 11, 661, 37] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[294, 456, 1126, 3, 8, 970, 757, 648, 1, 754, 21, 0, 11, 16, 643, 252, 6, 724, 781, 1425, 48, 210, 205, 10, 145, 15, 132, 185, 1553, 548, 1136, 947, 550, 552, 647, 934, 232, 868, 1018, 569, 1501, 335, 1644, 33, 820, 982, 46, 41, 786, 785, 803, 37, 1085, 942, 1185, 85, 1124, 50, 297, 3, 18, 48, 666, 52] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[96, 303, 1435, 0, 0, 465, 679, 444, 273, 243, 0, 0, 0, 0, 0, 0, 0, 470, 118, 1, 8, 8, 736, 118, 1626, 1140, 0, 0, 0, 0, 0, 0, 1, 769, 1092, 1715, 26, 185, 223, 0, 278, 19, 0, 0, 0, 0, 0, 0, 74, 16, 58, 364, 878, 695, 1173, 693, 0, 0, 0, 0, 0, 0, 0, 0, 721, 364, 456, 0, 204, 1026, 644, 856, 0, 0, 0, 0, 0, 0, 0, 0, 1820, 641, 208, 95, 807, 380, 8, 47, 0, 0, 0, 0, 0, 0, 0, 0, 902, 937, 398, 36, 501, 1291, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 1190, 12, 371, 3, 5, 20, 894, 34, 967, 372, 108, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3938, 4225, 4308, 3951, 4271, 4006, 4093, 3976] avg=4096.0000 max=4308 max_rank=2 min=3938 min_rank=0 imbalance_ratio=1.051758 max_load_reduction=28460 max_load_reduction_pct=0.868530 moved_to_echo=16346 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[156, 163, 1008, 1, 0, 646, 1134, 349, 293, 197, 0, 0, 0, 0, 0, 0, 0, 464, 17, 3, 21, 9, 277, 191, 1864, 1247, 0, 0, 0, 0, 0, 0, 2, 891, 765, 1474, 68, 135, 295, 12, 189, 15, 0, 0, 0, 0, 0, 0, 69, 41, 265, 233, 1007, 846, 1145, 741, 0, 0, 0, 0, 0, 0, 0, 0, 587, 630, 524, 0, 206, 1460, 515, 665, 0, 0, 0, 0, 0, 0, 0, 0, 1512, 415, 227, 73, 733, 1087, 24, 28, 0, 0, 0, 0, 0, 0, 0, 0, 885, 664, 689, 43, 754, 881, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 1171, 17, 322, 5, 32, 30, 481, 28, 1268, 399, 119, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3947, 4093, 3846, 4347, 4587, 4099, 3977, 3872] avg=4096.0000 max=4587 max_rank=4 min=3846 min_rank=2 imbalance_ratio=1.119873 max_load_reduction=28181 max_load_reduction_pct=0.860016 moved_to_echo=16535 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[101, 210, 1033, 0, 0, 601, 699, 406, 361, 272, 0, 0, 0, 0, 0, 0, 14, 393, 20, 3, 4, 39, 364, 136, 1470, 1136, 0, 0, 0, 0, 0, 0, 8, 1206, 976, 1914, 30, 241, 199, 12, 305, 21, 0, 0, 0, 0, 0, 0, 61, 4, 127, 421, 841, 396, 1246, 913, 0, 0, 0, 0, 0, 0, 0, 0, 808, 504, 382, 0, 116, 633, 666, 689, 0, 0, 0, 0, 0, 0, 0, 0, 1345, 518, 207, 172, 812, 767, 39, 94, 0, 0, 0, 0, 0, 0, 0, 0, 963, 503, 724, 20, 905, 1418, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 1079, 15, 238, 7, 15, 28, 1231, 43, 932, 492, 108, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3683, 3579, 4912, 4009, 3798, 3954, 4645, 4188] avg=4096.0000 max=4912 max_rank=2 min=3579 min_rank=1 imbalance_ratio=1.199219 max_load_reduction=27856 max_load_reduction_pct=0.850098 moved_to_echo=16585 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[186, 286, 1033, 0, 2, 550, 943, 214, 309, 195, 0, 0, 0, 0, 0, 0, 2, 478, 25, 4, 14, 44, 438, 254, 1681, 835, 0, 0, 0, 0, 0, 0, 19, 1162, 794, 1861, 21, 297, 224, 6, 203, 15, 0, 0, 0, 0, 0, 0, 66, 40, 99, 440, 1202, 708, 818, 783, 0, 0, 0, 0, 0, 0, 0, 0, 578, 239, 428, 0, 149, 821, 616, 744, 0, 0, 0, 0, 0, 0, 0, 0, 1273, 400, 152, 178, 811, 864, 13, 25, 0, 0, 0, 0, 0, 0, 0, 0, 773, 853, 610, 85, 956, 947, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 1508, 21, 267, 7, 29, 25, 1021, 46, 1495, 421, 79, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3718, 3775, 4602, 4156, 3575, 3716, 4307, 4919] avg=4096.0000 max=4919 max_rank=7 min=3575 min_rank=4 imbalance_ratio=1.200928 max_load_reduction=27849 max_load_reduction_pct=0.849884 moved_to_echo=16517 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[156, 240, 1316, 1, 2, 527, 705, 486, 407, 225, 0, 0, 0, 0, 0, 0, 1, 530, 31, 0, 30, 14, 492, 97, 1167, 1378, 0, 0, 0, 0, 0, 0, 8, 736, 705, 1451, 17, 184, 297, 0, 190, 17, 0, 0, 0, 0, 0, 0, 52, 10, 103, 412, 1033, 754, 1213, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 669, 590, 718, 0, 192, 1052, 417, 787, 0, 0, 0, 0, 0, 0, 0, 0, 1156, 656, 251, 214, 923, 633, 17, 97, 0, 0, 0, 0, 0, 0, 0, 0, 925, 542, 617, 58, 666, 882, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 1398, 18, 337, 12, 9, 23, 1265, 12, 906, 555, 131, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4065, 3740, 3605, 4607, 4425, 3947, 3713, 4666] avg=4096.0000 max=4666 max_rank=7 min=3605 min_rank=2 imbalance_ratio=1.139160 max_load_reduction=28102 max_load_reduction_pct=0.857605 moved_to_echo=16751 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[236, 135, 1614, 0, 0, 829, 635, 351, 352, 385, 0, 0, 0, 0, 0, 0, 0, 697, 68, 0, 14, 6, 477, 111, 1834, 1749, 0, 0, 0, 0, 0, 0, 15, 705, 913, 1546, 48, 94, 311, 0, 225, 30, 0, 0, 0, 0, 0, 0, 68, 26, 90, 231, 768, 870, 966, 891, 0, 0, 0, 0, 0, 0, 0, 0, 1141, 485, 451, 0, 153, 864, 211, 607, 0, 0, 0, 0, 0, 0, 0, 0, 2106, 326, 317, 56, 618, 577, 5, 23, 0, 0, 0, 0, 0, 0, 0, 0, 900, 541, 638, 41, 617, 1047, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 1104, 7, 266, 1, 4, 30, 625, 17, 1042, 480, 167, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4537, 4956, 3887, 3910, 3912, 4028, 3795, 3743] avg=4096.0000 max=4956 max_rank=1 min=3743 min_rank=7 imbalance_ratio=1.209961 max_load_reduction=27812 max_load_reduction_pct=0.848755 moved_to_echo=15478 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[294, 456, 1126, 3, 8, 970, 757, 648, 193, 136, 0, 0, 0, 0, 0, 0, 1, 754, 21, 0, 11, 16, 643, 252, 1185, 1287, 0, 0, 0, 0, 0, 0, 6, 724, 781, 1425, 48, 210, 205, 10, 166, 10, 0, 0, 0, 0, 0, 0, 145, 15, 132, 185, 1553, 548, 1136, 491, 0, 0, 0, 0, 0, 0, 0, 0, 404, 552, 647, 0, 232, 868, 1018, 569, 0, 0, 0, 0, 0, 0, 0, 0, 1501, 335, 235, 33, 820, 982, 46, 41, 0, 0, 0, 0, 0, 0, 0, 0, 786, 785, 803, 37, 1085, 776, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 1124, 50, 297, 3, 18, 48, 666, 52, 934, 263, 122, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4591, 4170, 3585, 4205, 4290, 3993, 4357, 3577] avg=4096.0000 max=4591 max_rank=0 min=3577 min_rank=7 imbalance_ratio=1.120850 max_load_reduction=28177 max_load_reduction_pct=0.859894 moved_to_echo=16217 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[79, 158, 1407, 3, 11, 664, 682, 644, 333, 308, 0, 0, 0, 0, 0, 0, 2, 567, 28, 5, 3, 9, 423, 120, 1613, 1460, 0, 0, 0, 0, 0, 0, 6, 919, 949, 1461, 26, 125, 287, 1, 221, 28, 0, 0, 0, 0, 0, 0, 83, 16, 226, 317, 760, 367, 997, 817, 0, 0, 0, 0, 0, 0, 0, 0, 896, 527, 636, 0, 99, 840, 364, 548, 0, 0, 0, 0, 0, 0, 0, 0, 2093, 350, 255, 59, 897, 1265, 9, 97, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 542, 602, 72, 603, 1005, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 1074, 12, 269, 7, 23, 11, 661, 37, 1140, 450, 143, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4289, 4230, 4023, 3583, 3910, 5025, 3881, 3827] avg=4096.0000 max=5025 max_rank=5 min=3583 min_rank=3 imbalance_ratio=1.226807 max_load_reduction=27743 max_load_reduction_pct=0.846649 moved_to_echo=16643 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[7168, 5376, 5120, 512, 6144, 8320, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[5888, 3968, 4352, 0, 1408, 7680, 4480, 5504, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[12928, 3712, 1920, 896, 6528, 6656, 256, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[1408, 2048, 9984, 128, 128, 5376, 6272, 3584, 2560, 2048, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 7168, 7040, 12928, 384, 1536, 2048, 128, 1792, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[640, 256, 1152, 2688, 8064, 5248, 8704, 6400, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[9728, 256, 2432, 128, 256, 256, 6912, 384, 8704, 3456, 1024, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 4480, 384, 128, 128, 256, 3968, 1280, 12544, 10240, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1686, 0, 24, 0, 3, 6, 0, 2037, 15, 362, 1700, 1932, 0, 0, 828, 14, 0, 0, 91, 304, 197, 58, 343, 0, 0, 2799, 0, 0, 689, 3625, 337, 0, 0, 0, 5, 0, 1094, 772, 926, 0, 3, 0, 46, 2, 0, 0, 1508, 0, 0, 33, 37, 3930, 3543, 0, 0, 0, 675, 0, 2956, 143, 28, 0, 17, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[2166, 0, 0, 0, 12, 557, 1, 1872, 137, 1324, 796, 3718, 0, 0, 1202, 0, 6, 0, 303, 118, 58, 17, 308, 5, 0, 532, 0, 0, 530, 2614, 647, 33, 0, 4, 1, 0, 1745, 1400, 718, 0, 122, 0, 713, 0, 0, 0, 1958, 0, 0, 0, 32, 1572, 3546, 2, 0, 0, 1087, 1, 93, 2296, 18, 0, 504, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1913, 5, 2, 0, 0, 102, 0, 982, 240, 841, 850, 2760, 2, 0, 2296, 0, 3, 0, 28, 1353, 305, 11, 102, 1, 3, 1658, 0, 0, 1054, 1341, 364, 121, 3, 0, 0, 0, 119, 668, 802, 0, 152, 0, 1778, 3, 0, 0, 1658, 0, 0, 1, 48, 3469, 3380, 0, 0, 0, 2191, 0, 1146, 892, 99, 0, 22, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1020, 4, 100, 0, 0, 118, 1, 573, 399, 767, 2427, 2682, 3, 2, 1878, 1, 0, 0, 303, 130, 2955, 264, 68, 0, 0, 752, 0, 0, 102, 1245, 1653, 0, 0, 0, 0, 0, 2308, 671, 340, 0, 175, 0, 417, 4, 1, 0, 710, 0, 31, 2, 392, 3782, 2302, 2, 7, 5, 2514, 1, 762, 494, 0, 3, 394, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[65, 4, 40, 0, 0, 191, 0, 632, 4, 616, 1305, 3554, 1, 0, 2130, 0, 6, 1, 21, 1465, 1551, 33, 123, 0, 0, 2167, 0, 0, 28, 3370, 439, 1, 0, 0, 10, 0, 1780, 1243, 1031, 0, 17, 0, 541, 5, 0, 17, 551, 0, 3, 0, 81, 2602, 1841, 22, 0, 0, 2504, 0, 1417, 163, 230, 0, 963, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1344, 0, 19, 0, 4, 69, 0, 2907, 16, 108, 1006, 1860, 0, 0, 3180, 0, 0, 0, 1, 476, 935, 78, 49, 0, 0, 347, 0, 4, 59, 2029, 844, 0, 0, 1, 0, 0, 574, 185, 684, 0, 134, 0, 253, 0, 0, 0, 1752, 1, 0, 0, 440, 3956, 3025, 0, 0, 0, 2082, 0, 1284, 2897, 4, 0, 161, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[2850, 0, 24, 0, 0, 353, 6, 881, 176, 289, 748, 2939, 4, 0, 1187, 11, 0, 0, 36, 3, 405, 1336, 305, 37, 1, 51, 0, 1, 206, 2005, 1703, 224, 1, 0, 0, 2, 1013, 1319, 932, 6, 1135, 0, 1130, 0, 0, 0, 460, 0, 0, 0, 145, 3449, 2289, 0, 22, 1, 326, 0, 739, 3894, 22, 0, 102, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[680, 0, 0, 0, 1, 972, 3, 1720, 755, 39, 1270, 1148, 0, 0, 2939, 0, 0, 0, 346, 139, 1450, 60, 224, 0, 0, 987, 0, 0, 987, 1987, 745, 17, 0, 0, 0, 0, 713, 524, 3820, 0, 1340, 3, 22, 0, 0, 0, 1482, 0, 4, 1, 437, 2768, 2681, 0, 0, 1, 200, 1, 120, 1910, 226, 0, 46, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1344, 0, 19, 0, 4, 69, 0, 2907, 440, 653, 0, 0, 0, 0, 0, 0, 16, 108, 1006, 82, 0, 0, 3180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 476, 935, 78, 49, 0, 1477, 0, 0, 0, 0, 0, 0, 0, 0, 347, 0, 4, 59, 1414, 844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 574, 185, 684, 0, 895, 175, 0, 0, 0, 0, 0, 0, 134, 0, 253, 0, 0, 0, 1752, 1, 1757, 301, 0, 0, 0, 0, 0, 0, 0, 0, 440, 1304, 3025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2082, 0, 1284, 2244, 4, 0, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5436, 4392, 3016, 2668, 2514, 4198, 4769, 5775] avg=4096.0000 max=5775 max_rank=7 min=2514 min_rank=4 imbalance_ratio=1.409912 max_load_reduction=26993 max_load_reduction_pct=0.823761 moved_to_echo=17256 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[65, 4, 40, 0, 0, 191, 0, 632, 731, 36, 0, 0, 0, 0, 0, 0, 4, 616, 1305, 157, 1, 0, 2130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 21, 1465, 1551, 33, 123, 0, 2822, 0, 0, 0, 0, 0, 0, 0, 0, 2167, 0, 0, 28, 2348, 439, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 1780, 1243, 1031, 0, 588, 291, 0, 0, 0, 0, 0, 0, 17, 0, 541, 5, 0, 17, 551, 0, 1155, 575, 0, 0, 0, 0, 0, 0, 3, 0, 81, 859, 1841, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2504, 0, 1417, 127, 230, 0, 963, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1699, 4213, 6022, 4983, 4943, 2861, 2806, 5241] avg=4096.0000 max=6022 max_rank=2 min=1699 min_rank=0 imbalance_ratio=1.470215 max_load_reduction=26746 max_load_reduction_pct=0.816223 moved_to_echo=15851 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1913, 5, 2, 0, 0, 102, 0, 982, 291, 201, 0, 0, 0, 0, 0, 0, 240, 841, 850, 121, 2, 0, 2296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 28, 1353, 305, 11, 102, 1, 2192, 0, 0, 0, 0, 0, 0, 0, 3, 1658, 0, 0, 1054, 934, 364, 121, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 119, 668, 802, 0, 785, 116, 0, 0, 0, 0, 0, 0, 152, 0, 1778, 3, 0, 0, 1658, 0, 1540, 447, 0, 0, 0, 0, 0, 0, 0, 1, 48, 1144, 3380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2191, 0, 1146, 691, 99, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3496, 4350, 3995, 4134, 2493, 5578, 4573, 4149] avg=4096.0000 max=5578 max_rank=5 min=2493 min_rank=4 imbalance_ratio=1.361816 max_load_reduction=27190 max_load_reduction_pct=0.829773 moved_to_echo=16793 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1020, 4, 100, 0, 0, 118, 1, 573, 270, 111, 0, 0, 0, 0, 0, 0, 399, 767, 2427, 118, 3, 2, 1878, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 303, 130, 2955, 264, 68, 0, 2130, 0, 0, 0, 0, 0, 0, 0, 0, 752, 0, 0, 102, 868, 1653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2308, 671, 340, 0, 855, 107, 0, 0, 0, 0, 0, 0, 175, 0, 417, 4, 1, 0, 710, 0, 1679, 434, 0, 0, 0, 0, 0, 0, 31, 2, 392, 1248, 2302, 2, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 2514, 1, 762, 383, 0, 3, 394, 4, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2197, 5595, 5850, 3375, 4281, 3420, 3989, 4061] avg=4096.0000 max=5850 max_rank=2 min=2197 min_rank=0 imbalance_ratio=1.428223 max_load_reduction=26918 max_load_reduction_pct=0.821472 moved_to_echo=15751 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[680, 0, 0, 0, 1, 972, 3, 1720, 431, 430, 0, 0, 0, 0, 0, 0, 755, 39, 1270, 52, 0, 0, 2939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 346, 139, 1450, 60, 224, 0, 911, 0, 0, 0, 0, 0, 0, 0, 0, 987, 0, 0, 987, 1384, 745, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 713, 524, 3820, 0, 626, 172, 0, 0, 0, 0, 0, 0, 1340, 3, 22, 0, 0, 0, 1482, 0, 1229, 185, 0, 0, 0, 0, 0, 0, 4, 1, 437, 913, 2681, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 200, 1, 120, 1480, 226, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4237, 5055, 3130, 4120, 5855, 4261, 4037, 2073] avg=4096.0000 max=5855 max_rank=4 min=2073 min_rank=7 imbalance_ratio=1.429443 max_load_reduction=26913 max_load_reduction_pct=0.821320 moved_to_echo=16226 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1686, 0, 24, 0, 3, 6, 0, 2037, 787, 32, 0, 0, 0, 0, 0, 0, 15, 362, 1700, 86, 0, 0, 828, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 304, 197, 58, 343, 0, 1534, 0, 0, 0, 0, 0, 0, 0, 0, 2799, 0, 0, 689, 2525, 337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 1094, 772, 926, 0, 889, 313, 0, 0, 0, 0, 0, 0, 3, 0, 46, 2, 0, 0, 1508, 0, 1745, 312, 0, 0, 0, 0, 0, 0, 0, 33, 37, 1296, 3543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 675, 0, 2956, 111, 28, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4575, 3005, 2527, 6350, 3999, 3616, 4909, 3787] avg=4096.0000 max=6350 max_rank=3 min=2527 min_rank=2 imbalance_ratio=1.550293 max_load_reduction=26418 max_load_reduction_pct=0.806213 moved_to_echo=16311 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2850, 0, 24, 0, 0, 353, 6, 881, 439, 882, 0, 0, 0, 0, 0, 0, 176, 289, 748, 121, 4, 0, 1187, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 3, 405, 1336, 305, 37, 2338, 0, 0, 0, 0, 0, 0, 0, 1, 51, 0, 1, 206, 1389, 1703, 224, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 1013, 1319, 932, 6, 784, 177, 0, 0, 0, 0, 0, 0, 1135, 0, 1130, 0, 0, 0, 460, 0, 1536, 480, 0, 0, 0, 0, 0, 0, 0, 0, 145, 1129, 2289, 0, 22, 1, 0, 0, 0, 0, 0, 0, 0, 0, 326, 0, 739, 3012, 22, 0, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5435, 2536, 4460, 3575, 4234, 4741, 3586, 4201] avg=4096.0000 max=5435 max_rank=0 min=2536 min_rank=1 imbalance_ratio=1.326904 max_load_reduction=27333 max_load_reduction_pct=0.834137 moved_to_echo=16762 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2166, 0, 0, 0, 12, 557, 1, 1872, 567, 518, 0, 0, 0, 0, 0, 0, 137, 1324, 796, 163, 0, 0, 1202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 303, 118, 58, 17, 308, 5, 2953, 0, 0, 0, 0, 0, 0, 0, 0, 532, 0, 0, 530, 1821, 647, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 1745, 1400, 718, 0, 355, 226, 0, 0, 0, 0, 0, 0, 122, 0, 713, 0, 0, 0, 1958, 0, 698, 602, 0, 0, 0, 0, 0, 0, 0, 0, 32, 519, 3546, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1087, 1, 93, 1778, 18, 0, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5693, 3622, 3768, 3563, 4449, 4093, 4099, 3481] avg=4096.0000 max=5693 max_rank=0 min=3481 min_rank=7 imbalance_ratio=1.389893 max_load_reduction=27075 max_load_reduction_pct=0.826263 moved_to_echo=16122 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1792, 4352, 10112, 1024, 128, 128, 15744, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[11648, 128, 8576, 9856, 640, 128, 2304, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 1152, 4096, 7936, 1920, 1536, 128, 16384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[3200, 128, 4992, 128, 128, 128, 10112, 128, 11392, 3456, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[11776, 128, 256, 0, 128, 2432, 128, 11648, 3968, 2944, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 9472, 6784, 9344, 128, 5888, 1664, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 9344, 0, 128, 3712, 12800, 6784, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 1664, 8448, 22656, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33792 slack_slots=31744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33792 slack_slots=31744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 43, 5, 1, 202, 0, 79, 0, 1380, 0, 17, 25, 0, 1630, 0, 26, 431, 0, 4039, 131, 184, 1, 0, 0, 0, 0, 0, 3067, 0, 43, 0, 0, 217, 1564, 14, 5, 43, 4038, 1802, 0, 0, 0, 7, 0, 3963, 1, 0, 0, 132, 3, 4080, 3573, 0, 1, 70, 0, 0, 380, 7, 2, 21, 91, 1448, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 563, 0, 799, 16, 669, 0, 0, 0, 0, 2561, 0, 18, 1205, 43, 3960, 39, 101, 0, 0, 0, 1, 0, 0, 1903, 336, 0, 0, 0, 8, 1140, 66, 0, 4, 2921, 2233, 0, 0, 0, 1, 0, 2329, 9, 0, 0, 29, 68, 4095, 3486, 0, 79, 350, 0, 2, 311, 7, 0, 215, 209, 2990, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 1780, 0, 855, 0, 2483, 0, 0, 21, 0, 29, 0, 20, 1595, 21, 2215, 5, 3, 0, 0, 0, 0, 0, 0, 2063, 192, 0, 0, 0, 0, 1718, 1843, 0, 127, 2622, 664, 0, 0, 0, 0, 0, 1269, 0, 0, 0, 83, 7, 3957, 1350, 0, 101, 3020, 0, 1, 1325, 19, 0, 8, 44, 3327, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 19, 0, 14, 105, 0, 116, 1, 1933, 0, 1, 6, 3, 1413, 0, 1244, 461, 5, 3953, 0, 0, 0, 0, 3, 0, 3, 0, 3076, 1, 1, 0, 0, 1, 2172, 14, 6, 0, 1336, 1162, 0, 3, 0, 0, 91, 2047, 0, 0, 0, 1580, 0, 4041, 3204, 0, 24, 1322, 0, 2, 1480, 0, 0, 0, 56, 1868, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 14, 11, 0, 200, 0, 223, 0, 2817, 0, 0, 0, 0, 452, 0, 2, 494, 13, 3433, 14, 0, 0, 0, 0, 0, 0, 0, 1724, 7, 6, 0, 0, 0, 4036, 44, 4, 21, 3267, 1429, 0, 0, 0, 0, 5, 2535, 0, 0, 0, 293, 1, 4094, 2301, 0, 37, 1108, 0, 0, 1078, 4, 0, 3, 133, 2964, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 224, 0, 25, 0, 267, 0, 3866, 5, 0, 6, 0, 676, 0, 160, 950, 185, 3953, 12, 0, 0, 0, 0, 0, 2, 0, 2413, 0, 346, 0, 0, 0, 1115, 59, 0, 18, 4077, 2384, 5, 0, 0, 0, 0, 2633, 12, 2, 0, 1600, 0, 4074, 2053, 1, 31, 532, 1, 0, 31, 0, 0, 0, 708, 341, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 20, 3, 0, 17, 2, 13, 0, 3818, 4, 1, 176, 6, 142, 0, 35, 706, 88, 3373, 15, 12, 8, 0, 0, 0, 5, 1, 2689, 0, 5, 0, 2, 48, 1393, 2, 0, 7, 1587, 3317, 0, 0, 0, 0, 1, 2496, 0, 1, 0, 230, 0, 3876, 1886, 0, 559, 1266, 12, 0, 3455, 3, 1, 1, 11, 1475, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1245, 0, 1638, 0, 657, 0, 2, 393, 0, 57, 0, 24, 1416, 2, 3711, 0, 2, 0, 0, 0, 0, 0, 0, 1395, 47, 13, 0, 0, 6, 1151, 56, 0, 54, 3879, 252, 0, 0, 0, 0, 0, 1959, 0, 1, 0, 1469, 60, 4074, 1511, 0, 540, 1137, 0, 0, 1982, 30, 0, 4, 827, 3174, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 563, 0, 799, 16, 3077, 0, 0, 0, 0, 0, 0, 0, 669, 0, 0, 0, 0, 2561, 0, 18, 501, 290, 0, 0, 0, 0, 0, 0, 1205, 43, 3406, 39, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1903, 336, 0, 0, 0, 1018, 664, 0, 0, 0, 0, 0, 0, 8, 1140, 66, 0, 4, 317, 2233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2329, 9, 0, 0, 1650, 0, 0, 0, 0, 0, 0, 0, 29, 68, 0, 3065, 0, 79, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 311, 7, 0, 215, 209, 2990, 0, 421, 53, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4457, 4039, 4794, 3922, 3768, 3989, 3591, 4208] avg=4096.0000 max=4794 max_rank=2 min=3591 min_rank=6 imbalance_ratio=1.170410 max_load_reduction=27974 max_load_reduction_pct=0.853699 moved_to_echo=15556 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 43, 5, 1, 202, 0, 79, 0, 3066, 0, 0, 0, 0, 0, 0, 0, 1380, 0, 17, 25, 0, 1630, 0, 26, 511, 401, 0, 0, 0, 0, 0, 0, 431, 0, 3474, 131, 184, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3067, 0, 43, 0, 0, 1014, 919, 0, 0, 0, 0, 0, 0, 217, 1564, 14, 5, 43, 437, 1802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 3963, 1, 0, 0, 2281, 0, 0, 0, 0, 0, 0, 0, 132, 3, 0, 3141, 0, 1, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 380, 7, 2, 21, 91, 1448, 2, 432, 54, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3396, 3990, 4221, 5043, 4082, 6252, 3347, 2437] avg=4096.0000 max=6252 max_rank=5 min=2437 min_rank=7 imbalance_ratio=1.526367 max_load_reduction=26516 max_load_reduction_pct=0.809204 moved_to_echo=16118 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 19, 0, 14, 105, 0, 116, 1, 3036, 0, 0, 0, 0, 0, 0, 0, 1933, 0, 1, 6, 3, 1413, 0, 1244, 501, 132, 0, 0, 0, 0, 0, 0, 461, 5, 3399, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3076, 1, 1, 0, 0, 1005, 304, 0, 0, 0, 0, 0, 0, 1, 2172, 14, 6, 0, 146, 1162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 91, 2047, 0, 0, 0, 754, 0, 0, 0, 0, 0, 0, 0, 1580, 0, 0, 2817, 0, 24, 1322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1480, 0, 0, 0, 56, 1868, 1, 387, 53, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3291, 5233, 3868, 4390, 3501, 2895, 5743, 3847] avg=4096.0000 max=5743 max_rank=6 min=2895 min_rank=5 imbalance_ratio=1.402100 max_load_reduction=27025 max_load_reduction_pct=0.824738 moved_to_echo=15986 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 1780, 0, 855, 0, 2973, 0, 0, 0, 0, 0, 0, 0, 2483, 0, 0, 21, 0, 29, 0, 20, 280, 260, 0, 0, 0, 0, 0, 0, 1595, 21, 1906, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2063, 192, 0, 0, 0, 984, 596, 0, 0, 0, 0, 0, 0, 0, 1718, 1843, 0, 127, 285, 664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1269, 0, 0, 0, 1481, 0, 0, 0, 0, 0, 0, 0, 83, 7, 0, 1187, 0, 101, 3020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1325, 19, 0, 8, 44, 3327, 0, 163, 29, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5609, 3093, 3530, 3835, 4637, 2750, 4398, 4916] avg=4096.0000 max=5609 max_rank=0 min=2750 min_rank=5 imbalance_ratio=1.369385 max_load_reduction=27159 max_load_reduction_pct=0.828827 moved_to_echo=16701 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 14, 11, 0, 200, 0, 223, 0, 3076, 0, 0, 0, 0, 0, 0, 0, 2817, 0, 0, 0, 0, 452, 0, 2, 435, 324, 0, 0, 0, 0, 0, 0, 494, 13, 2952, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1724, 7, 6, 0, 0, 1018, 743, 0, 0, 0, 0, 0, 0, 0, 4036, 44, 4, 21, 355, 1429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2535, 0, 0, 0, 1845, 0, 0, 0, 0, 0, 0, 0, 293, 1, 0, 2023, 0, 37, 1108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1078, 4, 0, 3, 133, 2964, 1, 278, 46, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3524, 4030, 3473, 3498, 5889, 4385, 3462, 4507] avg=4096.0000 max=5889 max_rank=4 min=3462 min_rank=6 imbalance_ratio=1.437744 max_load_reduction=26879 max_load_reduction_pct=0.820282 moved_to_echo=18243 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 224, 0, 25, 0, 267, 0, 3062, 0, 0, 0, 0, 0, 0, 0, 3866, 5, 0, 6, 0, 676, 0, 160, 501, 405, 0, 0, 0, 0, 0, 0, 950, 185, 3399, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2413, 0, 346, 0, 0, 1012, 927, 0, 0, 0, 0, 0, 0, 0, 1115, 59, 0, 18, 442, 2384, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2633, 12, 2, 0, 2303, 0, 0, 0, 0, 0, 0, 0, 1600, 0, 0, 1805, 1, 31, 532, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 708, 341, 0, 248, 53, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3579, 5619, 4546, 4700, 4023, 4950, 3970, 1381] avg=4096.0000 max=5619 max_rank=1 min=1381 min_rank=7 imbalance_ratio=1.371826 max_load_reduction=27149 max_load_reduction_pct=0.828522 moved_to_echo=14324 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 20, 3, 0, 17, 2, 13, 0, 2913, 0, 0, 0, 0, 0, 0, 0, 3818, 4, 1, 176, 6, 142, 0, 35, 431, 163, 0, 0, 0, 0, 0, 0, 706, 88, 2895, 15, 12, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 2689, 0, 5, 0, 2, 963, 365, 0, 0, 0, 0, 0, 0, 48, 1393, 2, 0, 7, 161, 3317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2496, 0, 1, 0, 898, 0, 0, 0, 0, 0, 0, 0, 230, 0, 0, 1655, 0, 559, 1266, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3455, 3, 1, 1, 11, 1475, 0, 231, 47, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2968, 4776, 3724, 4030, 4928, 3396, 3722, 5224] avg=4096.0000 max=5224 max_rank=7 min=2968 min_rank=0 imbalance_ratio=1.275391 max_load_reduction=27544 max_load_reduction_pct=0.840576 moved_to_echo=17270 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1245, 0, 1638, 0, 3061, 0, 0, 0, 0, 0, 0, 0, 657, 0, 2, 393, 0, 57, 0, 24, 470, 385, 0, 0, 0, 0, 0, 0, 1416, 2, 3192, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1395, 47, 13, 0, 0, 1013, 882, 0, 0, 0, 0, 0, 0, 6, 1151, 56, 0, 54, 421, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1959, 0, 1, 0, 2191, 0, 0, 0, 0, 0, 0, 0, 1469, 60, 0, 1329, 0, 540, 1137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1982, 30, 0, 4, 827, 3174, 0, 182, 49, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5944, 1988, 4612, 3350, 1940, 4151, 4535, 6248] avg=4096.0000 max=6248 max_rank=7 min=1940 min_rank=4 imbalance_ratio=1.525391 max_load_reduction=26520 max_load_reduction_pct=0.809326 moved_to_echo=16874 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[384, 14336, 2176, 128, 384, 2688, 13312, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 18432, 640, 512, 0, 128, 8064, 5504, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 19328, 128, 128, 0, 13440, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 10112, 128, 128, 256, 2176, 17664, 128, 2432, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[17664, 128, 128, 640, 128, 7040, 0, 1536, 3712, 2432, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[7296, 384, 24704, 256, 384, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[5504, 256, 0, 17024, 128, 1408, 8832, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 256, 128, 4224, 128, 4096, 128, 24320, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 2, 0, 0, 813, 2969, 0, 0, 2981, 0, 3299, 0, 0, 2177, 505, 7, 0, 0, 0, 0, 0, 0, 27, 0, 214, 0, 3897, 0, 0, 3370, 0, 0, 0, 0, 0, 0, 0, 1875, 896, 2, 0, 0, 0, 0, 2188, 0, 0, 4008, 0, 9, 0, 0, 0, 0, 0, 3523, 5, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 119, 0, 0, 0, 2659, 2654, 0, 0, 1893, 0, 1827, 2, 0, 2424, 354, 0, 0, 11, 0, 0, 0, 0, 39, 1, 461, 0, 4014, 0, 2, 1397, 0, 1, 0, 0, 0, 0, 0, 3497, 1863, 0, 0, 0, 16, 0, 1966, 0, 0, 4022, 0, 10, 0, 0, 0, 1, 2, 3532, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 24, 0, 448, 1686, 0, 0, 1463, 0, 255, 0, 20, 318, 811, 0, 0, 44, 0, 0, 0, 0, 1, 0, 3721, 0, 4050, 0, 0, 2014, 0, 0, 0, 0, 0, 0, 0, 4025, 2155, 0, 0, 0, 21, 0, 3962, 0, 0, 4039, 0, 189, 0, 0, 0, 0, 0, 3521, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 2, 1, 4, 20, 7, 13, 0, 1772, 723, 0, 0, 2554, 1, 427, 0, 0, 2954, 1422, 0, 0, 1, 0, 0, 0, 0, 2, 38, 3755, 0, 4087, 0, 0, 3178, 0, 25, 7, 0, 0, 11, 1, 2848, 751, 0, 1, 0, 29, 0, 2724, 0, 0, 1255, 0, 188, 0, 0, 0, 15, 1, 3951, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 0, 678, 0, 5, 0, 983, 3582, 0, 0, 13, 0, 3032, 0, 21, 2752, 2330, 0, 0, 65, 7, 0, 5, 1, 24, 8, 574, 4, 3198, 0, 7, 3892, 0, 1, 14, 6, 0, 0, 0, 1511, 38, 18, 0, 0, 0, 0, 3738, 0, 0, 3778, 10, 214, 0, 1, 0, 0, 9, 2244, 0, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 0, 12, 0, 0, 0, 27, 1046, 0, 0, 3897, 0, 1302, 0, 0, 2249, 2403, 0, 1, 0, 0, 0, 0, 0, 16, 0, 2965, 0, 4079, 0, 0, 2854, 0, 0, 0, 0, 0, 0, 0, 3225, 834, 0, 0, 0, 250, 0, 745, 0, 0, 3061, 0, 553, 0, 0, 0, 0, 0, 3240, 0, 0, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 1, 0, 14, 0, 285, 4070, 2, 0, 1660, 4, 3520, 0, 25, 2151, 2166, 0, 0, 120, 0, 0, 2, 0, 1, 4, 2999, 4, 4083, 0, 9, 1168, 0, 0, 0, 3, 0, 0, 6, 75, 8, 0, 0, 0, 138, 0, 2018, 0, 11, 3452, 0, 766, 0, 3, 1, 6, 0, 3988, 3, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[7, 4, 0, 0, 0, 486, 0, 0, 0, 2236, 268, 2, 0, 58, 1, 2665, 6, 0, 3535, 503, 3, 32, 0, 0, 0, 0, 0, 34, 38, 3159, 2, 4089, 3, 0, 1236, 1, 0, 0, 0, 1, 4, 1, 4036, 1595, 2, 0, 0, 0, 0, 1775, 0, 5, 3917, 6, 0, 0, 9, 5, 0, 0, 3009, 0, 3, 32] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 119, 0, 0, 2654, 1830, 0, 0, 0, 0, 0, 0, 0, 2659, 0, 0, 0, 1893, 0, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2424, 354, 0, 0, 11, 0, 374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 1, 461, 0, 1870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1397, 0, 1, 0, 0, 0, 1623, 314, 0, 0, 0, 0, 0, 0, 0, 0, 3497, 1863, 0, 0, 0, 16, 339, 0, 0, 0, 0, 0, 0, 0, 0, 1966, 0, 0, 1709, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3532, 0, 0, 1, 690, 103, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4603, 5563, 3165, 2371, 3337, 5715, 3685, 4329] avg=4096.0000 max=5715 max_rank=5 min=2371 min_rank=3 imbalance_ratio=1.395264 max_load_reduction=27053 max_load_reduction_pct=0.825592 moved_to_echo=17066 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 2, 0, 2969, 1777, 0, 0, 0, 0, 0, 0, 0, 813, 0, 0, 0, 2981, 0, 1823, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2177, 505, 7, 0, 0, 0, 676, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 214, 0, 1816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3370, 0, 0, 0, 0, 0, 1617, 304, 0, 0, 0, 0, 0, 0, 0, 0, 1875, 896, 2, 0, 0, 0, 613, 0, 0, 0, 0, 0, 0, 0, 0, 2188, 0, 0, 1703, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3523, 5, 0, 0, 688, 187, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4749, 5617, 3365, 2057, 5291, 3386, 3900, 4403] avg=4096.0000 max=5617 max_rank=1 min=2057 min_rank=3 imbalance_ratio=1.371338 max_load_reduction=27151 max_load_reduction_pct=0.828583 moved_to_echo=16980 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 0, 678, 0, 5, 3582, 1458, 0, 0, 0, 0, 0, 0, 0, 983, 0, 0, 0, 13, 0, 1677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 2752, 2330, 0, 0, 65, 7, 621, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 24, 8, 574, 4, 1490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3892, 0, 1, 14, 6, 0, 1525, 250, 0, 0, 0, 0, 0, 0, 0, 0, 1511, 38, 18, 0, 0, 0, 563, 0, 0, 0, 0, 0, 0, 0, 0, 3738, 0, 0, 1605, 10, 214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 9, 2244, 0, 2, 0, 648, 171, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5726, 2673, 5796, 2106, 5695, 2130, 5567, 3075] avg=4096.0000 max=5796 max_rank=2 min=2106 min_rank=3 imbalance_ratio=1.415039 max_load_reduction=26972 max_load_reduction_pct=0.823120 moved_to_echo=16467 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 1, 0, 14, 4070, 1861, 0, 0, 0, 0, 0, 0, 0, 285, 0, 2, 0, 1660, 4, 1945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 2151, 2166, 0, 0, 120, 0, 722, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 4, 2999, 4, 1903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1168, 0, 0, 0, 3, 0, 1393, 319, 0, 0, 0, 0, 0, 0, 0, 6, 75, 8, 0, 0, 0, 138, 654, 0, 0, 0, 0, 0, 0, 0, 0, 2018, 0, 11, 1467, 0, 766, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 6, 0, 3988, 3, 0, 0, 592, 199, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5948, 3896, 5184, 4913, 2892, 881, 4262, 4792] avg=4096.0000 max=5948 max_rank=0 min=881 min_rank=5 imbalance_ratio=1.452148 max_load_reduction=26820 max_load_reduction_pct=0.818481 moved_to_echo=12827 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 24, 1686, 1846, 0, 0, 0, 0, 0, 0, 0, 448, 0, 0, 0, 1463, 0, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 318, 811, 0, 0, 44, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3721, 0, 1888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2014, 0, 0, 0, 0, 0, 1630, 316, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 2155, 0, 0, 0, 21, 47, 0, 0, 0, 0, 0, 0, 0, 0, 3962, 0, 0, 1716, 0, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3521, 0, 0, 1, 693, 14, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3556, 2053, 1245, 5610, 3960, 6248, 5867, 4229] avg=4096.0000 max=6248 max_rank=5 min=1245 min_rank=2 imbalance_ratio=1.525391 max_load_reduction=26520 max_load_reduction_pct=0.809326 moved_to_echo=20304 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 0, 12, 0, 0, 1046, 1860, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 3897, 0, 720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2249, 2403, 0, 1, 0, 0, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 2965, 0, 1900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2854, 0, 0, 0, 0, 0, 1235, 319, 0, 0, 0, 0, 0, 0, 0, 0, 3225, 834, 0, 0, 0, 250, 242, 0, 0, 0, 0, 0, 0, 0, 0, 745, 0, 0, 1301, 0, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3240, 0, 0, 7, 525, 73, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2920, 4644, 4920, 4881, 4408, 4551, 2599, 3845] avg=4096.0000 max=4920 max_rank=2 min=2599 min_rank=6 imbalance_ratio=1.201172 max_load_reduction=27848 max_load_reduction_pct=0.849854 moved_to_echo=15403 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 1, 4, 20, 7, 13, 723, 1863, 0, 0, 0, 0, 0, 0, 0, 1772, 0, 0, 0, 2554, 1, 237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2954, 1422, 0, 0, 1, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 38, 3755, 0, 1905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3178, 0, 25, 7, 0, 0, 506, 319, 0, 0, 0, 0, 0, 0, 11, 1, 2848, 751, 0, 1, 0, 29, 79, 0, 0, 0, 0, 0, 0, 0, 0, 2724, 0, 0, 534, 0, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 1, 3951, 0, 0, 0, 215, 24, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2633, 4564, 4464, 5700, 4035, 3720, 3446, 4206] avg=4096.0000 max=5700 max_rank=3 min=2633 min_rank=0 imbalance_ratio=1.391602 max_load_reduction=27068 max_load_reduction_pct=0.826050 moved_to_echo=15407 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[7, 4, 0, 0, 0, 486, 0, 0, 268, 1868, 0, 0, 0, 0, 0, 0, 0, 2236, 0, 2, 0, 58, 1, 1461, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 3535, 503, 3, 32, 0, 0, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 38, 3159, 2, 1897, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1236, 1, 0, 0, 0, 1, 1585, 324, 0, 0, 0, 0, 0, 0, 4, 1, 4036, 1595, 2, 0, 0, 0, 499, 0, 0, 0, 0, 0, 0, 0, 0, 1775, 0, 5, 1656, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 5, 0, 0, 3009, 0, 3, 32, 676, 155, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2633, 3758, 4629, 5130, 3150, 6137, 3442, 3889] avg=4096.0000 max=6137 max_rank=5 min=2633 min_rank=0 imbalance_ratio=1.498291 max_load_reduction=26631 max_load_reduction_pct=0.812714 moved_to_echo=16618 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[128, 128, 19200, 128, 128, 128, 128, 128, 11136, 2560, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 21120, 8192, 128, 128, 0, 512, 3072, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 128, 1408, 128, 128, 17024, 14464, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 9344, 0, 128, 0, 14592, 128, 9088, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 18560, 10496, 128, 128, 256, 128, 3456, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 256, 128, 17920, 128, 14720, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 19200, 0, 128, 11776, 128, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 27008, 128, 128, 128, 4736, 1024, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33792 slack_slots=31744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33792 slack_slots=31744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 3579, 0, 0, 4004, 0, 0, 2, 4060, 0, 2, 0, 1, 0, 0, 0, 0, 2, 4, 0, 0, 0, 0, 105, 0, 0, 0, 0, 686, 0, 0, 120, 0, 0, 6, 0, 0, 0, 2, 0, 1, 3960, 386, 3997, 5, 3455, 599, 0, 2, 0, 0, 0, 0, 0, 38, 0, 3680, 0, 3, 0, 0, 0, 1, 4068] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[2, 2285, 4, 0, 2999, 1, 0, 108, 4093, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 19, 0, 347, 0, 0, 0, 0, 691, 14, 0, 1034, 0, 0, 0, 0, 0, 0, 16, 0, 1, 3651, 2037, 2963, 2, 3683, 369, 0, 2, 0, 31, 0, 331, 0, 330, 0, 3674, 0, 7, 0, 0, 0, 1, 4069] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 2408, 0, 0, 1887, 8, 0, 23, 4091, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 0, 0, 0, 0, 2, 0, 0, 3210, 0, 0, 0, 0, 0, 0, 44, 0, 0, 3241, 1172, 4035, 0, 3250, 1065, 0, 0, 0, 0, 0, 106, 0, 3, 0, 4063, 0, 0, 0, 0, 0, 0, 3959] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 2190, 0, 0, 1509, 0, 0, 19, 4096, 0, 0, 0, 133, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 540, 0, 0, 0, 1, 17, 0, 0, 485, 0, 0, 0, 0, 0, 4, 78, 0, 0, 3853, 3494, 3980, 1, 2380, 2316, 0, 0, 0, 0, 0, 28, 8, 32, 0, 3553, 0, 0, 0, 0, 0, 0, 4046] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 2828, 0, 21, 2433, 0, 0, 168, 4056, 0, 0, 0, 317, 1, 5, 0, 35, 0, 0, 0, 8, 22, 0, 243, 2, 6, 2, 0, 215, 0, 0, 2297, 0, 1, 0, 0, 0, 0, 468, 0, 0, 3558, 1086, 4047, 0, 3622, 450, 0, 0, 1, 0, 0, 86, 8, 15, 12, 2548, 1, 108, 0, 7, 0, 0, 4091] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 3197, 0, 0, 1573, 0, 0, 2, 4082, 40, 0, 0, 89, 2, 10, 0, 0, 0, 0, 3, 8, 0, 2, 23, 15, 0, 0, 0, 4, 0, 0, 1577, 0, 3, 0, 3, 10, 0, 345, 0, 0, 4057, 2380, 3966, 0, 3748, 179, 0, 0, 0, 0, 0, 11, 0, 613, 5, 2725, 7, 23, 0, 6, 0, 0, 4060] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 2071, 0, 0, 2667, 5, 0, 79, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 71, 0, 0, 0, 0, 28, 0, 0, 3098, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2663, 3399, 3444, 0, 1636, 381, 0, 0, 0, 0, 0, 13, 0, 1296, 0, 3729, 0, 0, 0, 0, 0, 0, 4089] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 135, 0, 51, 3622, 0, 1, 293, 4060, 1, 0, 0, 2, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 577, 1, 0, 2, 0, 51, 0, 8, 36, 1, 6, 0, 0, 3, 0, 81, 0, 15, 4005, 3906, 3820, 0, 3194, 716, 0, 0, 0, 2, 60, 13, 0, 48, 5, 3845, 9, 57, 1, 4, 0, 0, 4088] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 3579, 0, 0, 2572, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4001, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 0, 0, 0, 0, 105, 3762, 370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 686, 0, 0, 120, 1561, 1432, 59, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 2, 0, 3997, 198, 0, 0, 0, 0, 0, 0, 1, 0, 386, 0, 5, 1220, 599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 38, 0, 3106, 674, 0, 0, 0, 0, 0, 0, 3680, 0, 3, 0, 0, 0, 1, 592, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6153, 4004, 4243, 3858, 4203, 2211, 3820, 4276] avg=4096.0000 max=6153 max_rank=0 min=2211 min_rank=5 imbalance_ratio=1.502197 max_load_reduction=26615 max_load_reduction_pct=0.812225 moved_to_echo=14510 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2408, 0, 0, 1212, 8, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 3079, 360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3210, 1468, 675, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 4035, 162, 0, 0, 0, 0, 0, 0, 0, 0, 1172, 0, 0, 1148, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 0, 3, 0, 3023, 634, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 0, 0, 0, 0, 576, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3651, 4033, 3639, 5414, 4241, 3385, 3766, 4639] avg=4096.0000 max=5414 max_rank=3 min=3385 min_rank=5 imbalance_ratio=1.321777 max_load_reduction=27354 max_load_reduction_pct=0.834778 moved_to_echo=16031 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2, 2285, 4, 0, 1926, 1, 0, 108, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 19, 0, 347, 3470, 370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 691, 14, 0, 1034, 1664, 1073, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 2963, 181, 0, 0, 0, 0, 0, 0, 1, 0, 2037, 0, 2, 1300, 369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 31, 0, 331, 0, 330, 0, 3107, 719, 0, 0, 0, 0, 0, 0, 3674, 0, 7, 0, 0, 0, 1, 592, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4326, 4034, 4210, 4535, 3160, 3709, 4520, 4274] avg=4096.0000 max=4535 max_rank=3 min=3160 min_rank=4 imbalance_ratio=1.107178 max_load_reduction=28233 max_load_reduction_pct=0.861603 moved_to_echo=15663 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2828, 0, 21, 1563, 0, 0, 168, 0, 0, 0, 0, 0, 0, 0, 0, 3997, 0, 0, 0, 317, 1, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 8, 22, 0, 243, 3380, 372, 0, 0, 0, 0, 0, 0, 2, 6, 2, 0, 215, 0, 0, 2297, 1637, 870, 59, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 468, 0, 4047, 178, 0, 0, 0, 0, 0, 0, 0, 0, 1086, 0, 0, 1278, 450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 86, 8, 15, 12, 3123, 707, 0, 0, 0, 0, 0, 0, 2548, 1, 108, 0, 7, 0, 0, 596, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4580, 4320, 4060, 5088, 4694, 2814, 3952, 3260] avg=4096.0000 max=5088 max_rank=3 min=2814 min_rank=5 imbalance_ratio=1.242188 max_load_reduction=27680 max_load_reduction_pct=0.844727 moved_to_echo=14720 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2190, 0, 0, 969, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 0, 0, 0, 133, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 540, 3662, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 17, 0, 0, 485, 1075, 540, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 78, 0, 3980, 191, 0, 0, 0, 0, 0, 0, 0, 0, 3494, 0, 1, 841, 2316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 8, 32, 0, 3089, 464, 0, 0, 0, 0, 0, 0, 3553, 0, 0, 0, 0, 0, 0, 589, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3178, 4174, 4571, 2177, 4253, 6652, 3621, 4142] avg=4096.0000 max=6652 max_rank=5 min=2177 min_rank=3 imbalance_ratio=1.624023 max_load_reduction=26116 max_load_reduction_pct=0.796997 moved_to_echo=18668 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 3197, 0, 0, 1011, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 40, 0, 0, 89, 2, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8, 0, 2, 23, 3855, 370, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 4, 0, 0, 1577, 1694, 562, 59, 0, 0, 0, 0, 0, 0, 3, 0, 3, 10, 0, 345, 0, 3966, 202, 0, 0, 0, 0, 0, 0, 0, 0, 2380, 0, 0, 1323, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 613, 5, 3100, 731, 0, 0, 0, 0, 0, 0, 2725, 7, 23, 0, 6, 0, 0, 590, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4210, 4164, 4261, 3911, 4529, 3882, 4460, 3351] avg=4096.0000 max=4529 max_rank=4 min=3351 min_rank=7 imbalance_ratio=1.105713 max_load_reduction=28239 max_load_reduction_pct=0.861786 moved_to_echo=16222 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2071, 0, 0, 1713, 5, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 71, 2530, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 3098, 739, 954, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3444, 133, 0, 0, 0, 0, 0, 0, 0, 0, 3399, 0, 0, 578, 381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 1296, 0, 3122, 319, 0, 0, 0, 0, 0, 0, 3729, 0, 0, 0, 0, 0, 0, 595, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3868, 4038, 2974, 4878, 3578, 4358, 4750, 4324] avg=4096.0000 max=4878 max_rank=3 min=2974 min_rank=2 imbalance_ratio=1.190918 max_load_reduction=27890 max_load_reduction_pct=0.851135 moved_to_echo=17010 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 135, 0, 51, 2322, 0, 1, 293, 0, 0, 0, 0, 0, 0, 0, 0, 3998, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 577, 3806, 378, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 51, 0, 8, 36, 1447, 1300, 62, 0, 0, 0, 0, 0, 1, 6, 0, 0, 3, 0, 81, 0, 3820, 199, 0, 0, 0, 0, 0, 0, 15, 0, 3906, 0, 0, 1120, 716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 60, 13, 0, 48, 5, 3124, 627, 0, 0, 0, 0, 0, 0, 3845, 9, 57, 1, 4, 0, 0, 586, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2802, 4001, 4810, 2907, 4110, 5757, 3879, 4502] avg=4096.0000 max=5757 max_rank=5 min=2802 min_rank=0 imbalance_ratio=1.405518 max_load_reduction=27011 max_load_reduction_pct=0.824310 moved_to_echo=18248 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 1792, 128, 128, 11904, 11392, 7424, 512, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 17920, 0, 128, 8832, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 18816, 128, 128, 13312, 128, 128, 768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[27904, 128, 256, 128, 128, 0, 128, 4736, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 2176, 27648, 3072, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 640, 128, 2432, 128, 24832, 4992, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 128, 128, 1152, 0, 30336, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[32256, 128, 128, 0, 640, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33792 slack_slots=31744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33792 slack_slots=31744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33792 slack_slots=31744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 631, 0, 249, 0, 0, 0, 0, 0, 385, 0, 382, 0, 0, 0, 3898, 936, 51, 0, 1, 0, 0, 1481, 377, 6, 0, 0, 3854, 0, 0, 3922, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 4, 2661, 3796, 0, 2302, 0, 0, 4069, 0, 0, 0, 0, 0, 2680, 7, 0, 1066, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 727, 0, 3902, 0, 0, 0, 0, 0, 3477, 0, 1202, 0, 3, 6, 711, 330, 0, 4, 0, 216, 0, 134, 130, 0, 0, 0, 4016, 0, 0, 3200, 0, 0, 0, 40, 0, 54, 0, 0, 0, 0, 0, 3328, 1694, 4, 2926, 0, 1, 3895, 8, 8, 0, 21, 8, 101, 0, 0, 2608, 0, 3, 0, 11] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 699, 0, 369, 0, 0, 0, 1, 0, 3025, 0, 1226, 10, 0, 9, 2179, 1368, 0, 1, 0, 276, 0, 149, 420, 0, 1, 5, 766, 7, 0, 4082, 2, 10, 7, 0, 31, 260, 0, 0, 0, 0, 0, 4004, 3041, 8, 2324, 0, 0, 4089, 0, 0, 0, 15, 10, 771, 0, 0, 3591, 0, 8, 4, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[9, 0, 23, 705, 0, 3504, 14, 1, 12, 0, 0, 826, 0, 41, 0, 0, 0, 1603, 2473, 6, 0, 0, 0, 0, 65, 19, 1, 0, 0, 442, 0, 0, 3608, 0, 6, 0, 24, 1, 23, 11, 0, 0, 1, 0, 4050, 3734, 0, 2649, 1, 0, 4091, 0, 0, 0, 0, 0, 2168, 0, 2, 2654, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 2657, 0, 2002, 0, 0, 0, 0, 0, 1158, 0, 1858, 1, 0, 0, 1511, 35, 4, 0, 0, 0, 0, 1501, 2, 0, 0, 0, 3773, 0, 0, 3953, 0, 0, 0, 0, 6, 7, 0, 0, 0, 0, 0, 3429, 1033, 0, 2328, 0, 0, 4096, 0, 0, 0, 0, 0, 2887, 0, 0, 527, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 1483, 0, 3533, 0, 0, 0, 0, 0, 1787, 0, 3303, 6, 0, 0, 2473, 103, 13, 0, 0, 0, 0, 4, 15, 0, 0, 0, 2481, 0, 0, 3978, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 2728, 1521, 0, 740, 0, 0, 4096, 0, 0, 0, 0, 0, 3669, 10, 0, 821, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 3518, 0, 565, 0, 0, 0, 0, 0, 250, 0, 994, 0, 0, 0, 3771, 77, 44, 0, 0, 0, 0, 1997, 10, 0, 0, 0, 164, 0, 0, 3943, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 2202, 1194, 0, 3802, 0, 0, 4096, 0, 0, 0, 0, 0, 2380, 0, 0, 3748, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 18, 0, 3135, 0, 103, 175, 0, 0, 2, 0, 1739, 0, 3579, 1, 0, 4, 659, 83, 0, 4, 0, 0, 0, 3099, 233, 18, 0, 8, 851, 5, 0, 4043, 1, 0, 0, 0, 44, 1824, 0, 0, 0, 0, 0, 623, 3938, 14, 1411, 0, 0, 3801, 5, 5, 0, 0, 10, 3162, 0, 0, 163, 0, 8, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2657, 0, 2002, 0, 0, 236, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1158, 0, 1858, 1, 0, 1116, 0, 0, 0, 0, 0, 0, 0, 0, 1511, 35, 4, 0, 0, 0, 0, 1477, 0, 0, 0, 0, 0, 0, 0, 1501, 2, 0, 0, 0, 3773, 0, 0, 836, 58, 0, 0, 0, 0, 0, 0, 3911, 0, 0, 0, 0, 6, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 739, 0, 2328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 45, 20, 0, 0, 0, 0, 0, 0, 2842, 0, 0, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4917, 4133, 3027, 6170, 3924, 3067, 4161, 3369] avg=4096.0000 max=6170 max_rank=3 min=3027 min_rank=2 imbalance_ratio=1.506348 max_load_reduction=26598 max_load_reduction_pct=0.811707 moved_to_echo=14521 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 631, 0, 249, 0, 0, 869, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 385, 0, 382, 0, 0, 866, 0, 0, 0, 0, 0, 0, 0, 0, 3898, 936, 51, 0, 1, 0, 0, 1148, 0, 0, 0, 0, 0, 0, 0, 1481, 377, 6, 0, 0, 3854, 0, 0, 647, 214, 0, 0, 0, 0, 0, 0, 3880, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2713, 0, 2302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 42, 20, 0, 0, 0, 0, 0, 0, 2638, 7, 0, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1771, 1633, 6034, 6579, 3890, 5019, 4131, 3711] avg=4096.0000 max=6579 max_rank=3 min=1633 min_rank=1 imbalance_ratio=1.606201 max_load_reduction=26189 max_load_reduction_pct=0.799225 moved_to_echo=16751 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 727, 0, 3902, 0, 0, 387, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3477, 0, 1202, 0, 3, 1084, 0, 0, 0, 0, 0, 0, 0, 6, 711, 330, 0, 4, 0, 216, 0, 1433, 0, 0, 0, 0, 0, 0, 0, 134, 130, 0, 0, 0, 4016, 0, 0, 811, 95, 0, 0, 0, 0, 0, 0, 3166, 0, 0, 0, 40, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1212, 4, 2926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3895, 8, 8, 0, 21, 8, 1, 16, 0, 0, 0, 0, 0, 0, 100, 0, 0, 2608, 0, 3, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5034, 5766, 2700, 5186, 3260, 4142, 3958, 2722] avg=4096.0000 max=5766 max_rank=1 min=2700 min_rank=2 imbalance_ratio=1.407715 max_load_reduction=27002 max_load_reduction_pct=0.824036 moved_to_echo=14082 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[9, 0, 23, 705, 0, 3504, 14, 1, 855, 20, 0, 0, 0, 0, 0, 0, 12, 0, 0, 826, 0, 41, 0, 0, 1320, 0, 0, 0, 0, 0, 0, 0, 0, 1603, 2473, 6, 0, 0, 0, 0, 1744, 0, 0, 0, 0, 0, 0, 0, 65, 19, 1, 0, 0, 442, 0, 0, 986, 211, 0, 0, 0, 0, 0, 0, 3570, 0, 6, 0, 24, 1, 23, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2668, 0, 2649, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4091, 0, 0, 0, 0, 0, 34, 18, 0, 0, 0, 0, 0, 0, 2134, 0, 2, 2654, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5131, 2199, 5826, 1724, 3635, 5318, 4144, 4791] avg=4096.0000 max=5826 max_rank=2 min=1724 min_rank=3 imbalance_ratio=1.422363 max_load_reduction=26942 max_load_reduction_pct=0.822205 moved_to_echo=17888 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 699, 0, 369, 0, 0, 696, 23, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3025, 0, 1226, 10, 0, 1305, 0, 0, 0, 0, 0, 0, 0, 9, 2179, 1368, 0, 1, 0, 276, 0, 1725, 0, 0, 0, 0, 0, 0, 0, 149, 420, 0, 1, 5, 766, 7, 0, 974, 171, 0, 0, 0, 0, 0, 0, 4038, 2, 10, 7, 0, 31, 260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2174, 8, 2324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 15, 10, 12, 21, 0, 0, 0, 0, 0, 0, 759, 0, 0, 3591, 0, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1787, 5567, 5558, 2493, 4348, 4506, 4147, 4362] avg=4096.0000 max=5567 max_rank=1 min=1787 min_rank=0 imbalance_ratio=1.359131 max_load_reduction=27201 max_load_reduction_pct=0.830109 moved_to_echo=17363 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1483, 0, 3533, 0, 0, 348, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1787, 0, 3303, 6, 0, 888, 0, 0, 0, 0, 0, 0, 0, 0, 2473, 103, 13, 0, 0, 0, 0, 1176, 0, 0, 0, 0, 0, 0, 0, 4, 15, 0, 0, 0, 2481, 0, 0, 664, 85, 0, 0, 0, 0, 0, 0, 3936, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1088, 0, 740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 58, 20, 0, 0, 0, 0, 0, 0, 3611, 10, 0, 821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5386, 5984, 3765, 3249, 3939, 1829, 4174, 4442] avg=4096.0000 max=5984 max_rank=1 min=1829 min_rank=5 imbalance_ratio=1.460938 max_load_reduction=26784 max_load_reduction_pct=0.817383 moved_to_echo=14384 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3518, 0, 565, 0, 0, 273, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 250, 0, 994, 0, 0, 717, 0, 0, 0, 0, 0, 0, 0, 0, 3771, 77, 44, 0, 0, 0, 0, 948, 0, 0, 0, 0, 0, 0, 0, 1997, 10, 0, 0, 0, 164, 0, 0, 537, 67, 0, 0, 0, 0, 0, 0, 3901, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 854, 0, 3802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 37, 20, 0, 0, 0, 0, 0, 0, 2343, 0, 0, 3748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4378, 1961, 4840, 2775, 3914, 4656, 4153, 6091] avg=4096.0000 max=6091 max_rank=7 min=1961 min_rank=1 imbalance_ratio=1.487061 max_load_reduction=26677 max_load_reduction_pct=0.814117 moved_to_echo=18814 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 18, 0, 3135, 0, 103, 175, 0, 905, 28, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1739, 0, 3579, 1, 0, 204, 0, 0, 0, 0, 0, 0, 0, 4, 659, 83, 0, 4, 0, 0, 0, 268, 0, 0, 0, 0, 0, 0, 0, 3099, 233, 18, 0, 8, 851, 5, 0, 151, 227, 0, 0, 0, 0, 0, 0, 3989, 1, 0, 0, 0, 44, 1824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2806, 14, 1411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3801, 5, 5, 0, 0, 10, 53, 26, 0, 0, 0, 0, 0, 0, 3109, 0, 0, 163, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4364, 5525, 1018, 4592, 5858, 4231, 3900, 3280] avg=4096.0000 max=5858 max_rank=4 min=1018 min_rank=2 imbalance_ratio=1.430176 max_load_reduction=26910 max_load_reduction_pct=0.821228 moved_to_echo=17269 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 0, 12672, 0, 12672, 128, 128, 7552, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 16896, 5504, 128, 128, 128, 512, 0, 9984, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 0, 14336, 128, 18560, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 13568, 0, 14336, 256, 128, 4608, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[17536, 128, 128, 15232, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8448, 1280, 128, 128, 128, 16384, 128, 0, 5632, 1152, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[30464, 128, 128, 128, 128, 128, 2304, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 32256, 128, 128, 0, 128, 128, 384, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 373, 3924, 0, 33, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3206, 4095, 1, 4078, 0, 3988, 12, 0, 0, 0, 4074, 0, 0, 0, 0, 1, 3, 0, 0, 0, 2071, 31, 0, 0, 3, 2, 0, 2863, 155, 0, 0, 0, 0, 0, 0, 3721, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[60, 0, 51, 0, 7, 0, 0, 4, 15, 0, 0, 3, 1776, 3449, 0, 142, 0, 2, 0, 0, 0, 569, 32, 2111, 0, 0, 625, 4074, 5, 3958, 0, 2220, 159, 0, 12, 0, 4044, 2, 72, 0, 0, 0, 348, 11, 0, 9, 232, 0, 0, 0, 0, 0, 0, 2874, 1801, 4, 1, 0, 7, 0, 0, 4074, 14, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[31, 0, 737, 0, 0, 170, 0, 0, 0, 0, 0, 9, 1881, 1922, 0, 2, 1, 0, 0, 0, 0, 9, 0, 0, 0, 0, 909, 4091, 0, 4086, 0, 4031, 27, 0, 0, 0, 4071, 2, 7, 1, 0, 1, 197, 0, 0, 0, 3564, 51, 0, 0, 10, 0, 0, 3004, 68, 0, 2, 0, 0, 0, 0, 3884, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[3, 0, 1030, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1459, 3937, 0, 1, 0, 0, 0, 0, 0, 407, 4, 442, 0, 0, 1737, 4080, 0, 4095, 0, 3498, 29, 0, 0, 0, 4096, 0, 21, 0, 0, 0, 19, 0, 0, 0, 1530, 0, 1, 0, 0, 0, 0, 3005, 64, 0, 0, 0, 0, 0, 0, 3309, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[14, 0, 999, 0, 33, 0, 0, 7, 16, 0, 0, 0, 408, 3092, 0, 51, 0, 0, 0, 0, 0, 128, 42, 1, 4, 0, 1653, 4039, 118, 3925, 0, 4085, 0, 0, 2, 0, 4048, 0, 215, 0, 0, 0, 680, 7, 0, 0, 1535, 0, 0, 14, 82, 0, 0, 2906, 602, 7, 0, 4, 5, 0, 0, 4022, 13, 11] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 627, 0, 0, 4, 0, 0, 0, 0, 0, 0, 847, 3586, 0, 4, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 1803, 4094, 0, 4088, 0, 3856, 43, 0, 0, 0, 4096, 0, 907, 0, 0, 0, 554, 0, 0, 0, 401, 0, 0, 0, 0, 0, 0, 2971, 1131, 0, 0, 0, 0, 0, 0, 3750, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 989, 0, 0, 62, 0, 0, 0, 0, 0, 0, 645, 4094, 0, 0, 0, 0, 0, 0, 0, 288, 0, 98, 0, 0, 1942, 4096, 1, 3887, 0, 3706, 9, 0, 0, 0, 4096, 0, 302, 0, 0, 0, 314, 0, 0, 0, 448, 0, 0, 0, 1, 0, 0, 3208, 989, 0, 0, 0, 0, 0, 0, 3593, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 3181, 0, 3, 27, 43, 0, 12, 7, 0, 0, 7, 3767, 0, 172, 0, 10, 0, 20, 1, 7, 0, 0, 1, 38, 954, 4066, 16, 4037, 0, 3787, 14, 0, 0, 0, 4050, 1, 25, 0, 0, 0, 587, 16, 0, 0, 61, 644, 0, 0, 0, 18, 37, 3908, 565, 0, 0, 0, 5, 0, 0, 2667, 14, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 133, 0, 0, 0, 0, 0, 2600, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 373, 3520, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3206, 0, 1, 0, 0, 2701, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 3866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 2071, 31, 1478, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 2863, 155, 0, 208, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3721, 0, 0, 291, 220, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3240, 3926, 3589, 5908, 3878, 4651, 3344, 4232] avg=4096.0000 max=5908 max_rank=3 min=3240 min_rank=0 imbalance_ratio=1.442383 max_load_reduction=26860 max_load_reduction_pct=0.819702 moved_to_echo=16105 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[31, 0, 737, 0, 0, 170, 0, 0, 2606, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1881, 1725, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 9, 0, 0, 3583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 909, 0, 0, 0, 0, 2731, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 3863, 2, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 197, 0, 0, 0, 3564, 51, 1480, 1078, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 3004, 68, 0, 208, 55, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3884, 0, 0, 142, 222, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4052, 3617, 3593, 3640, 3900, 6371, 3345, 4250] avg=4096.0000 max=6371 max_rank=5 min=3345 min_rank=6 imbalance_ratio=1.555420 max_load_reduction=26397 max_load_reduction_pct=0.805573 moved_to_echo=17866 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 627, 0, 0, 4, 0, 0, 2607, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 847, 3216, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1803, 0, 0, 0, 0, 2613, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 3887, 0, 907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 0, 0, 0, 401, 0, 1481, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2971, 1131, 0, 209, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3750, 0, 0, 266, 212, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3745, 4067, 3593, 4416, 4837, 3467, 4415, 4228] avg=4096.0000 max=4837 max_rank=4 min=3467 min_rank=5 imbalance_ratio=1.180908 max_load_reduction=27931 max_load_reduction_pct=0.852386 moved_to_echo=16947 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[14, 0, 999, 0, 33, 0, 0, 7, 2503, 500, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 408, 2774, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 42, 1, 3539, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1653, 0, 118, 0, 0, 2768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3841, 0, 215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 680, 7, 0, 0, 1535, 0, 1422, 1092, 0, 0, 0, 0, 0, 0, 0, 14, 82, 0, 0, 2906, 602, 7, 207, 89, 0, 0, 0, 0, 0, 0, 0, 4, 5, 0, 0, 4022, 13, 11, 229, 225, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4056, 3249, 3710, 4543, 4058, 4736, 3907, 4509] avg=4096.0000 max=4736 max_rank=5 min=3249 min_rank=1 imbalance_ratio=1.156250 max_load_reduction=28032 max_load_reduction_pct=0.855469 moved_to_echo=17210 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[60, 0, 51, 0, 7, 0, 0, 4, 2525, 506, 0, 0, 0, 0, 0, 0, 15, 0, 0, 3, 1776, 3094, 0, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 569, 32, 2111, 3568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 625, 0, 5, 0, 0, 1505, 0, 0, 0, 0, 0, 0, 0, 0, 159, 0, 12, 0, 3837, 2, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 348, 11, 0, 9, 232, 0, 1433, 593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2874, 1801, 4, 207, 100, 0, 0, 0, 0, 0, 0, 1, 0, 7, 0, 0, 4074, 14, 1, 255, 122, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3153, 5030, 6282, 2135, 4082, 2626, 4986, 4474] avg=4096.0000 max=6282 max_rank=2 min=2135 min_rank=3 imbalance_ratio=1.533691 max_load_reduction=26486 max_load_reduction_pct=0.808289 moved_to_echo=16168 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 1030, 0, 1, 0, 0, 0, 2612, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1459, 3531, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 407, 4, 442, 3574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1737, 0, 0, 0, 0, 2370, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 3887, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 1530, 0, 1483, 935, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3005, 64, 0, 209, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3309, 0, 0, 292, 193, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4152, 4991, 4427, 4107, 3937, 3967, 3393, 3794] avg=4096.0000 max=4991 max_rank=1 min=3393 min_rank=6 imbalance_ratio=1.218506 max_load_reduction=27777 max_load_reduction_pct=0.847687 moved_to_echo=15091 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3181, 0, 3, 27, 43, 0, 2575, 505, 0, 0, 0, 0, 0, 0, 12, 7, 0, 0, 7, 3372, 0, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 20, 1, 7, 0, 0, 3561, 0, 0, 0, 0, 0, 0, 0, 1, 38, 954, 0, 16, 0, 0, 2556, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 3838, 1, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 587, 16, 0, 0, 61, 644, 1462, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 37, 3908, 565, 0, 212, 113, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 2667, 14, 0, 282, 213, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6334, 3570, 3599, 3565, 3878, 3788, 4853, 3181] avg=4096.0000 max=6334 max_rank=0 min=3181 min_rank=7 imbalance_ratio=1.546387 max_load_reduction=26434 max_load_reduction_pct=0.806702 moved_to_echo=15700 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 989, 0, 0, 62, 0, 0, 2478, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 645, 3673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 288, 0, 98, 3589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1942, 0, 1, 0, 0, 2511, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 3887, 0, 302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 314, 0, 0, 0, 448, 0, 1409, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3208, 989, 0, 209, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3593, 0, 0, 303, 204, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4036, 4318, 3975, 4454, 4198, 3162, 4525, 4100] avg=4096.0000 max=4525 max_rank=6 min=3162 min_rank=5 imbalance_ratio=1.104736 max_load_reduction=28243 max_load_reduction_pct=0.861908 moved_to_echo=15985 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 7808, 0, 128, 384, 128, 128, 20608, 4096, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 12928, 0, 256, 0, 0, 19840, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 2816, 128, 0, 128, 9856, 768, 11648, 7808, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 24832, 5376, 128, 1792, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 0, 29056, 128, 128, 2176, 1664, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[384, 0, 128, 0, 30976, 128, 1664, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 128, 1408, 128, 2688, 28672, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 7424, 24960, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 9, 0, 0, 71, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 16, 53, 0, 0, 0, 0, 0, 0, 3833, 4096, 4063, 173, 0, 0, 1, 2117, 0, 1, 0, 2226, 3467, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 301, 18] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[5, 3, 16, 484, 0, 0, 73, 0, 0, 12, 0, 0, 2, 0, 3633, 142, 0, 0, 25, 0, 0, 1, 0, 4085, 0, 3, 0, 0, 0, 0, 3, 10, 0, 0, 15, 4072, 3805, 2821, 0, 0, 23, 0, 4038, 2, 51, 0, 2691, 2715, 0, 0, 0, 5, 0, 0, 0, 21, 0, 1, 3898, 0, 0, 0, 0, 113] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 4088, 4095, 4078, 155, 0, 0, 0, 2976, 0, 0, 0, 1340, 3658, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 107, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[3, 7, 10, 0, 0, 0, 9, 0, 0, 10, 224, 0, 2, 4, 4052, 79, 0, 0, 20, 0, 0, 0, 0, 4086, 16, 1, 0, 0, 7, 5, 221, 1, 0, 1, 22, 3977, 3868, 3499, 24, 0, 7, 0, 2837, 13, 1, 4, 1819, 3794, 0, 0, 0, 4, 0, 0, 3, 0, 0, 0, 4078, 1, 0, 0, 59, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 19, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 165, 0, 0, 4016, 0, 0, 0, 0, 1190, 0, 17, 0, 8, 0, 0, 3459, 4084, 4077, 94, 0, 0, 0, 3748, 0, 1, 0, 952, 2200, 0, 11, 0, 0, 0, 1, 32, 0, 0, 0, 4037, 0, 0, 0, 575, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 74, 0, 0, 0, 9, 0, 0, 138, 0, 0, 0, 3946, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 3, 48, 0, 0, 0, 0, 0, 0, 4007, 4096, 3719, 35, 0, 0, 0, 2436, 0, 0, 0, 2832, 3158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 35, 43] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4090, 4035, 2, 0, 0, 0, 3711, 0, 0, 0, 351, 4063, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 110, 19] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1, 18, 0, 0, 0, 0, 0, 0, 0, 13, 145, 0, 9, 44, 4053, 18, 0, 11, 0, 11, 0, 7, 76, 4079, 0, 2, 0, 0, 0, 2, 77, 0, 0, 0, 9, 4046, 3891, 4000, 3, 0, 17, 0, 638, 5, 0, 11, 3491, 2778, 0, 937, 0, 12, 0, 0, 0, 0, 0, 0, 3938, 2, 0, 0, 422, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 9, 0, 0, 71, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 53, 0, 0, 0, 3204, 524, 170, 23, 12, 0, 0, 0, 0, 0, 0, 235, 0, 4063, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1593, 0, 1, 0, 2226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 3598, 263, 9, 0, 0, 0, 0, 0, 0, 0, 3903, 0, 0, 0, 301, 18, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4167, 4079, 4065, 4002, 4471, 3821, 3941, 4222] avg=4096.0000 max=4471 max_rank=4 min=3821 min_rank=5 imbalance_ratio=1.091553 max_load_reduction=28297 max_load_reduction_pct=0.863556 moved_to_echo=16455 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 3380, 736, 171, 23, 12, 0, 0, 0, 0, 0, 0, 251, 0, 4078, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2240, 0, 0, 0, 1340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3837, 278, 9, 0, 0, 0, 0, 0, 0, 0, 3925, 0, 0, 0, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4061, 4072, 4328, 4484, 3580, 4124, 4032] avg=4096.0000 max=4484 max_rank=4 min=3580 min_rank=5 imbalance_ratio=1.094727 max_load_reduction=28284 max_load_reduction_pct=0.863159 moved_to_echo=16220 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 19, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 0, 0, 3993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1190, 0, 17, 0, 2034, 928, 168, 23, 12, 0, 0, 0, 8, 0, 0, 212, 0, 4077, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2820, 0, 1, 0, 952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 1, 32, 0, 3247, 166, 8, 0, 0, 0, 0, 0, 0, 0, 3869, 0, 0, 0, 575, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4069, 4158, 4372, 4391, 3773, 3465, 4444] avg=4096.0000 max=4444 max_rank=7 min=3465 min_rank=6 imbalance_ratio=1.084961 max_load_reduction=28324 max_load_reduction_pct=0.864380 moved_to_echo=16073 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[5, 3, 16, 484, 0, 0, 73, 0, 3796, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 2, 0, 3622, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 1, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3, 10, 2509, 999, 162, 23, 11, 0, 0, 0, 0, 0, 15, 250, 0, 2821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 3039, 2, 51, 0, 2691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 21, 3822, 206, 9, 0, 0, 0, 0, 0, 0, 1, 3736, 0, 0, 0, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4377, 3778, 4088, 3720, 3086, 5806, 4063, 3850] avg=4096.0000 max=5806 max_rank=5 min=3086 min_rank=4 imbalance_ratio=1.417480 max_load_reduction=26962 max_load_reduction_pct=0.822815 moved_to_echo=16805 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3, 7, 10, 0, 0, 0, 9, 0, 3859, 0, 0, 0, 0, 0, 0, 0, 0, 10, 224, 0, 2, 4, 4040, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 16, 1, 0, 0, 7, 5, 221, 1, 3507, 702, 170, 23, 12, 0, 0, 0, 0, 1, 22, 244, 0, 3499, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 2135, 13, 1, 4, 1819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 3, 0, 3733, 287, 9, 0, 0, 0, 0, 0, 0, 0, 3908, 1, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3888, 4359, 4083, 4665, 3790, 3979, 4036, 3968] avg=4096.0000 max=4665 max_rank=3 min=3790 min_rank=4 imbalance_ratio=1.138916 max_load_reduction=28103 max_load_reduction_pct=0.857635 moved_to_echo=15773 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 74, 0, 0, 0, 9, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 0, 0, 0, 3934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 48, 0, 0, 0, 2919, 603, 171, 23, 12, 0, 0, 0, 0, 0, 0, 246, 0, 3719, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1833, 0, 0, 0, 2832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3761, 239, 9, 0, 0, 0, 0, 0, 0, 0, 3923, 0, 0, 0, 35, 43, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4170, 4072, 4072, 3779, 4000, 4665, 4009, 4001] avg=4096.0000 max=4665 max_rank=5 min=3779 min_rank=3 imbalance_ratio=1.138916 max_load_reduction=28103 max_load_reduction_pct=0.857635 moved_to_echo=16675 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3755, 918, 171, 23, 12, 0, 0, 0, 0, 0, 0, 251, 0, 4035, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2793, 0, 0, 0, 351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3845, 308, 9, 0, 0, 0, 0, 0, 0, 0, 3925, 0, 0, 0, 110, 19, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4084, 4074, 4880, 4288, 3144, 4163, 4054] avg=4096.0000 max=4880 max_rank=3 min=3144 min_rank=5 imbalance_ratio=1.191406 max_load_reduction=27888 max_load_reduction_pct=0.851074 moved_to_echo=15649 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 18, 0, 0, 0, 0, 0, 0, 3883, 0, 0, 0, 0, 0, 0, 0, 0, 13, 145, 0, 9, 44, 4037, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 11, 0, 7, 76, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 77, 0, 2568, 162, 167, 28, 16, 0, 0, 0, 0, 0, 9, 246, 0, 4000, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 476, 5, 0, 11, 3491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 937, 0, 12, 0, 0, 0, 0, 3800, 210, 8, 0, 0, 0, 0, 0, 0, 0, 3771, 2, 0, 0, 422, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3902, 4266, 4156, 3022, 4258, 4000, 4967, 4197] avg=4096.0000 max=4967 max_rank=6 min=3022 min_rank=3 imbalance_ratio=1.212646 max_load_reduction=27801 max_load_reduction_pct=0.848419 moved_to_echo=17422 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 128, 1408, 128, 384, 128, 23936, 5632, 1408, 256, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 17024, 128, 128, 128, 15744, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 2048, 0, 30336, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 640, 128, 0, 128, 128, 32000, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 256, 128, 128, 32512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 640, 0, 128, 128, 32000, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 1024, 0, 128, 0, 128, 128, 128, 29696, 2048, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 30976, 128, 0, 0, 1664, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33792 slack_slots=31744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 3657, 0, 3158, 0, 0, 4095, 0, 4073, 1331, 0, 0, 709, 0, 0, 0, 10, 0, 0, 4096, 0, 1, 0, 4096, 0, 0, 17, 1, 395, 0, 0, 0, 0, 0, 0, 4095, 32, 0, 0, 2988, 0, 0, 0, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 3300, 0, 2147, 0, 0, 4096, 2, 3809, 159, 0, 0, 2606, 0, 0, 0, 10, 0, 0, 4096, 0, 0, 0, 4093, 0, 0, 1867, 0, 493, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 1945, 0, 0, 0, 40, 2, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 2147, 0, 258, 0, 0, 4096, 0, 3993, 760, 0, 0, 2118, 0, 0, 0, 72, 0, 0, 4059, 0, 0, 0, 4096, 0, 0, 1965, 0, 1069, 0, 0, 0, 0, 0, 0, 3972, 4, 0, 0, 4090, 0, 0, 0, 2, 21, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 3259, 17, 2021, 38, 0, 4078, 0, 3748, 20, 0, 0, 1556, 0, 36, 0, 0, 20, 0, 4081, 0, 0, 0, 3790, 0, 0, 358, 0, 2268, 9, 0, 10, 0, 0, 0, 4075, 412, 25, 4, 1784, 0, 0, 0, 1065, 0, 0, 8, 0, 0, 8, 0, 50, 0, 10, 0, 0, 0, 0, 5, 0, 5, 0, 8, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[8, 3664, 1, 3282, 38, 0, 4075, 0, 3437, 448, 0, 0, 2201, 6, 52, 0, 8, 4, 1, 4074, 0, 0, 0, 4054, 1, 0, 403, 0, 580, 14, 0, 15, 0, 0, 0, 4077, 5, 38, 10, 1721, 4, 0, 0, 231, 276, 0, 8, 0, 6, 0, 0, 0, 0, 13, 0, 0, 0, 4, 5, 0, 0, 4, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[2, 3620, 0, 1080, 0, 0, 4056, 0, 4077, 555, 0, 0, 2386, 0, 0, 1, 23, 0, 0, 4092, 0, 0, 0, 4087, 0, 0, 108, 6, 254, 0, 0, 0, 0, 5, 0, 4088, 135, 0, 0, 3976, 0, 0, 0, 158, 20, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 4019, 0, 2456, 0, 0, 4096, 0, 4068, 70, 0, 0, 1086, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 700, 0, 878, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 3022, 0, 0, 0, 1, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 4017, 17, 2215, 49, 0, 4069, 0, 3490, 39, 73, 0, 21, 0, 25, 0, 14, 0, 0, 4079, 0, 0, 0, 4044, 0, 0, 50, 0, 1845, 32, 0, 14, 0, 0, 0, 4019, 11, 18, 17, 3904, 0, 10, 0, 677, 1, 0, 5, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2111, 0, 3158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2192, 1331, 0, 0, 709, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 32, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 1, 395, 0, 0, 0, 1546, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1086, 32, 0, 0, 2988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 0, 0, 2982, 858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4055, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5269, 4232, 4139, 2982, 4106, 3846, 4090, 4104] avg=4096.0000 max=5269 max_rank=0 min=2982 min_rank=3 imbalance_ratio=1.286377 max_load_reduction=27499 max_load_reduction_pct=0.839203 moved_to_echo=16146 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1905, 0, 2147, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2050, 159, 0, 0, 2606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 32, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1867, 0, 493, 0, 0, 0, 1395, 956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1085, 0, 0, 0, 1945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 2, 0, 0, 0, 2982, 803, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 4055, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4054, 4815, 4135, 4711, 3030, 3827, 4091, 4105] avg=4096.0000 max=4815 max_rank=1 min=3030 min_rank=4 imbalance_ratio=1.175537 max_load_reduction=27953 max_load_reduction_pct=0.853058 moved_to_echo=15053 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1882, 17, 2021, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2017, 20, 0, 0, 1556, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 32, 0, 0, 0, 3790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 358, 0, 2268, 9, 0, 10, 1377, 941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1081, 412, 25, 4, 1784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1065, 0, 0, 8, 0, 2968, 790, 0, 0, 0, 0, 0, 0, 0, 8, 0, 50, 0, 10, 0, 0, 4040, 26, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 5, 0, 8, 0, 4078, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3958, 3629, 3842, 4963, 3306, 4831, 4134, 4105] avg=4096.0000 max=4963 max_rank=3 min=3306 min_rank=4 imbalance_ratio=1.211670 max_load_reduction=27805 max_load_reduction_pct=0.848541 moved_to_echo=16376 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2, 2090, 0, 1080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2194, 555, 0, 0, 2386, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 32, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 6, 254, 0, 0, 0, 1530, 1024, 0, 0, 0, 0, 0, 0, 0, 5, 0, 1084, 135, 0, 0, 3976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 20, 0, 0, 0, 2977, 859, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 30, 4051, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3172, 5136, 4142, 2922, 5200, 4014, 4117, 4065] avg=4096.0000 max=5200 max_rank=4 min=2922 min_rank=3 imbalance_ratio=1.269531 max_load_reduction=27568 max_load_reduction_pct=0.841309 moved_to_echo=17396 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[8, 2115, 1, 3282, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1850, 448, 0, 0, 2201, 6, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4, 1, 32, 0, 0, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 403, 0, 580, 14, 0, 15, 1549, 863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1082, 5, 38, 10, 1721, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 231, 276, 0, 8, 0, 2969, 724, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 13, 0, 0, 4033, 26, 0, 0, 0, 0, 0, 0, 0, 4, 5, 0, 0, 4, 0, 0, 4075, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5444, 4557, 4099, 3425, 2856, 4212, 4078, 4097] avg=4096.0000 max=5444 max_rank=0 min=2856 min_rank=4 imbalance_ratio=1.329102 max_load_reduction=27324 max_load_reduction_pct=0.833862 moved_to_echo=15243 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1240, 0, 258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2148, 760, 0, 0, 2118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 32, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1965, 0, 1069, 0, 0, 0, 907, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 4, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 21, 0, 0, 0, 2893, 842, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 19, 4018, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1498, 5026, 4200, 4944, 5147, 3758, 4090, 4105] avg=4096.0000 max=5147 max_rank=4 min=1498 min_rank=0 imbalance_ratio=1.256592 max_load_reduction=27621 max_load_reduction_pct=0.842926 moved_to_echo=17100 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2320, 0, 2456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2189, 70, 0, 0, 1086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 700, 0, 878, 0, 0, 0, 1699, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1086, 6, 0, 0, 3022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 75, 0, 0, 0, 2983, 857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4055, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4776, 3345, 4128, 4299, 4114, 3916, 4085, 4105] avg=4096.0000 max=4776 max_rank=0 min=3345 min_rank=1 imbalance_ratio=1.166016 max_load_reduction=27992 max_load_reduction_pct=0.854248 moved_to_echo=16220 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2316, 17, 2215, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1870, 39, 73, 0, 21, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 25, 0, 0, 0, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 1845, 32, 0, 14, 1701, 880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 11, 18, 17, 3904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 677, 1, 0, 5, 0, 2931, 740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 4041, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 13, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4597, 2028, 4083, 4522, 5009, 4364, 4083, 4082] avg=4096.0000 max=5009 max_rank=4 min=2028 min_rank=1 imbalance_ratio=1.222900 max_load_reduction=27759 max_load_reduction_pct=0.847137 moved_to_echo=17538 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 5504, 128, 7808, 128, 0, 128, 11776, 7808, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 16000, 128, 16640, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 0, 2304, 512, 0, 128, 0, 23808, 6528, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 0, 8704, 640, 128, 128, 23552, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[16512, 3456, 128, 0, 12800, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 0, 128, 0, 128, 32384, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 128, 128, 256, 0, 128, 0, 32384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 128, 128, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1395, 3, 0, 1, 0, 15, 0, 20, 1118, 0, 0, 1661, 2067, 0, 2385, 0, 0, 0, 18, 4073, 0, 0, 124, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 3651, 0, 4062, 0, 0, 0, 0, 3969, 4077, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 230, 0, 0, 3880, 0, 0, 43, 3829, 0, 670, 0, 0, 0, 1, 4090, 0, 0, 421, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 4074, 0, 4092, 0, 0, 0, 0, 3252, 4083, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 38, 6, 0, 3, 2, 0, 36, 0, 0, 3084, 0, 0, 1293, 3813, 10, 2343, 0, 0, 0, 0, 4063, 34, 5, 129, 10, 0, 0, 6, 14, 66, 0, 127, 11, 0, 3, 0, 24, 226, 0, 4003, 0, 0, 12, 0, 4075, 0, 4080, 3, 16, 4, 0, 1152, 4009, 0, 0, 1, 10, 19, 16, 18] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[17, 0, 9, 0, 0, 0, 0, 78, 0, 0, 1744, 0, 0, 423, 0, 0, 1427, 1660, 11, 1462, 0, 21, 0, 0, 4027, 0, 0, 1480, 0, 0, 0, 0, 2, 7, 0, 50, 8, 0, 8, 54, 423, 61, 0, 3954, 0, 0, 6, 0, 4076, 0, 4070, 4, 18, 3, 0, 3669, 3940, 0, 0, 2, 10, 18, 16, 10] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 903, 0, 1, 3381, 0, 0, 255, 2814, 0, 431, 3, 0, 0, 0, 4087, 0, 0, 1117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 4009, 0, 4094, 0, 0, 0, 0, 3479, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 85, 0, 0, 1624, 0, 0, 659, 4027, 0, 1558, 0, 0, 0, 14, 4075, 0, 0, 1735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 264, 0, 4089, 0, 0, 0, 0, 4083, 0, 4078, 0, 0, 0, 0, 2349, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 150, 0, 0, 3089, 0, 0, 263, 3219, 0, 1055, 0, 0, 0, 0, 4093, 0, 0, 1697, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 38, 2, 0, 4096, 0, 0, 0, 0, 4091, 0, 4096, 0, 0, 0, 0, 2757, 4077, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 6, 0, 23, 0, 0, 0, 0, 0, 6, 196, 0, 327, 0, 1, 344, 2302, 13, 1041, 0, 0, 0, 0, 4013, 29, 0, 4054, 6, 0, 0, 0, 69, 24, 6, 123, 2, 0, 4, 0, 3, 45, 0, 4055, 0, 0, 0, 0, 4070, 0, 4077, 29, 24, 19, 0, 3845, 3923, 0, 21, 4, 0, 30, 17, 17] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1395, 3, 0, 2721, 834, 62, 0, 0, 0, 0, 0, 1, 0, 15, 0, 20, 1118, 0, 0, 691, 493, 71, 0, 0, 0, 0, 0, 1661, 1376, 0, 2385, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 2746, 0, 0, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 4000, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 930, 0, 0, 0, 0, 0, 0, 3969, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5015, 2409, 5440, 2870, 4031, 3995, 4899, 4109] avg=4096.0000 max=5440 max_rank=2 min=2409 min_rank=1 imbalance_ratio=1.328125 max_load_reduction=27328 max_load_reduction_pct=0.833984 moved_to_echo=17034 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 3037, 837, 62, 0, 0, 0, 0, 0, 0, 0, 230, 0, 0, 3880, 0, 0, 1281, 495, 71, 0, 0, 0, 0, 0, 43, 2548, 0, 670, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2758, 0, 0, 421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 0, 0, 0, 0, 0, 0, 3252, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3936, 5957, 3262, 3179, 4036, 3994, 4289, 4115] avg=4096.0000 max=5957 max_rank=1 min=3179 min_rank=3 imbalance_ratio=1.454346 max_load_reduction=26811 max_load_reduction_pct=0.818207 moved_to_echo=16434 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 38, 6, 0, 3, 3038, 832, 63, 0, 0, 0, 0, 0, 2, 0, 36, 0, 0, 3084, 0, 0, 1276, 492, 69, 0, 0, 0, 0, 0, 1293, 2537, 10, 2343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2739, 34, 5, 129, 10, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 14, 66, 0, 127, 11, 0, 3, 0, 4017, 0, 0, 0, 0, 0, 0, 0, 24, 226, 0, 3903, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 0, 0, 3, 16, 4, 0, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 0, 0, 1, 10, 19, 16, 18, 31, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3984, 4959, 6183, 2923, 4238, 4165, 2212, 4104] avg=4096.0000 max=6183 max_rank=2 min=2212 min_rank=6 imbalance_ratio=1.509521 max_load_reduction=26585 max_load_reduction_pct=0.811310 moved_to_echo=14719 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[17, 0, 9, 0, 0, 0, 0, 78, 3038, 824, 61, 0, 0, 0, 0, 0, 0, 0, 1744, 0, 0, 423, 0, 0, 555, 487, 69, 0, 0, 0, 0, 0, 1427, 1105, 11, 1462, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2716, 0, 0, 1480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7, 0, 50, 8, 0, 8, 54, 4009, 0, 0, 0, 0, 0, 0, 0, 423, 61, 0, 3854, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 0, 0, 4, 18, 3, 0, 3669, 0, 0, 0, 0, 0, 0, 0, 0, 3940, 0, 0, 2, 10, 18, 16, 10, 31, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4027, 3278, 4026, 4196, 4138, 4344, 4732, 4027] avg=4096.0000 max=4732 max_rank=6 min=3278 min_rank=1 imbalance_ratio=1.155273 max_load_reduction=28036 max_load_reduction_pct=0.855591 moved_to_echo=17241 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 31, 0, 3043, 834, 62, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 1624, 0, 0, 1348, 493, 71, 0, 0, 0, 0, 0, 659, 2679, 0, 1558, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 2748, 0, 0, 1735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4016, 0, 0, 0, 0, 0, 0, 0, 1, 264, 0, 3986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 0, 0, 0, 0, 0, 0, 2349, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3970, 3621, 4910, 4483, 4016, 4251, 3389, 4128] avg=4096.0000 max=4910 max_rank=2 min=3389 min_rank=6 imbalance_ratio=1.198730 max_load_reduction=27858 max_load_reduction_pct=0.850159 moved_to_echo=15784 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6, 0, 23, 0, 0, 0, 3039, 825, 63, 0, 0, 0, 0, 0, 0, 0, 6, 196, 0, 327, 0, 1, 775, 490, 75, 0, 0, 0, 0, 0, 344, 1527, 13, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2698, 29, 0, 4054, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 24, 6, 123, 2, 0, 4, 0, 4014, 0, 0, 0, 0, 0, 0, 0, 3, 45, 0, 3944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 0, 0, 29, 24, 19, 0, 3845, 0, 0, 0, 0, 0, 0, 0, 0, 3923, 0, 21, 4, 0, 30, 17, 17, 36, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3956, 1870, 2925, 6787, 4242, 3992, 4948, 4048] avg=4096.0000 max=6787 max_rank=3 min=1870 min_rank=1 imbalance_ratio=1.656982 max_load_reduction=25981 max_load_reduction_pct=0.792877 moved_to_echo=17230 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 44, 0, 0, 3049, 838, 62, 0, 0, 0, 0, 0, 0, 0, 150, 0, 0, 3089, 0, 0, 1077, 495, 71, 0, 0, 0, 0, 0, 263, 2142, 0, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2760, 0, 0, 1697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 0, 0, 0, 0, 0, 0, 38, 2, 0, 3993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 0, 0, 0, 0, 0, 0, 2757, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3993, 4882, 3460, 4457, 4035, 4033, 3799, 4109] avg=4096.0000 max=4882 max_rank=1 min=3460 min_rank=2 imbalance_ratio=1.191895 max_load_reduction=27886 max_load_reduction_pct=0.851013 moved_to_echo=15976 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 2988, 836, 62, 0, 0, 0, 0, 0, 0, 0, 903, 0, 1, 3381, 0, 0, 941, 495, 71, 0, 0, 0, 0, 0, 255, 1873, 0, 431, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2756, 0, 0, 1117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 0, 0, 0, 0, 0, 0, 3479, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3887, 5792, 2562, 3873, 4032, 3994, 4500, 4128] avg=4096.0000 max=5792 max_rank=1 min=2562 min_rank=2 imbalance_ratio=1.414062 max_load_reduction=26976 max_load_reduction_pct=0.823242 moved_to_echo=16654 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 0, 128, 0, 128, 1536, 128, 128, 24064, 6784, 512, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[6016, 15872, 128, 11008, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 0, 3200, 256, 128, 17024, 0, 128, 8064, 3968, 640, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[22016, 128, 128, 10880, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 0, 0, 128, 128, 128, 0, 24576, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[512, 640, 0, 31744, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 384, 128, 0, 128, 128, 32256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[32384, 0, 128, 128, 128, 128, 128, 128, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 502, 4, 23, 0, 375, 0, 0, 3759, 0, 0, 3139, 0, 0, 1, 0, 0, 88, 0, 177, 0, 0, 0, 0, 3993, 3497, 2510, 0, 0, 0, 0, 3845, 0, 0, 0, 0, 0, 0, 0, 938, 0, 0, 0, 0, 2688, 0, 0, 0, 4096, 0, 0, 217, 0, 0, 1840, 0, 0, 1076, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 2697, 0, 5, 0, 1935, 0, 0, 3958, 0, 0, 2819, 0, 0, 0, 0, 0, 27, 0, 64, 0, 0, 0, 0, 4044, 2542, 2250, 0, 0, 0, 0, 2477, 0, 0, 0, 0, 0, 0, 0, 185, 0, 0, 0, 0, 4023, 0, 0, 0, 4096, 0, 0, 37, 0, 0, 1598, 0, 0, 11, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 2137, 44, 0, 0, 1117, 30, 15, 4018, 0, 0, 2040, 0, 51, 0, 0, 0, 0, 0, 864, 15, 0, 87, 8, 4047, 2747, 1963, 0, 0, 1, 34, 1964, 6, 0, 8, 0, 0, 0, 0, 105, 0, 16, 15, 21, 3918, 0, 24, 18, 4015, 0, 0, 111, 0, 0, 1388, 0, 0, 1828, 1, 0, 12, 15, 4, 81] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 34, 96, 0, 0, 754, 0, 21, 3724, 0, 0, 848, 0, 31, 0, 25, 2, 0, 0, 40, 0, 0, 1, 3, 3849, 3807, 3999, 0, 0, 0, 15, 3389, 5, 0, 28, 1, 0, 0, 0, 33, 0, 16, 19, 19, 3733, 0, 0, 4, 4063, 0, 531, 77, 3, 0, 3421, 0, 0, 67, 21, 0, 0, 14, 75, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 1387, 0, 0, 0, 3117, 0, 0, 4088, 4, 0, 1254, 0, 0, 0, 0, 0, 4, 0, 184, 0, 0, 0, 0, 4096, 3650, 3731, 0, 0, 0, 0, 2554, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4, 13, 0, 0, 353, 0, 0, 85, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 1209, 0, 1, 0, 922, 0, 0, 4077, 0, 0, 2167, 0, 0, 0, 0, 0, 0, 0, 31, 0, 7, 0, 0, 4084, 1941, 3227, 0, 0, 0, 0, 1987, 0, 0, 0, 0, 0, 0, 0, 148, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 9, 0, 0, 2823, 0, 0, 1945, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 294, 0, 0, 0, 1752, 0, 0, 3994, 18, 0, 3003, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 4095, 1905, 3365, 0, 0, 0, 0, 2767, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 4048, 0, 0, 0, 4096, 0, 1, 49, 0, 0, 1440, 0, 0, 1820, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[4, 1536, 0, 0, 0, 2436, 0, 21, 3806, 73, 0, 282, 0, 0, 0, 4, 0, 0, 0, 37, 0, 0, 19, 6, 3905, 3118, 3857, 0, 0, 4, 0, 1543, 0, 0, 0, 14, 0, 0, 0, 813, 0, 30, 25, 27, 4016, 0, 5, 7, 4055, 0, 324, 0, 0, 0, 2037, 0, 15, 562, 0, 0, 63, 25, 55, 44] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 2697, 0, 5, 0, 1935, 0, 0, 998, 149, 0, 0, 0, 0, 0, 0, 2143, 0, 0, 2819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 64, 0, 0, 0, 0, 3907, 0, 0, 0, 0, 0, 0, 0, 0, 1336, 0, 0, 0, 0, 0, 2477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 2250, 552, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 0, 0, 0, 208, 0, 0, 0, 0, 0, 0, 0, 2072, 0, 0, 37, 0, 0, 1598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 1666, 1472, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5784, 4962, 3998, 3813, 3124, 4231, 3707, 3149] avg=4096.0000 max=5784 max_rank=0 min=3124 min_rank=4 imbalance_ratio=1.412109 max_load_reduction=26984 max_load_reduction_pct=0.823486 moved_to_echo=14211 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 502, 4, 23, 0, 375, 0, 0, 1373, 141, 0, 0, 0, 0, 0, 0, 2036, 0, 0, 3139, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 177, 0, 0, 0, 0, 3858, 0, 0, 0, 0, 0, 0, 0, 0, 1837, 0, 0, 0, 0, 0, 3845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 938, 2510, 552, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2688, 0, 0, 0, 287, 0, 0, 0, 0, 0, 0, 0, 2072, 0, 0, 217, 0, 0, 1840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1076, 0, 0, 0, 0, 0, 0, 1582, 1472, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2418, 5176, 4123, 5682, 4135, 2975, 4129, 4130] avg=4096.0000 max=5682 max_rank=3 min=2418 min_rank=0 imbalance_ratio=1.387207 max_load_reduction=27086 max_load_reduction_pct=0.826599 moved_to_echo=15369 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1387, 0, 0, 0, 3117, 0, 0, 1433, 154, 0, 0, 0, 0, 0, 0, 2214, 4, 0, 1254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 184, 0, 0, 0, 0, 3957, 0, 0, 0, 0, 0, 0, 0, 0, 1918, 0, 0, 0, 0, 0, 2554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 3731, 552, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 299, 0, 0, 0, 0, 0, 0, 0, 2072, 0, 4, 13, 0, 0, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 1720, 1472, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6091, 3472, 4145, 4472, 4474, 4395, 2442, 3277] avg=4096.0000 max=6091 max_rank=0 min=2442 min_rank=6 imbalance_ratio=1.487061 max_load_reduction=26677 max_load_reduction_pct=0.814117 moved_to_echo=14588 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 2137, 44, 0, 0, 1117, 30, 15, 1078, 151, 0, 0, 0, 0, 0, 0, 2176, 0, 0, 2040, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 864, 15, 0, 87, 8, 3911, 0, 0, 0, 0, 0, 0, 0, 0, 1444, 0, 0, 0, 1, 34, 1964, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 8, 0, 0, 0, 0, 105, 1963, 541, 136, 0, 0, 0, 0, 0, 0, 16, 15, 21, 3918, 0, 24, 18, 225, 0, 0, 0, 0, 0, 0, 0, 2032, 0, 0, 111, 0, 0, 1388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1828, 1, 0, 12, 15, 4, 81, 1691, 1442, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4572, 4267, 4885, 3443, 2759, 4237, 3531, 5074] avg=4096.0000 max=5074 max_rank=7 min=2759 min_rank=4 imbalance_ratio=1.238770 max_load_reduction=27694 max_load_reduction_pct=0.845154 moved_to_echo=15601 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 34, 96, 0, 0, 754, 0, 21, 1494, 140, 0, 0, 0, 0, 0, 0, 2017, 0, 0, 848, 0, 31, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 40, 0, 0, 1, 3, 3718, 0, 0, 0, 0, 0, 0, 0, 0, 2001, 0, 0, 0, 0, 15, 3389, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 28, 1, 0, 0, 0, 33, 3999, 548, 131, 0, 0, 0, 0, 0, 0, 16, 19, 19, 3733, 0, 0, 4, 312, 0, 0, 0, 0, 0, 0, 0, 2055, 0, 531, 77, 3, 0, 3421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 21, 0, 0, 14, 75, 0, 1567, 1460, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2539, 2921, 3764, 5405, 4745, 4103, 6087, 3204] avg=4096.0000 max=6087 max_rank=6 min=2539 min_rank=0 imbalance_ratio=1.486084 max_load_reduction=26681 max_load_reduction_pct=0.814240 moved_to_echo=18139 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1209, 0, 1, 0, 922, 0, 0, 762, 154, 0, 0, 0, 0, 0, 0, 2207, 0, 0, 2167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 7, 0, 0, 3946, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 0, 0, 0, 0, 0, 1987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 3227, 552, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 159, 0, 0, 0, 0, 0, 0, 0, 2072, 0, 0, 9, 0, 0, 2823, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1945, 0, 0, 0, 0, 0, 0, 1716, 1472, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3048, 4374, 3984, 3007, 4065, 4253, 4904, 5133] avg=4096.0000 max=5133 max_rank=7 min=3007 min_rank=3 imbalance_ratio=1.253174 max_load_reduction=27635 max_load_reduction_pct=0.843353 moved_to_echo=18355 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 294, 0, 0, 0, 1752, 0, 0, 748, 150, 0, 0, 0, 0, 0, 0, 2163, 18, 0, 3003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 3956, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 0, 0, 0, 0, 0, 2767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 3365, 552, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 0, 0, 156, 0, 0, 0, 0, 0, 0, 0, 2072, 0, 1, 49, 0, 0, 1440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1820, 0, 0, 0, 0, 0, 0, 1681, 1472, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2944, 5184, 4034, 3768, 4099, 4204, 3562, 4973] avg=4096.0000 max=5184 max_rank=1 min=2944 min_rank=0 imbalance_ratio=1.265625 max_load_reduction=27584 max_load_reduction_pct=0.841797 moved_to_echo=16838 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4, 1536, 0, 0, 0, 2436, 0, 21, 1227, 148, 0, 0, 0, 0, 0, 0, 2053, 73, 0, 282, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 19, 6, 3773, 0, 0, 0, 0, 0, 0, 0, 0, 1631, 0, 0, 0, 4, 0, 1543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 813, 3857, 551, 132, 0, 0, 0, 0, 0, 0, 30, 25, 27, 4016, 0, 5, 7, 260, 0, 0, 0, 0, 0, 0, 0, 2045, 0, 324, 0, 0, 0, 2037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 562, 0, 0, 63, 25, 55, 44, 1605, 1459, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5372, 2412, 3835, 3178, 5367, 4370, 4406, 3828] avg=4096.0000 max=5372 max_rank=0 min=2412 min_rank=1 imbalance_ratio=1.311523 max_load_reduction=27396 max_load_reduction_pct=0.836060 moved_to_echo=17971 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 7424, 128, 0, 128, 128, 256, 128, 13312, 11776, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 12288, 0, 0, 0, 128, 128, 20608, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 9856, 256, 128, 0, 12416, 128, 128, 9216, 1280, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[17024, 128, 0, 15616, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 0, 0, 0, 2432, 24960, 4480, 1152, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[16512, 0, 896, 640, 128, 0, 14976, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 30720, 0, 128, 128, 1920, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 0, 1536, 128, 128, 128, 128, 31104, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 4008, 0, 4084, 931, 0, 0, 4096, 0, 3635, 0, 0, 0, 0, 3885, 0, 0, 4041, 940, 0, 0, 0, 0, 12, 0, 41, 0, 2409, 0, 93, 772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3687, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 128, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3950, 0, 4022, 1294, 0, 0, 4035, 20, 3507, 18, 0, 0, 0, 2615, 14, 0, 4078, 2781, 0, 0, 0, 0, 56, 0, 5, 12, 279, 38, 912, 772, 5, 11, 0, 0, 0, 67, 0, 0, 0, 3794, 0, 14, 22, 59, 0, 15, 26, 0, 22, 0, 28, 277, 5, 3, 2, 0, 0, 0, 10, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 4096, 1812, 0, 0, 4096, 0, 3956, 0, 1, 0, 0, 1123, 0, 0, 4048, 1105, 0, 0, 8, 0, 0, 0, 0, 0, 3241, 0, 472, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3648, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3994, 0, 4046, 653, 0, 0, 4067, 8, 3062, 16, 0, 0, 0, 3011, 0, 0, 4080, 2462, 0, 0, 71, 0, 194, 0, 86, 16, 193, 29, 500, 2179, 3, 0, 3, 0, 0, 88, 0, 0, 0, 3660, 0, 73, 20, 125, 0, 17, 21, 0, 21, 0, 0, 1, 0, 19, 7, 0, 0, 0, 43, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[40, 0, 0, 0, 4078, 0, 4096, 3524, 0, 0, 4096, 0, 3921, 0, 0, 0, 0, 3363, 0, 0, 4083, 2043, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 574, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 4096, 3733, 0, 0, 4096, 0, 4092, 0, 0, 0, 0, 640, 0, 0, 4096, 301, 0, 0, 1, 0, 0, 0, 0, 0, 2665, 0, 13, 846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 18, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3814, 0, 4067, 2273, 0, 0, 4057, 26, 3151, 25, 0, 0, 0, 1073, 27, 0, 4074, 3687, 0, 0, 1409, 0, 0, 0, 0, 14, 781, 0, 15, 10, 1, 0, 28, 0, 0, 37, 0, 0, 0, 4049, 0, 21, 33, 10, 0, 15, 35, 0, 29, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 4096, 1238, 0, 0, 4096, 0, 3968, 0, 0, 0, 0, 2774, 0, 0, 4096, 847, 0, 0, 35, 0, 0, 0, 0, 0, 3599, 0, 2, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3714, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 2155, 0, 0, 931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 423, 0, 3635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3885, 0, 0, 10, 940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 41, 0, 2409, 0, 1478, 717, 0, 0, 0, 0, 0, 0, 93, 772, 0, 0, 0, 0, 0, 0, 2956, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3687, 0, 0, 0, 1, 375, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3988, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 128, 0, 4072, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3088, 4058, 4835, 4657, 3864, 4063, 4001, 4202] avg=4096.0000 max=4835 max_rank=2 min=3088 min_rank=0 imbalance_ratio=1.180420 max_load_reduction=27933 max_load_reduction_pct=0.852448 moved_to_echo=16130 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2202, 0, 0, 1812, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 423, 0, 3956, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1123, 0, 0, 11, 1105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 3241, 0, 1510, 717, 0, 0, 0, 0, 0, 0, 472, 1065, 0, 0, 0, 0, 0, 0, 2956, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3648, 0, 2, 0, 0, 383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3994, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4014, 4380, 2239, 5476, 4536, 4033, 4005, 4085] avg=4096.0000 max=5476 max_rank=3 min=2239 min_rank=2 imbalance_ratio=1.336914 max_load_reduction=27292 max_load_reduction_pct=0.832886 moved_to_echo=16659 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2203, 0, 0, 3733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 423, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 640, 0, 0, 11, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2665, 0, 1510, 717, 0, 0, 0, 0, 0, 0, 13, 846, 0, 0, 0, 0, 0, 0, 2956, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 18, 0, 4085, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5936, 4515, 952, 4893, 3858, 4457, 4053, 4104] avg=4096.0000 max=5936 max_rank=0 min=952 min_rank=2 imbalance_ratio=1.449219 max_load_reduction=26832 max_load_reduction_pct=0.818848 moved_to_echo=16472 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2148, 0, 0, 653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 420, 8, 3062, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3011, 0, 0, 11, 2462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 194, 0, 86, 16, 193, 29, 1472, 712, 0, 0, 0, 0, 0, 0, 500, 2179, 3, 0, 3, 0, 0, 88, 2935, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3660, 0, 73, 20, 125, 374, 0, 0, 0, 0, 0, 0, 0, 0, 17, 21, 0, 21, 0, 0, 1, 4026, 11, 0, 0, 0, 0, 0, 0, 0, 19, 7, 0, 0, 0, 43, 0, 4035, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2801, 3506, 5484, 2773, 5751, 4252, 4097, 4104] avg=4096.0000 max=5751 max_rank=4 min=2773 min_rank=3 imbalance_ratio=1.404053 max_load_reduction=27017 max_load_reduction_pct=0.824493 moved_to_echo=18204 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2124, 0, 0, 1294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 416, 20, 3507, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2615, 14, 0, 11, 2781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 5, 12, 279, 38, 1456, 707, 0, 0, 0, 0, 0, 0, 912, 772, 5, 11, 0, 0, 0, 67, 2912, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3794, 0, 14, 22, 59, 370, 0, 0, 0, 0, 0, 0, 0, 0, 15, 26, 0, 22, 0, 28, 277, 4024, 10, 0, 0, 0, 0, 0, 0, 5, 3, 2, 0, 0, 0, 10, 0, 4012, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3418, 3961, 5421, 2553, 4722, 4259, 4402, 4032] avg=4096.0000 max=5421 max_rank=2 min=2553 min_rank=3 imbalance_ratio=1.323486 max_load_reduction=27347 max_load_reduction_pct=0.834564 moved_to_echo=17415 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[40, 0, 0, 0, 2193, 0, 0, 3524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 423, 0, 3921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3363, 0, 0, 11, 2043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 1503, 717, 0, 0, 0, 0, 0, 0, 574, 188, 0, 0, 0, 0, 0, 0, 2956, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2749, 0, 0, 0, 0, 382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4085, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5757, 4344, 5417, 2226, 3761, 3131, 4040, 4092] avg=4096.0000 max=5757 max_rank=0 min=2226 min_rank=3 imbalance_ratio=1.405518 max_load_reduction=27011 max_load_reduction_pct=0.824310 moved_to_echo=15024 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2041, 0, 0, 2273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 411, 26, 3151, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1073, 27, 0, 1, 3687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1409, 0, 0, 0, 0, 14, 781, 0, 1411, 715, 0, 0, 0, 0, 0, 0, 15, 10, 1, 0, 28, 0, 0, 37, 2931, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 21, 33, 10, 362, 0, 0, 0, 0, 0, 0, 0, 0, 15, 35, 0, 29, 0, 0, 0, 4024, 12, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4055, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4314, 3613, 4788, 4330, 3071, 4475, 4115, 4062] avg=4096.0000 max=4788 max_rank=2 min=3071 min_rank=4 imbalance_ratio=1.168945 max_load_reduction=27980 max_load_reduction_pct=0.853882 moved_to_echo=15723 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2202, 0, 0, 1238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 423, 0, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2774, 0, 0, 11, 847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 3599, 0, 1509, 717, 0, 0, 0, 0, 0, 0, 2, 204, 0, 0, 0, 0, 0, 0, 2956, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3714, 0, 0, 0, 1, 383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4042, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4085, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3440, 4391, 3632, 5860, 3205, 4098, 4055, 4087] avg=4096.0000 max=5860 max_rank=3 min=3205 min_rank=4 imbalance_ratio=1.430664 max_load_reduction=26908 max_load_reduction_pct=0.821167 moved_to_echo=15445 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1536, 0, 384, 0, 256, 128, 13184, 128, 11904, 5760, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 0, 0, 0, 17280, 0, 0, 15488, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 18560, 128, 0, 128, 14208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[2688, 6144, 128, 128, 128, 0, 0, 256, 23680, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 29440, 0, 128, 128, 256, 3072, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 3456, 128, 29312, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 0, 0, 256, 0, 32640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 128, 0, 128, 384, 32256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[35, 2, 0, 2057, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4017, 0, 0, 0, 0, 1647, 0, 0, 2, 0, 0, 0, 0, 3579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 921, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 25, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[34, 480, 0, 1060, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 121, 3996, 0, 0, 0, 0, 528, 0, 0, 15, 0, 2, 0, 0, 2789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3404, 0, 3749, 0, 0, 4096, 0, 0, 0, 0, 13, 0, 0, 0, 4084, 0, 0, 4096, 0, 173, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[9, 407, 0, 1614, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 27, 4015, 0, 0, 0, 0, 224, 0, 0, 11, 0, 0, 0, 0, 3651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 2399, 0, 0, 4096, 0, 1, 0, 0, 9, 0, 0, 0, 4081, 0, 0, 4094, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 2267, 18, 3761, 0, 51, 0, 3, 22, 6, 9, 2, 2, 0, 0, 27, 0, 3992, 36, 0, 4054, 0, 161, 0, 1, 2082, 0, 19, 0, 0, 0, 0, 0, 1214, 11, 0, 0, 2, 0, 0, 5, 0, 0, 2553, 0, 1849, 37, 4, 4044, 0, 0, 0, 0, 0, 0, 9, 0, 4066, 0, 0, 2431, 0, 22, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 77, 15, 3432, 2, 18, 0, 0, 27, 289, 9, 5, 0, 0, 1, 25, 0, 4008, 6, 1, 4075, 0, 238, 0, 7, 1377, 0, 21, 116, 0, 115, 0, 12, 716, 0, 0, 0, 104, 0, 0, 8, 0, 0, 2062, 0, 3898, 47, 3, 4055, 0, 0, 0, 0, 0, 0, 11, 0, 4072, 0, 0, 3728, 0, 187, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[38, 1007, 0, 3673, 0, 0, 0, 0, 0, 222, 0, 0, 0, 2, 0, 0, 0, 4094, 0, 31, 4006, 0, 0, 0, 0, 2558, 0, 0, 28, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2384, 3, 2398, 0, 0, 4083, 0, 5, 0, 0, 0, 0, 0, 0, 4077, 3, 0, 4082, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[15, 18, 0, 740, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 4077, 0, 0, 0, 0, 694, 0, 0, 0, 0, 0, 0, 0, 3866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2932, 0, 3783, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4086, 0, 265, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 191, 15, 3145, 5, 12, 0, 0, 32, 113, 4, 0, 17, 0, 0, 23, 0, 4004, 35, 0, 4058, 0, 355, 0, 6, 994, 0, 20, 24, 0, 111, 0, 0, 837, 0, 0, 0, 0, 0, 0, 40, 0, 3, 3365, 0, 3470, 26, 0, 4014, 0, 0, 0, 0, 0, 0, 22, 0, 4070, 0, 0, 3722, 0, 28, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[9, 407, 0, 1614, 0, 0, 0, 0, 761, 52, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 3949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 224, 0, 0, 11, 0, 0, 0, 2697, 74, 0, 0, 0, 0, 0, 0, 0, 3651, 0, 0, 0, 0, 0, 0, 1168, 614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 9, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 216, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2843, 4023, 3976, 3006, 5433, 5057, 4120, 4310] avg=4096.0000 max=5433 max_rank=4 min=2843 min_rank=0 imbalance_ratio=1.326416 max_load_reduction=27335 max_load_reduction_pct=0.834198 moved_to_echo=18920 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[35, 2, 0, 2057, 0, 0, 0, 0, 1300, 52, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1647, 0, 0, 2, 0, 0, 0, 2707, 74, 0, 0, 0, 0, 0, 0, 0, 3579, 0, 0, 0, 0, 0, 0, 1173, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 921, 0, 1748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 216, 0, 0, 4096, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3446, 4024, 3952, 4430, 5800, 2669, 4110, 4337] avg=4096.0000 max=5800 max_rank=4 min=2669 min_rank=5 imbalance_ratio=1.416016 max_load_reduction=26968 max_load_reduction_pct=0.822998 moved_to_echo=16916 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[38, 1007, 0, 3673, 0, 0, 0, 0, 761, 52, 0, 0, 0, 0, 0, 0, 0, 222, 0, 0, 0, 2, 0, 0, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 3940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2558, 0, 0, 28, 0, 0, 0, 2695, 74, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 1167, 613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2384, 3, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 5, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 215, 3, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5531, 4244, 3971, 5355, 1854, 3411, 4102, 4300] avg=4096.0000 max=5531 max_rank=0 min=1854 min_rank=4 imbalance_ratio=1.350342 max_load_reduction=27237 max_load_reduction_pct=0.831207 moved_to_echo=13667 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 77, 15, 3432, 2, 18, 0, 0, 1237, 53, 0, 0, 0, 0, 0, 0, 27, 289, 9, 5, 0, 0, 1, 25, 3935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 4008, 0, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1377, 0, 21, 116, 0, 115, 0, 2691, 73, 0, 0, 0, 0, 0, 0, 12, 716, 0, 0, 0, 104, 0, 0, 1166, 997, 0, 0, 0, 0, 0, 0, 8, 0, 0, 2062, 0, 1664, 47, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 0, 0, 0, 0, 0, 11, 14, 0, 0, 0, 0, 0, 0, 0, 0, 215, 0, 0, 3728, 0, 187, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4834, 4291, 4253, 4400, 2995, 3784, 4080, 4131] avg=4096.0000 max=4834 max_rank=0 min=2995 min_rank=4 imbalance_ratio=1.180176 max_load_reduction=27934 max_load_reduction_pct=0.852478 moved_to_echo=14990 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[34, 480, 0, 1060, 0, 0, 0, 0, 1190, 52, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, 3930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 0, 0, 15, 0, 2, 0, 2699, 74, 0, 0, 0, 0, 0, 0, 0, 2789, 0, 0, 0, 0, 0, 0, 1169, 959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3404, 0, 1600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 13, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 216, 0, 0, 4096, 0, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2816, 4054, 4051, 3318, 4917, 5004, 4123, 4485] avg=4096.0000 max=5004 max_rank=5 min=2816 min_rank=0 imbalance_ratio=1.221680 max_load_reduction=27764 max_load_reduction_pct=0.847290 moved_to_echo=18529 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 2267, 18, 3761, 0, 51, 0, 3, 587, 53, 0, 0, 0, 0, 0, 0, 22, 6, 9, 2, 2, 0, 0, 27, 3920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 3987, 0, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2082, 0, 19, 0, 0, 0, 0, 2687, 72, 0, 0, 0, 0, 0, 0, 0, 1214, 11, 0, 0, 2, 0, 0, 1164, 473, 0, 0, 0, 0, 0, 0, 5, 0, 0, 2553, 0, 789, 37, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 0, 0, 0, 0, 0, 9, 14, 0, 0, 0, 0, 0, 0, 0, 0, 215, 0, 0, 2431, 0, 22, 8, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6740, 3988, 4184, 4861, 2864, 3388, 4067, 2676] avg=4096.0000 max=6740 max_rank=0 min=2676 min_rank=7 imbalance_ratio=1.645508 max_load_reduction=26028 max_load_reduction_pct=0.794312 moved_to_echo=12995 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[15, 18, 0, 740, 0, 0, 0, 0, 1201, 53, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 694, 0, 0, 0, 0, 0, 0, 2707, 75, 0, 0, 0, 0, 0, 0, 0, 3866, 0, 0, 0, 0, 0, 0, 1173, 968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2932, 0, 1614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 216, 0, 0, 4086, 0, 265, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2027, 4023, 4012, 3476, 6007, 4546, 4110, 4567] avg=4096.0000 max=6007 max_rank=4 min=2027 min_rank=0 imbalance_ratio=1.466553 max_load_reduction=26761 max_load_reduction_pct=0.816681 moved_to_echo=19230 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 191, 15, 3145, 5, 12, 0, 0, 1104, 59, 0, 0, 0, 0, 0, 0, 32, 113, 4, 0, 17, 0, 0, 23, 3932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 3979, 0, 355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 994, 0, 20, 24, 0, 111, 0, 2695, 72, 0, 0, 0, 0, 0, 0, 0, 837, 0, 0, 0, 0, 0, 0, 1170, 891, 0, 0, 0, 0, 0, 0, 40, 0, 3, 3365, 0, 1475, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4014, 0, 0, 0, 0, 0, 0, 22, 20, 0, 0, 0, 0, 0, 0, 0, 0, 205, 0, 0, 3722, 0, 28, 7, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4531, 4121, 4369, 3922, 2898, 4909, 4056, 3962] avg=4096.0000 max=4909 max_rank=5 min=2898 min_rank=4 imbalance_ratio=1.198486 max_load_reduction=27859 max_load_reduction_pct=0.850189 moved_to_echo=15825 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 16768, 128, 0, 0, 128, 0, 0, 9472, 6656, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[256, 4480, 128, 19584, 128, 128, 0, 128, 8192, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 10112, 0, 128, 256, 0, 256, 0, 21632, 640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 128, 21760, 128, 11008, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 1792, 128, 0, 30336, 0, 768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 768, 128, 128, 128, 128, 128, 128, 32000, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 256, 31872, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32640, 0, 128, 0, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3881, 0, 4096, 0, 0, 4086, 3347, 0, 0, 0, 0, 0, 24, 0, 0, 0, 2262, 0, 0, 2, 2, 0, 389, 0, 0, 243, 0, 4047, 4095, 65, 0, 0, 0, 0, 2158, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 4096, 0, 0, 4068, 3435, 0, 0, 0, 0, 0, 86, 0, 0, 0, 3751, 0, 0, 0, 29, 0, 44, 0, 0, 169, 0, 4096, 4096, 210, 0, 0, 0, 0, 514, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4075, 0, 4086, 0, 0, 2773, 2088, 9, 7, 0, 0, 0, 46, 0, 0, 0, 2280, 0, 0, 41, 0, 0, 2593, 0, 0, 407, 0, 4053, 4095, 1061, 0, 0, 0, 0, 990, 0, 0, 0, 0, 0, 0, 0, 79] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[12, 6, 0, 0, 0, 24, 4069, 0, 0, 0, 0, 0, 24, 26, 16, 0, 0, 0, 0, 1, 3, 3571, 21, 4069, 15, 6, 3581, 3110, 0, 0, 24, 20, 0, 434, 3, 36, 17, 3054, 21, 18, 33, 1, 0, 193, 0, 0, 50, 0, 3826, 4071, 1128, 0, 0, 0, 2, 1281, 0, 0, 0, 0, 0, 0, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[16, 21, 0, 118, 0, 27, 4067, 0, 0, 0, 0, 0, 39, 25, 12, 0, 0, 0, 1, 12, 0, 3311, 0, 4071, 0, 5, 1271, 3639, 0, 0, 34, 16, 37, 62, 0, 2, 0, 4045, 20, 19, 93, 0, 0, 37, 0, 0, 1250, 0, 3776, 4062, 81, 0, 0, 0, 0, 2593, 0, 0, 0, 0, 4, 0, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4049, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3129, 0, 4091, 0, 0, 3617, 3601, 0, 0, 0, 0, 0, 421, 0, 0, 0, 2933, 0, 0, 0, 29, 0, 248, 0, 0, 389, 0, 4096, 4096, 307, 0, 0, 0, 0, 1749, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3994, 0, 4096, 0, 0, 753, 3893, 0, 0, 0, 0, 0, 121, 0, 0, 0, 3754, 0, 0, 10, 0, 0, 2, 0, 0, 2059, 0, 4085, 4096, 271, 0, 0, 0, 0, 1519, 0, 0, 0, 0, 0, 0, 0, 22] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[24, 12, 0, 0, 0, 32, 4072, 20, 0, 0, 0, 0, 1, 37, 20, 0, 0, 0, 0, 12, 3, 3999, 0, 4061, 84, 4, 3921, 1173, 0, 0, 32, 20, 48, 81, 0, 8, 3, 3952, 14, 26, 2, 0, 0, 7, 0, 0, 317, 0, 3937, 4064, 2416, 0, 0, 0, 0, 349, 0, 0, 0, 0, 17, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3761, 349, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3881, 0, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 1162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 2262, 0, 0, 639, 18, 0, 0, 0, 0, 0, 0, 2, 2, 0, 389, 0, 0, 243, 0, 1836, 1369, 0, 0, 0, 0, 0, 0, 2039, 0, 65, 0, 0, 0, 0, 2158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4053, 4114, 4216, 5248, 2943, 3841, 4262, 4091] avg=4096.0000 max=5248 max_rank=3 min=2943 min_rank=4 imbalance_ratio=1.281250 max_load_reduction=27520 max_load_reduction_pct=0.839844 moved_to_echo=15137 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3761, 358, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 1193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 3751, 0, 0, 646, 18, 0, 0, 0, 0, 0, 0, 0, 29, 0, 44, 0, 0, 169, 0, 1884, 1386, 0, 0, 0, 0, 0, 0, 2064, 0, 210, 0, 0, 0, 0, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4123, 4415, 5261, 4501, 3512, 2788, 4092] avg=4096.0000 max=5261 max_rank=3 min=2788 min_rank=6 imbalance_ratio=1.284424 max_load_reduction=27507 max_load_reduction_pct=0.839447 moved_to_echo=14893 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3752, 218, 4, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4075, 0, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2773, 725, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 2280, 0, 0, 640, 18, 0, 0, 0, 0, 0, 0, 41, 0, 0, 2593, 0, 0, 407, 0, 1145, 1371, 0, 0, 0, 0, 0, 0, 2042, 0, 1061, 0, 0, 0, 0, 990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 4091, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 3974, 4416, 3514, 2984, 5557, 4093, 4170] avg=4096.0000 max=5557 max_rank=5 min=2984 min_rank=4 imbalance_ratio=1.356689 max_load_reduction=27211 max_load_reduction_pct=0.830414 moved_to_echo=16804 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[16, 21, 0, 118, 0, 27, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 25, 12, 0, 3738, 380, 4, 0, 0, 0, 0, 0, 0, 0, 1, 12, 0, 3311, 0, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1271, 1263, 0, 0, 34, 16, 0, 0, 0, 0, 0, 0, 0, 0, 37, 62, 0, 2, 0, 4045, 20, 19, 596, 18, 0, 0, 0, 0, 0, 0, 93, 0, 0, 37, 0, 0, 1250, 0, 1996, 1277, 0, 0, 0, 0, 0, 0, 1903, 0, 81, 0, 0, 0, 0, 2593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 2, 4058, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4231, 4198, 3657, 2589, 4799, 4653, 4577, 4064] avg=4096.0000 max=4799 max_rank=4 min=2589 min_rank=3 imbalance_ratio=1.171631 max_load_reduction=27969 max_load_reduction_pct=0.853546 moved_to_echo=18093 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[12, 6, 0, 0, 0, 24, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 26, 16, 0, 3736, 324, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 3571, 21, 333, 0, 0, 0, 0, 0, 0, 0, 0, 15, 6, 3581, 1080, 0, 0, 24, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 434, 3, 36, 17, 3054, 21, 18, 604, 18, 0, 0, 0, 0, 0, 0, 33, 1, 0, 193, 0, 0, 50, 0, 1706, 1294, 0, 0, 0, 0, 0, 0, 1928, 0, 1128, 0, 0, 0, 2, 1281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4067, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4130, 3929, 4726, 4205, 3277, 4339, 4069] avg=4096.0000 max=4726 max_rank=3 min=3277 min_rank=5 imbalance_ratio=1.153809 max_load_reduction=28042 max_load_reduction_pct=0.855774 moved_to_echo=15890 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4031, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3756, 376, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3129, 0, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3617, 1249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 421, 0, 0, 0, 2933, 0, 0, 646, 18, 0, 0, 0, 0, 0, 0, 0, 29, 0, 248, 0, 0, 389, 0, 1976, 1386, 0, 0, 0, 0, 0, 0, 2064, 0, 307, 0, 0, 0, 0, 1749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4044, 4136, 3464, 4866, 4018, 4028, 4120, 4092] avg=4096.0000 max=4866 max_rank=3 min=3464 min_rank=2 imbalance_ratio=1.187988 max_load_reduction=27902 max_load_reduction_pct=0.851501 moved_to_echo=16258 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[24, 12, 0, 0, 0, 32, 4048, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 37, 20, 0, 3733, 126, 5, 0, 0, 0, 0, 0, 0, 0, 0, 12, 3, 3999, 0, 328, 0, 0, 0, 0, 0, 0, 0, 0, 84, 4, 3921, 399, 0, 0, 32, 20, 0, 0, 0, 0, 0, 0, 0, 0, 48, 81, 0, 8, 3, 3952, 14, 26, 624, 24, 0, 0, 0, 0, 0, 0, 2, 0, 0, 7, 0, 0, 317, 0, 648, 1337, 0, 0, 0, 0, 0, 0, 1976, 0, 2416, 0, 0, 0, 0, 349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4136, 3922, 4342, 4460, 4780, 2311, 4741, 4076] avg=4096.0000 max=4780 max_rank=4 min=2311 min_rank=5 imbalance_ratio=1.166992 max_load_reduction=27988 max_load_reduction_pct=0.854126 moved_to_echo=15908 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3761, 406, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3994, 0, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 753, 1351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, 0, 0, 0, 3754, 0, 0, 645, 18, 0, 0, 0, 0, 0, 0, 10, 0, 0, 2, 0, 0, 2059, 0, 2136, 1382, 0, 0, 0, 0, 0, 0, 2058, 0, 271, 0, 0, 0, 0, 1519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4092, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4171, 4329, 2104, 4538, 5589, 3848, 4114] avg=4096.0000 max=5589 max_rank=5 min=2104 min_rank=3 imbalance_ratio=1.364502 max_load_reduction=27179 max_load_reduction_pct=0.829437 moved_to_echo=18089 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 128, 0, 3584, 0, 0, 4992, 0, 13440, 10880, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[16128, 0, 5632, 0, 0, 0, 128, 11264, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 24192, 8448, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 1280, 128, 128, 128, 26112, 128, 128, 5120, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 128, 128, 0, 30080, 2560, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 128, 30080, 128, 2688, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 128, 0, 128, 32512, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 3957, 0, 0, 2, 0, 0, 1547, 0, 1, 0, 0, 0, 4096, 0, 0, 170, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4078, 4096, 0, 0, 0, 2378, 0, 0, 0, 0, 0, 0, 3441, 0, 4095, 2089, 0, 0, 0, 0, 0, 0, 0, 2815, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 83, 3992, 0, 0, 25, 4, 0, 7, 0, 0, 0, 0, 0, 4082, 0, 0, 701, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3773, 4089, 0, 0, 0, 3370, 0, 0, 0, 1, 0, 0, 3365, 0, 4095, 3045, 0, 0, 0, 0, 0, 0, 0, 2134, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1139, 2192, 0, 8, 3, 0, 10, 18, 0, 27, 0, 0, 0, 4070, 0, 13, 1547, 15, 0, 0, 0, 0, 0, 0, 0, 24, 7, 4056, 4067, 0, 0, 31, 980, 230, 0, 5, 0, 0, 13, 3501, 0, 4071, 3869, 10, 0, 24, 0, 0, 0, 1, 2735, 14, 25, 4, 19, 19, 0, 0, 18, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1070, 3497, 0, 18, 0, 0, 12, 8, 0, 0, 0, 0, 0, 4072, 0, 35, 103, 4, 0, 0, 0, 0, 0, 0, 0, 24, 5, 4071, 4058, 0, 0, 26, 3934, 28, 0, 7, 0, 0, 27, 3263, 0, 4073, 2353, 17, 0, 64, 0, 0, 0, 20, 1902, 0, 25, 3, 0, 20, 0, 0, 26, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4, 4073, 0, 0, 24, 0, 0, 14, 0, 0, 0, 0, 0, 4084, 0, 4, 316, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 4057, 4075, 0, 0, 0, 3442, 0, 0, 0, 0, 0, 0, 3435, 0, 4096, 2107, 0, 0, 0, 0, 0, 0, 0, 3013, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4, 3461, 0, 0, 32, 0, 0, 403, 0, 0, 0, 0, 0, 4096, 0, 0, 231, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4078, 4096, 0, 0, 0, 2348, 0, 0, 0, 1, 0, 0, 3142, 0, 4096, 3148, 0, 0, 0, 0, 0, 0, 0, 3631, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 467, 2336, 0, 4, 0, 0, 13, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 3814, 21, 0, 0, 0, 0, 0, 0, 1, 30, 17, 4061, 4063, 0, 0, 20, 1061, 156, 0, 8, 0, 0, 21, 2948, 0, 4065, 1825, 22, 0, 4, 0, 0, 13, 36, 3600, 0, 31, 31, 3, 24, 0, 0, 7, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 245, 4096, 0, 0, 61, 0, 0, 2, 0, 0, 0, 7, 0, 4084, 0, 20, 404, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 4055, 4063, 0, 0, 0, 3034, 0, 0, 0, 0, 0, 0, 4092, 0, 4082, 446, 0, 0, 14, 0, 0, 0, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 83, 3992, 0, 0, 265, 0, 0, 0, 0, 0, 0, 0, 25, 4, 0, 7, 0, 0, 0, 0, 2620, 1173, 21, 0, 0, 0, 0, 0, 0, 3173, 0, 0, 701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3773, 48, 0, 0, 0, 0, 0, 0, 0, 1469, 0, 0, 0, 3370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1706, 0, 0, 3045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2134, 0, 596, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4340, 3850, 3874, 3823, 4839, 4752, 3216, 4074] avg=4096.0000 max=4839 max_rank=4 min=3216 min_rank=6 imbalance_ratio=1.181396 max_load_reduction=27929 max_load_reduction_pct=0.852325 moved_to_echo=16881 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 3957, 0, 0, 265, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1547, 0, 1, 0, 0, 2625, 1200, 21, 0, 0, 0, 0, 0, 0, 3185, 0, 0, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4078, 48, 0, 0, 0, 0, 0, 0, 0, 1471, 0, 0, 0, 2378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1744, 0, 0, 2089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2815, 0, 598, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4224, 5396, 3355, 4127, 3849, 3833, 3910, 4074] avg=4096.0000 max=5396 max_rank=1 min=3355 min_rank=2 imbalance_ratio=1.317383 max_load_reduction=27372 max_load_reduction_pct=0.835327 moved_to_echo=15666 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1139, 2192, 0, 8, 264, 0, 0, 0, 0, 0, 0, 0, 3, 0, 10, 18, 0, 27, 0, 0, 2606, 1221, 20, 0, 0, 0, 0, 0, 0, 3164, 0, 13, 1547, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 7, 4056, 48, 0, 0, 0, 0, 0, 0, 0, 1461, 0, 0, 31, 980, 230, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1775, 0, 0, 3869, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 1, 2735, 14, 594, 505, 0, 0, 0, 0, 0, 0, 25, 4, 19, 19, 0, 0, 18, 3, 4051, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3603, 3905, 4739, 4135, 2707, 5667, 3873, 4139] avg=4096.0000 max=5667 max_rank=5 min=2707 min_rank=4 imbalance_ratio=1.383545 max_load_reduction=27101 max_load_reduction_pct=0.827057 moved_to_echo=16386 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4, 4073, 0, 0, 265, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 14, 0, 0, 0, 0, 2611, 1198, 21, 0, 0, 0, 0, 0, 0, 3175, 0, 4, 316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 4057, 48, 0, 0, 0, 0, 0, 0, 0, 1464, 0, 0, 0, 3442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1741, 0, 0, 2107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3013, 0, 596, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4342, 3868, 3495, 4129, 4906, 3848, 4105, 4075] avg=4096.0000 max=4906 max_rank=4 min=3495 min_rank=2 imbalance_ratio=1.197754 max_load_reduction=27862 max_load_reduction_pct=0.850281 moved_to_echo=16934 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1070, 3497, 0, 18, 264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 8, 0, 0, 0, 0, 2600, 1138, 21, 0, 0, 0, 0, 0, 0, 3165, 0, 35, 103, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 5, 4071, 48, 0, 0, 0, 0, 0, 0, 0, 1458, 0, 0, 26, 3934, 28, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 1654, 0, 0, 2353, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 20, 1902, 0, 595, 471, 0, 0, 0, 0, 0, 0, 25, 3, 0, 20, 0, 0, 26, 3, 4052, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4849, 3779, 3307, 4148, 5453, 4051, 3052, 4129] avg=4096.0000 max=5453 max_rank=4 min=3052 min_rank=6 imbalance_ratio=1.331299 max_load_reduction=27315 max_load_reduction_pct=0.833588 moved_to_echo=16685 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4, 3461, 0, 0, 265, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 403, 0, 0, 0, 0, 2625, 1096, 20, 0, 0, 0, 0, 0, 0, 3185, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4078, 48, 0, 0, 0, 0, 0, 0, 0, 1471, 0, 0, 0, 2348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1593, 0, 0, 3148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3631, 0, 598, 453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3730, 4176, 3416, 4127, 3819, 4742, 4682, 4076] avg=4096.0000 max=4742 max_rank=5 min=3416 min_rank=2 imbalance_ratio=1.157715 max_load_reduction=28026 max_load_reduction_pct=0.855286 moved_to_echo=17319 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 467, 2336, 0, 4, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 2607, 1032, 21, 0, 0, 0, 0, 0, 0, 3148, 0, 0, 3814, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 30, 17, 4061, 52, 0, 0, 0, 0, 0, 0, 0, 1456, 0, 0, 20, 1061, 156, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 1487, 0, 0, 1825, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 13, 36, 3600, 0, 599, 429, 0, 0, 0, 0, 0, 0, 31, 31, 3, 24, 0, 0, 7, 0, 4044, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3074, 3673, 6983, 4161, 2701, 3355, 4681, 4140] avg=4096.0000 max=6983 max_rank=2 min=2701 min_rank=4 imbalance_ratio=1.704834 max_load_reduction=25785 max_load_reduction_pct=0.786896 moved_to_echo=14877 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 245, 4096, 0, 0, 265, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 2, 0, 0, 0, 7, 2603, 1427, 21, 0, 0, 0, 0, 0, 0, 3175, 0, 20, 404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 4055, 48, 0, 0, 0, 0, 0, 0, 0, 1460, 0, 0, 0, 3034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2074, 0, 0, 446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 4048, 0, 596, 591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4606, 4121, 3599, 4118, 4494, 2520, 5249, 4061] avg=4096.0000 max=5249 max_rank=6 min=2520 min_rank=5 imbalance_ratio=1.281494 max_load_reduction=27519 max_load_reduction_pct=0.839813 moved_to_echo=16324 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 13824, 0, 0, 18944, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[256, 128, 128, 2048, 0, 128, 0, 128, 20992, 9600, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[11776, 0, 0, 128, 20608, 512, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 25472, 0, 128, 7296, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 128, 128, 23936, 128, 4864, 3968, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 3072, 27648, 0, 128, 2176, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 0, 0, 128, 128, 32512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 128, 128, 128, 32256, 512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 3673, 4061, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 40, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 486, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3955, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 3757, 4072, 0, 0, 0, 0, 4, 4043, 0, 41, 0, 0, 4, 3997, 0, 0, 0, 0, 0, 36, 8, 13, 0, 1, 3, 23, 0, 0, 16, 0, 0, 18, 32, 0, 4050, 27, 12, 42, 0, 18, 0, 4070, 0, 23, 6, 0, 16, 0, 0, 748, 0, 0, 5, 6, 0, 0, 0, 0, 0, 0, 3617, 4060, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 4002, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 9, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 114, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 3713, 4066, 0, 52, 0, 0, 22, 3995, 0, 22, 0, 0, 0, 4025, 0, 0, 7, 0, 0, 0, 9, 1, 0, 0, 0, 26, 0, 0, 20, 0, 0, 32, 0, 0, 4052, 23, 10, 36, 1, 37, 0, 4064, 0, 24, 0, 1, 0, 0, 0, 849, 0, 4, 4, 0, 19, 0, 0, 0, 0, 0, 3586, 4068, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 3420, 4039, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 28, 0, 0, 4096, 0, 0, 32, 0, 0, 0, 0, 655, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 4081, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 2650, 4060, 0, 104, 0, 9, 0, 4010, 0, 32, 0, 0, 3, 3996, 0, 0, 4, 0, 0, 0, 11, 581, 0, 0, 23, 31, 0, 0, 11, 0, 0, 51, 0, 0, 4049, 58, 5, 46, 0, 3, 0, 4058, 0, 29, 1, 12, 0, 6, 0, 1261, 0, 62, 1, 27, 0, 0, 0, 0, 0, 0, 3523, 4051, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 3418, 3947, 0, 4, 15, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 43, 0, 0, 4087, 4, 0, 91, 0, 0, 0, 0, 706, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3673, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3836, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 69, 0, 0, 0, 0, 3391, 44, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3955, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4156, 4118, 4027, 4105, 4092, 4072, 4014, 4184] avg=4096.0000 max=4184 max_rank=7 min=4014 min_rank=6 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16362 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3757, 484, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 41, 0, 0, 4, 3997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 8, 13, 0, 3803, 188, 0, 0, 0, 0, 0, 0, 1, 3, 23, 0, 0, 16, 0, 0, 4022, 31, 0, 0, 0, 0, 0, 0, 18, 32, 0, 4006, 27, 12, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 4046, 0, 23, 6, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 748, 0, 0, 5, 6, 0, 3400, 44, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3617, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4245, 4063, 4048, 4096, 4137, 4109, 4227, 3843] avg=4096.0000 max=4245 max_rank=0 min=3843 min_rank=7 imbalance_ratio=1.036377 max_load_reduction=28523 max_load_reduction_pct=0.870453 moved_to_echo=16316 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4002, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3836, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 2, 0, 0, 0, 0, 3420, 44, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4488, 4118, 4025, 4105, 4061, 4074, 3604, 4293] avg=4096.0000 max=4488 max_rank=0 min=3604 min_rank=6 imbalance_ratio=1.095703 max_load_reduction=28280 max_load_reduction_pct=0.863037 moved_to_echo=16032 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3713, 483, 0, 52, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 22, 0, 0, 0, 4025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 9, 1, 0, 3810, 188, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 20, 0, 0, 3974, 31, 0, 0, 0, 0, 0, 0, 32, 0, 0, 4008, 23, 10, 36, 1, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 4040, 0, 24, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 849, 0, 4, 4, 0, 19, 3395, 44, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3586, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4270, 4068, 4015, 4051, 4110, 4102, 4339, 3813] avg=4096.0000 max=4339 max_rank=6 min=3813 min_rank=7 imbalance_ratio=1.059326 max_load_reduction=28429 max_load_reduction_pct=0.867584 moved_to_echo=16364 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3420, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3836, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 655, 48, 0, 0, 0, 0, 3373, 44, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3900, 4118, 4022, 4105, 4078, 4104, 4144, 4297] avg=4096.0000 max=4297 max_rank=7 min=3900 min_rank=0 imbalance_ratio=1.049072 max_load_reduction=28471 max_load_reduction_pct=0.868866 moved_to_echo=16623 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4081, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3836, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 3421, 44, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4567, 4118, 4025, 4105, 4052, 4072, 3522, 4307] avg=4096.0000 max=4567 max_rank=0 min=3522 min_rank=6 imbalance_ratio=1.114990 max_load_reduction=28201 max_load_reduction_pct=0.860626 moved_to_echo=15953 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2650, 473, 0, 104, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 32, 0, 0, 3, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 11, 581, 0, 3800, 192, 0, 0, 0, 0, 0, 0, 0, 23, 31, 0, 0, 11, 0, 0, 3994, 37, 0, 0, 0, 0, 0, 0, 51, 0, 0, 4000, 58, 5, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4032, 0, 29, 1, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1261, 0, 62, 1, 27, 0, 3395, 49, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3523, 214, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3236, 4047, 4588, 4096, 4160, 4077, 4827, 3737] avg=4096.0000 max=4827 max_rank=6 min=3236 min_rank=0 imbalance_ratio=1.178467 max_load_reduction=27941 max_load_reduction_pct=0.852692 moved_to_echo=16801 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3418, 469, 0, 4, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3836, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 4, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 706, 21, 0, 0, 0, 0, 3296, 44, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3906, 4118, 4018, 4105, 4078, 4158, 4091, 4294] avg=4096.0000 max=4294 max_rank=7 min=3906 min_rank=0 imbalance_ratio=1.048340 max_load_reduction=28474 max_load_reduction_pct=0.868958 moved_to_echo=16621 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 28800, 3968, 0, 256, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 0, 4864, 256, 128, 128, 128, 128, 27136, 384, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 128, 128, 640, 0, 30720, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 0, 32256, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 30976, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 0, 0, 128, 0, 0, 32384, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 32512, 128, 128, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[256, 0, 128, 0, 0, 128, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True + [2026-09-21 08:24:26] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 293.2 | throughput per GPU (TFLOP/s/GPU): 109.9 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.644594E+00 | loss scale: 1.0 | grad norm: 3.520 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[254, 343, 857, 0, 2, 685, 935, 541, 2, 484, 28, 1, 2, 53, 554, 282, 7, 446, 848, 1422, 42, 372, 528, 13, 172, 33, 128, 531, 1013, 882, 1130, 1461, 685, 729, 524, 1373, 150, 1108, 796, 666, 1176, 489, 1305, 31, 982, 680, 9, 64, 841, 680, 507, 105, 863, 662, 1907, 43, 967, 18, 287, 4, 92, 10, 922, 42] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[166, 223, 1185, 2, 0, 511, 602, 454, 0, 417, 14, 2, 34, 9, 867, 128, 9, 1294, 1009, 1245, 46, 187, 217, 3, 111, 45, 77, 316, 979, 838, 1122, 1854, 989, 373, 579, 1054, 280, 829, 699, 1006, 1181, 561, 1344, 107, 882, 588, 19, 77, 758, 950, 685, 38, 974, 1156, 1088, 32, 1023, 36, 311, 48, 52, 112, 954, 17] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[183, 256, 1248, 0, 0, 694, 926, 395, 3, 357, 50, 0, 12, 4, 488, 135, 13, 1504, 795, 1360, 91, 91, 275, 1, 146, 38, 139, 263, 769, 585, 1075, 1673, 927, 403, 491, 1233, 213, 890, 755, 522, 1522, 285, 1324, 74, 660, 560, 21, 101, 756, 955, 836, 34, 934, 1698, 1576, 114, 826, 42, 447, 22, 23, 33, 894, 28] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[128, 286, 1651, 0, 1, 603, 686, 603, 4, 886, 95, 1, 14, 8, 479, 162, 2, 912, 606, 1474, 34, 119, 241, 6, 72, 18, 119, 397, 563, 711, 921, 1533, 1179, 694, 550, 851, 175, 516, 536, 637, 1616, 844, 1678, 103, 817, 742, 11, 73, 797, 892, 691, 33, 801, 1674, 1264, 25, 893, 16, 288, 1, 3, 22, 978, 33] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[145, 350, 1060, 2, 0, 604, 567, 418, 2, 478, 37, 1, 12, 33, 464, 171, 6, 986, 897, 1560, 36, 121, 335, 7, 121, 11, 114, 351, 937, 549, 1381, 1430, 747, 823, 846, 879, 218, 669, 705, 1019, 1551, 656, 1283, 110, 1088, 1067, 14, 30, 993, 804, 462, 36, 714, 1383, 1408, 40, 1008, 21, 264, 0, 8, 8, 705, 23] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[160, 251, 967, 2, 0, 599, 728, 532, 3, 680, 33, 0, 14, 31, 397, 135, 6, 996, 523, 1625, 27, 366, 324, 3, 51, 5, 66, 551, 1022, 488, 738, 1742, 1140, 952, 774, 1131, 174, 682, 778, 1067, 1175, 662, 1967, 149, 835, 893, 6, 18, 764, 831, 737, 36, 596, 1211, 1093, 18, 776, 13, 338, 14, 11, 34, 814, 14] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[308, 330, 1087, 1, 0, 939, 658, 318, 0, 1091, 41, 3, 4, 32, 557, 340, 1, 654, 1204, 1488, 32, 230, 297, 8, 131, 18, 129, 544, 1006, 893, 1013, 1497, 628, 495, 651, 1453, 190, 784, 1007, 782, 1501, 356, 1065, 179, 866, 565, 6, 17, 711, 620, 736, 75, 502, 1087, 1371, 134, 1107, 43, 233, 5, 10, 18, 693, 24] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[82, 327, 1481, 2, 2, 776, 655, 770, 0, 437, 55, 2, 39, 10, 789, 80, 1, 795, 677, 2104, 20, 291, 165, 10, 150, 32, 68, 207, 995, 743, 781, 1309, 772, 408, 596, 810, 149, 1096, 705, 585, 1561, 613, 2003, 88, 481, 627, 20, 105, 862, 941, 772, 36, 857, 1456, 1292, 75, 722, 45, 582, 5, 58, 52, 524, 15] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[183, 256, 1248, 0, 0, 694, 926, 395, 358, 245, 0, 0, 0, 0, 0, 0, 3, 357, 50, 0, 12, 4, 488, 135, 1576, 1233, 159, 0, 0, 0, 0, 0, 13, 1504, 795, 1360, 91, 91, 275, 1, 289, 0, 0, 0, 0, 0, 0, 0, 146, 38, 139, 263, 769, 585, 1075, 868, 0, 0, 0, 0, 0, 0, 0, 0, 328, 403, 491, 0, 213, 890, 755, 522, 0, 0, 0, 0, 0, 0, 0, 0, 1522, 285, 361, 74, 660, 560, 21, 101, 0, 0, 0, 0, 0, 0, 0, 0, 756, 955, 836, 34, 934, 1164, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 826, 42, 447, 22, 23, 33, 894, 28, 804, 805, 241, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4305, 4017, 4419, 3883, 3602, 3584, 4793, 4165] avg=4096.0000 max=4793 max_rank=6 min=3584 min_rank=5 imbalance_ratio=1.170166 max_load_reduction=27975 max_load_reduction_pct=0.853729 moved_to_echo=16144 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[254, 343, 857, 0, 2, 685, 935, 541, 264, 95, 0, 0, 0, 0, 0, 0, 2, 484, 28, 1, 2, 53, 554, 282, 1907, 1373, 156, 0, 0, 0, 0, 0, 7, 446, 848, 1422, 42, 372, 528, 13, 112, 0, 0, 0, 0, 0, 0, 0, 172, 33, 128, 531, 1013, 882, 1130, 758, 0, 0, 0, 0, 0, 0, 0, 0, 243, 729, 524, 0, 150, 1108, 796, 666, 0, 0, 0, 0, 0, 0, 0, 0, 1176, 489, 356, 31, 982, 680, 9, 64, 0, 0, 0, 0, 0, 0, 0, 0, 841, 680, 507, 105, 863, 455, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 967, 18, 287, 4, 92, 10, 922, 42, 793, 703, 178, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3976, 4842, 3790, 4647, 4216, 3787, 3494, 4016] avg=4096.0000 max=4842 max_rank=1 min=3494 min_rank=6 imbalance_ratio=1.182129 max_load_reduction=27926 max_load_reduction_pct=0.852234 moved_to_echo=15513 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[166, 223, 1185, 2, 0, 511, 602, 454, 382, 167, 0, 0, 0, 0, 0, 0, 0, 417, 14, 2, 34, 9, 867, 128, 1088, 1054, 161, 0, 0, 0, 0, 0, 9, 1294, 1009, 1245, 46, 187, 217, 3, 197, 0, 0, 0, 0, 0, 0, 0, 111, 45, 77, 316, 979, 838, 1122, 962, 0, 0, 0, 0, 0, 0, 0, 0, 350, 373, 579, 0, 280, 829, 699, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1181, 561, 366, 107, 882, 588, 19, 77, 0, 0, 0, 0, 0, 0, 0, 0, 758, 950, 685, 38, 974, 792, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 36, 311, 48, 52, 112, 954, 17, 817, 892, 257, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3692, 3774, 4207, 4450, 4116, 3781, 4229, 4519] avg=4096.0000 max=4519 max_rank=7 min=3692 min_rank=0 imbalance_ratio=1.103271 max_load_reduction=28249 max_load_reduction_pct=0.862091 moved_to_echo=16645 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[145, 350, 1060, 2, 0, 604, 567, 418, 288, 200, 0, 0, 0, 0, 0, 0, 2, 478, 37, 1, 12, 33, 464, 171, 1408, 879, 154, 0, 0, 0, 0, 0, 6, 986, 897, 1560, 36, 121, 335, 7, 235, 0, 0, 0, 0, 0, 0, 0, 121, 11, 114, 351, 937, 549, 1381, 742, 0, 0, 0, 0, 0, 0, 0, 0, 265, 823, 846, 0, 218, 669, 705, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1551, 656, 350, 110, 1088, 1067, 14, 30, 0, 0, 0, 0, 0, 0, 0, 0, 993, 804, 462, 36, 714, 948, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 21, 264, 0, 8, 8, 705, 23, 779, 688, 194, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3634, 3639, 4183, 4206, 4545, 4866, 3997, 3698] avg=4096.0000 max=4866 max_rank=5 min=3634 min_rank=0 imbalance_ratio=1.187988 max_load_reduction=27902 max_load_reduction_pct=0.851501 moved_to_echo=17106 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[128, 286, 1651, 0, 1, 603, 686, 603, 455, 242, 0, 0, 0, 0, 0, 0, 4, 886, 95, 1, 14, 8, 479, 162, 1264, 851, 201, 0, 0, 0, 0, 0, 2, 912, 606, 1474, 34, 119, 241, 6, 285, 0, 0, 0, 0, 0, 0, 0, 72, 18, 119, 397, 563, 711, 921, 795, 0, 0, 0, 0, 0, 0, 0, 0, 417, 694, 550, 0, 175, 516, 536, 637, 0, 0, 0, 0, 0, 0, 0, 0, 1616, 844, 457, 103, 817, 742, 11, 73, 0, 0, 0, 0, 0, 0, 0, 0, 797, 892, 691, 33, 801, 1147, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 893, 16, 288, 1, 3, 22, 978, 33, 1020, 738, 307, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4655, 3965, 3679, 3596, 3525, 4663, 4386, 4299] avg=4096.0000 max=4663 max_rank=5 min=3525 min_rank=4 imbalance_ratio=1.138428 max_load_reduction=28105 max_load_reduction_pct=0.857697 moved_to_echo=16873 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[308, 330, 1087, 1, 0, 939, 658, 318, 242, 157, 0, 0, 0, 0, 0, 0, 0, 1091, 41, 3, 4, 32, 557, 340, 1371, 1453, 128, 0, 0, 0, 0, 0, 1, 654, 1204, 1488, 32, 230, 297, 8, 185, 0, 0, 0, 0, 0, 0, 0, 131, 18, 129, 544, 1006, 893, 1013, 777, 0, 0, 0, 0, 0, 0, 0, 0, 223, 495, 651, 0, 190, 784, 1007, 782, 0, 0, 0, 0, 0, 0, 0, 0, 1501, 356, 290, 179, 866, 565, 6, 17, 0, 0, 0, 0, 0, 0, 0, 0, 711, 620, 736, 75, 502, 745, 0, 134, 0, 0, 0, 0, 0, 0, 0, 0, 1107, 43, 233, 5, 10, 18, 693, 24, 647, 720, 163, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4040, 5020, 4099, 4511, 4132, 3780, 3523, 3663] avg=4096.0000 max=5020 max_rank=1 min=3523 min_rank=6 imbalance_ratio=1.225586 max_load_reduction=27748 max_load_reduction_pct=0.846802 moved_to_echo=15098 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[160, 251, 967, 2, 0, 599, 728, 532, 446, 178, 0, 0, 0, 0, 0, 0, 3, 680, 33, 0, 14, 31, 397, 135, 1093, 1131, 240, 0, 0, 0, 0, 0, 6, 996, 523, 1625, 27, 366, 324, 3, 210, 0, 0, 0, 0, 0, 0, 0, 51, 5, 66, 551, 1022, 488, 738, 899, 0, 0, 0, 0, 0, 0, 0, 0, 393, 952, 774, 0, 174, 682, 778, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 1175, 662, 528, 149, 835, 893, 6, 18, 0, 0, 0, 0, 0, 0, 0, 0, 764, 831, 737, 36, 596, 823, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 776, 13, 338, 14, 11, 34, 814, 14, 1199, 843, 301, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3863, 3757, 4080, 3820, 4820, 4266, 3805, 4357] avg=4096.0000 max=4820 max_rank=4 min=3757 min_rank=1 imbalance_ratio=1.176758 max_load_reduction=27948 max_load_reduction_pct=0.852905 moved_to_echo=17248 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[82, 327, 1481, 2, 2, 776, 655, 770, 298, 210, 0, 0, 0, 0, 0, 0, 0, 437, 55, 2, 39, 10, 789, 80, 1292, 810, 240, 0, 0, 0, 0, 0, 1, 795, 677, 2104, 20, 291, 165, 10, 248, 0, 0, 0, 0, 0, 0, 0, 150, 32, 68, 207, 995, 743, 781, 679, 0, 0, 0, 0, 0, 0, 0, 0, 273, 408, 596, 0, 149, 1096, 705, 585, 0, 0, 0, 0, 0, 0, 0, 0, 1561, 613, 546, 88, 481, 627, 20, 105, 0, 0, 0, 0, 0, 0, 0, 0, 862, 941, 772, 36, 857, 998, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 722, 45, 582, 5, 58, 52, 524, 15, 1217, 630, 201, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4603, 3754, 4311, 3655, 3812, 4041, 4541, 4051] avg=4096.0000 max=4603 max_rank=0 min=3655 min_rank=3 imbalance_ratio=1.123779 max_load_reduction=28165 max_load_reduction_pct=0.859528 moved_to_echo=16445 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[1536, 2432, 9600, 128, 128, 5504, 5760, 4096, 2816, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[2560, 4992, 5120, 0, 1664, 6656, 6016, 6400, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 4864, 384, 128, 256, 256, 4608, 1536, 11008, 8832, 1536, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 7680, 6656, 12288, 384, 1792, 2432, 128, 1792, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[7424, 256, 2816, 128, 384, 384, 6528, 256, 7296, 6144, 1920, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1024, 256, 896, 3200, 7296, 5760, 8192, 6528, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[6528, 6784, 5504, 512, 6272, 7168, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[11392, 4480, 3328, 896, 6656, 5760, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[793, 0, 895, 0, 0, 29, 4, 590, 596, 796, 3, 2983, 0, 0, 3436, 4, 1, 0, 22, 2345, 535, 114, 573, 0, 1, 1396, 0, 0, 251, 3633, 108, 8, 0, 0, 2, 1, 1558, 33, 79, 4, 0, 1, 1175, 0, 0, 0, 167, 0, 32, 0, 996, 3478, 3247, 0, 1, 0, 2415, 1, 343, 65, 7, 0, 47, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1668, 1, 143, 0, 1, 5, 0, 1665, 55, 1452, 0, 2702, 0, 0, 3510, 0, 4, 0, 1, 815, 4, 990, 56, 1, 0, 964, 0, 0, 262, 2072, 2, 147, 11, 0, 22, 0, 223, 753, 842, 0, 22, 12, 1853, 0, 0, 0, 662, 0, 94, 0, 1235, 3665, 569, 0, 5, 0, 3330, 0, 650, 1987, 16, 0, 297, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1028, 0, 62, 0, 0, 64, 2, 1871, 2657, 2631, 76, 3251, 0, 0, 2399, 2, 0, 0, 2, 2050, 769, 83, 8, 1, 0, 822, 0, 0, 120, 3558, 242, 21, 1, 0, 0, 0, 1218, 133, 24, 1, 34, 1, 942, 0, 0, 0, 727, 0, 7, 1, 38, 2911, 1951, 0, 18, 1, 1670, 3, 357, 920, 31, 0, 60, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[937, 0, 11, 0, 0, 324, 2, 3611, 2089, 1565, 270, 2946, 0, 0, 1161, 2, 0, 0, 7, 3229, 415, 429, 79, 0, 0, 574, 0, 0, 617, 2348, 539, 214, 0, 0, 0, 0, 713, 24, 264, 4, 0, 5, 460, 2, 0, 0, 136, 0, 0, 2, 299, 3482, 2122, 0, 17, 0, 2487, 0, 1308, 15, 7, 0, 51, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1143, 0, 86, 0, 0, 574, 0, 1810, 178, 731, 11, 2851, 0, 0, 2754, 0, 1, 0, 44, 930, 560, 710, 333, 0, 0, 675, 0, 0, 2025, 1694, 216, 380, 0, 0, 0, 0, 406, 4, 1196, 0, 65, 3, 649, 0, 0, 0, 940, 0, 121, 1, 955, 3568, 3368, 0, 3, 0, 3291, 0, 405, 82, 3, 0, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[2198, 1, 76, 0, 0, 143, 35, 2884, 1550, 3443, 11, 1300, 0, 0, 1889, 0, 6, 0, 173, 169, 365, 572, 1007, 2, 0, 940, 0, 0, 607, 1554, 531, 9, 0, 0, 1, 1, 623, 54, 768, 8, 22, 0, 826, 0, 0, 1, 456, 16, 47, 0, 28, 3935, 2607, 0, 0, 0, 2336, 4, 334, 1165, 35, 0, 36, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[732, 1, 133, 0, 6, 91, 3, 3405, 2206, 758, 356, 1987, 0, 0, 1856, 0, 0, 0, 130, 140, 1768, 232, 35, 0, 0, 464, 0, 0, 633, 2774, 12, 0, 0, 0, 0, 0, 380, 9, 290, 0, 36, 0, 1977, 0, 0, 0, 3559, 1, 12, 0, 191, 3940, 1802, 0, 4, 0, 725, 0, 1891, 83, 50, 0, 96, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1157, 0, 57, 0, 2, 175, 0, 3341, 2007, 3838, 137, 3566, 0, 0, 1324, 28, 0, 9, 24, 1234, 3664, 472, 103, 0, 0, 360, 0, 0, 550, 1492, 254, 40, 0, 0, 2, 0, 250, 21, 35, 1, 91, 0, 168, 0, 0, 0, 13, 0, 0, 32, 839, 1383, 3411, 0, 0, 0, 1460, 0, 597, 160, 155, 0, 316, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1668, 1, 143, 0, 1, 5, 0, 1665, 127, 36, 0, 0, 0, 0, 0, 0, 55, 1452, 0, 0, 0, 0, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 815, 4, 990, 56, 1, 1446, 0, 0, 0, 0, 0, 0, 0, 0, 964, 0, 0, 262, 2036, 2, 147, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 22, 0, 223, 753, 842, 0, 2702, 170, 0, 0, 0, 0, 0, 0, 22, 12, 1853, 0, 0, 0, 662, 0, 2283, 252, 0, 0, 0, 0, 0, 0, 94, 0, 1235, 1212, 569, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3330, 0, 650, 1987, 16, 0, 297, 0, 665, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3646, 2527, 3317, 3411, 4723, 5084, 3115, 6945] avg=4096.0000 max=6945 max_rank=7 min=2527 min_rank=1 imbalance_ratio=1.695557 max_load_reduction=25823 max_load_reduction_pct=0.788055 moved_to_echo=19867 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1028, 0, 62, 0, 0, 64, 2, 1871, 87, 63, 0, 0, 0, 0, 0, 0, 2657, 2631, 76, 0, 0, 0, 698, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2050, 769, 83, 8, 1, 988, 0, 0, 0, 0, 0, 0, 0, 0, 822, 0, 0, 120, 3495, 242, 21, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1218, 133, 24, 1, 3251, 135, 0, 0, 0, 0, 0, 0, 34, 1, 942, 0, 0, 0, 727, 0, 1813, 172, 0, 0, 0, 0, 0, 0, 7, 1, 38, 963, 1951, 0, 18, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1670, 3, 357, 920, 31, 0, 60, 0, 454, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3177, 6064, 3901, 4700, 4763, 3689, 2979, 3495] avg=4096.0000 max=6064 max_rank=1 min=2979 min_rank=6 imbalance_ratio=1.480469 max_load_reduction=26704 max_load_reduction_pct=0.814941 moved_to_echo=14926 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[793, 0, 895, 0, 0, 29, 4, 590, 125, 64, 0, 0, 0, 0, 0, 0, 596, 796, 3, 0, 0, 0, 998, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 22, 2345, 535, 114, 573, 0, 1415, 0, 0, 0, 0, 0, 0, 0, 1, 1396, 0, 0, 251, 3569, 108, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1558, 33, 79, 4, 2983, 161, 0, 0, 0, 0, 0, 0, 0, 1, 1175, 0, 0, 0, 167, 0, 2167, 247, 0, 0, 0, 0, 0, 0, 32, 0, 996, 1150, 3247, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2415, 1, 343, 65, 7, 0, 47, 0, 651, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2500, 2397, 5005, 5333, 4821, 3757, 5426, 3529] avg=4096.0000 max=5426 max_rank=6 min=2397 min_rank=1 imbalance_ratio=1.324707 max_load_reduction=27342 max_load_reduction_pct=0.834412 moved_to_echo=17533 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2198, 1, 76, 0, 0, 143, 35, 2884, 68, 27, 0, 0, 0, 0, 0, 0, 1550, 3443, 11, 0, 0, 0, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 173, 169, 365, 572, 1007, 2, 778, 0, 0, 0, 0, 0, 0, 0, 0, 940, 0, 0, 607, 1527, 531, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 623, 54, 768, 8, 1300, 182, 0, 0, 0, 0, 0, 0, 22, 0, 826, 0, 0, 1, 456, 16, 2452, 135, 0, 0, 0, 0, 0, 0, 47, 0, 28, 1301, 2607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2336, 4, 334, 1165, 35, 0, 36, 0, 358, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5432, 5554, 3072, 3614, 2937, 3908, 3983, 4268] avg=4096.0000 max=5554 max_rank=1 min=2937 min_rank=4 imbalance_ratio=1.355957 max_load_reduction=27214 max_load_reduction_pct=0.830505 moved_to_echo=15096 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[937, 0, 11, 0, 0, 324, 2, 3611, 42, 41, 0, 0, 0, 0, 0, 0, 2089, 1565, 270, 0, 0, 0, 338, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3229, 415, 429, 79, 0, 478, 0, 0, 0, 0, 0, 0, 0, 0, 574, 0, 0, 617, 2307, 539, 214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 713, 24, 264, 4, 2946, 161, 0, 0, 0, 0, 0, 0, 0, 5, 460, 2, 0, 0, 136, 0, 2169, 83, 0, 0, 0, 0, 0, 0, 0, 2, 299, 1152, 2122, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2487, 0, 1308, 15, 7, 0, 51, 1, 220, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4968, 4264, 4637, 4251, 4112, 2855, 3592, 4089] avg=4096.0000 max=4968 max_rank=0 min=2855 min_rank=5 imbalance_ratio=1.212891 max_load_reduction=27800 max_load_reduction_pct=0.848389 moved_to_echo=14648 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1143, 0, 86, 0, 0, 574, 0, 1810, 100, 30, 0, 0, 0, 0, 0, 0, 178, 731, 11, 0, 0, 0, 801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 44, 930, 560, 710, 333, 0, 1134, 0, 0, 0, 0, 0, 0, 0, 0, 675, 0, 0, 2025, 1664, 216, 380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 406, 4, 1196, 0, 2851, 165, 0, 0, 0, 0, 0, 0, 65, 3, 649, 0, 0, 0, 940, 0, 2223, 198, 0, 0, 0, 0, 0, 0, 121, 1, 955, 1180, 3368, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3291, 0, 405, 82, 3, 0, 2, 0, 521, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3743, 1721, 3712, 4960, 4622, 4078, 5628, 4304] avg=4096.0000 max=5628 max_rank=6 min=1721 min_rank=1 imbalance_ratio=1.374023 max_load_reduction=27140 max_load_reduction_pct=0.828247 moved_to_echo=18632 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[732, 1, 133, 0, 6, 91, 3, 3405, 71, 54, 0, 0, 0, 0, 0, 0, 2206, 758, 356, 0, 0, 0, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 140, 1768, 232, 35, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0, 464, 0, 0, 633, 2720, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 380, 9, 290, 0, 1987, 185, 0, 0, 0, 0, 0, 0, 36, 0, 1977, 0, 0, 0, 3559, 1, 2459, 137, 0, 0, 0, 0, 0, 0, 12, 0, 191, 1296, 1802, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 725, 0, 1891, 83, 50, 0, 96, 0, 355, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4496, 3845, 3073, 3829, 2851, 8169, 3305, 3200] avg=4096.0000 max=8169 max_rank=5 min=2851 min_rank=4 imbalance_ratio=1.994385 max_load_reduction=24599 max_load_reduction_pct=0.750702 moved_to_echo=17525 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1157, 0, 57, 0, 2, 175, 0, 3341, 48, 26, 0, 0, 0, 0, 0, 0, 2007, 3838, 137, 0, 0, 0, 386, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 24, 1234, 3664, 472, 103, 0, 545, 0, 0, 0, 0, 0, 0, 0, 0, 360, 0, 0, 550, 1466, 254, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 250, 21, 35, 1, 3566, 64, 0, 0, 0, 0, 0, 0, 91, 0, 168, 0, 0, 0, 13, 0, 861, 95, 0, 0, 0, 0, 0, 0, 0, 32, 839, 458, 3411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1460, 0, 597, 160, 155, 0, 316, 0, 250, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4806, 6396, 6051, 2670, 3939, 1228, 4740, 2938] avg=4096.0000 max=6396 max_rank=1 min=1228 min_rank=5 imbalance_ratio=1.561523 max_load_reduction=26372 max_load_reduction_pct=0.804810 moved_to_echo=12845 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[11392, 15232, 896, 0, 0, 0, 5376, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[384, 128, 8064, 128, 0, 128, 6784, 128, 16512, 1408, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 512, 11008, 8192, 3712, 2304, 128, 7552, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[17792, 128, 5888, 4480, 384, 0, 1024, 128, 3584, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[9728, 128, 1536, 0, 128, 1408, 128, 19200, 768, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 5376, 1152, 3584, 128, 21632, 1280, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 6272, 0, 0, 5120, 18816, 1920, 896, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[384, 128, 4608, 8832, 19200, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 8, 2, 30, 0, 159, 0, 3956, 0, 1, 79, 0, 293, 1, 402, 2917, 849, 3374, 446, 31, 0, 0, 0, 3, 4, 9, 3846, 0, 14, 0, 0, 1, 52, 1301, 0, 99, 2056, 1361, 3, 0, 0, 0, 11, 91, 1, 0, 0, 655, 0, 3502, 32, 0, 0, 0, 0, 49, 3801, 2, 0, 1, 1553, 1773, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[5, 0, 1, 0, 1, 0, 1096, 0, 2204, 0, 3, 0, 0, 21, 0, 619, 638, 657, 387, 10, 480, 0, 0, 0, 0, 0, 0, 2230, 3, 10, 0, 0, 24, 248, 1022, 0, 245, 2651, 2609, 0, 0, 0, 0, 4, 1659, 0, 0, 0, 2703, 7, 2907, 920, 12, 0, 19, 0, 59, 3965, 0, 0, 2, 1647, 3700, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 1, 0, 51, 0, 3430, 0, 11, 14, 0, 1, 0, 101, 2, 26, 313, 245, 4, 0, 0, 0, 1, 0, 0, 3158, 894, 404, 0, 1, 0, 1492, 2009, 1, 1042, 3699, 2980, 1, 0, 0, 0, 0, 519, 2, 0, 0, 1109, 35, 2932, 333, 6, 177, 5, 0, 0, 2939, 0, 0, 0, 2655, 2171, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 255, 1, 1, 0, 328, 0, 2071, 0, 49, 0, 23, 537, 0, 1, 93, 136, 1391, 2, 6, 0, 0, 0, 1, 0, 0, 3449, 52, 2, 0, 0, 12, 828, 3375, 0, 224, 2715, 3026, 0, 0, 0, 0, 0, 515, 0, 0, 0, 2140, 6, 3461, 27, 0, 2, 3, 0, 19, 4021, 0, 0, 19, 2047, 1929, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 64, 0, 68, 7, 38, 0, 3235, 0, 0, 0, 11, 54, 1, 1354, 724, 147, 2106, 38, 4, 1, 0, 0, 0, 0, 2, 4027, 0, 3014, 0, 0, 0, 559, 1332, 0, 373, 3628, 132, 0, 0, 0, 0, 3, 205, 0, 0, 0, 567, 130, 3621, 127, 0, 44, 275, 0, 23, 1832, 0, 0, 0, 1646, 3376, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 39, 0, 0, 95, 0, 1911, 0, 1821, 0, 7, 0, 0, 187, 0, 52, 35, 474, 825, 52, 168, 0, 0, 1, 2, 0, 0, 720, 20, 1, 0, 0, 60, 363, 3875, 0, 1456, 2951, 521, 0, 0, 4, 0, 23, 258, 0, 0, 0, 2452, 0, 3750, 722, 1, 3, 0, 0, 0, 4077, 0, 7, 10, 2034, 3791, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 3, 0, 73, 0, 2537, 0, 13, 2, 0, 632, 0, 243, 61, 57, 2405, 5, 51, 0, 0, 0, 0, 0, 0, 3835, 10, 3, 0, 0, 728, 1363, 479, 0, 680, 3009, 3519, 0, 0, 0, 0, 0, 241, 0, 0, 0, 292, 0, 1866, 209, 0, 86, 28, 0, 66, 4091, 5, 0, 8, 3238, 2927, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 67, 0, 2440, 0, 0, 51, 0, 100, 0, 22, 0, 30, 3552, 98, 29, 0, 0, 0, 0, 0, 0, 3137, 1, 0, 0, 0, 17, 540, 1309, 0, 3089, 2069, 1762, 0, 0, 0, 0, 0, 1932, 0, 0, 0, 2356, 640, 1197, 212, 1, 0, 41, 0, 2, 4042, 1, 0, 9, 2936, 1086, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8, 2, 30, 0, 159, 0, 3759, 0, 0, 0, 0, 0, 0, 0, 3956, 0, 1, 79, 0, 293, 1, 402, 614, 180, 0, 0, 0, 0, 0, 0, 2917, 849, 3374, 446, 31, 0, 0, 0, 610, 416, 0, 0, 0, 0, 0, 0, 3, 4, 9, 3846, 0, 14, 0, 0, 334, 0, 0, 0, 0, 0, 0, 0, 1, 52, 1301, 0, 99, 0, 391, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 91, 1, 0, 0, 2056, 360, 42, 0, 0, 0, 0, 0, 655, 0, 2472, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 2, 0, 1, 1553, 1259, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3958, 5526, 8643, 4210, 1847, 2561, 3159, 2864] avg=4096.0000 max=8643 max_rank=2 min=1847 min_rank=4 imbalance_ratio=2.110107 max_load_reduction=24125 max_load_reduction_pct=0.736237 moved_to_echo=10431 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 1, 0, 51, 0, 2907, 0, 0, 0, 0, 0, 0, 0, 3430, 0, 11, 14, 0, 1, 0, 101, 514, 220, 0, 0, 0, 0, 0, 0, 2, 26, 313, 245, 4, 0, 0, 0, 1336, 348, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3158, 894, 404, 0, 1, 409, 0, 0, 0, 0, 0, 0, 0, 0, 1492, 2009, 1, 1042, 0, 856, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 2, 0, 0, 3699, 788, 32, 0, 0, 0, 0, 0, 1109, 35, 2070, 333, 6, 177, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2655, 1542, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2963, 4291, 2274, 4867, 5401, 5040, 3735, 4197] avg=4096.0000 max=5401 max_rank=4 min=2274 min_rank=2 imbalance_ratio=1.318604 max_load_reduction=27367 max_load_reduction_pct=0.835175 moved_to_echo=18373 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 255, 1, 1, 0, 328, 0, 3978, 0, 0, 0, 0, 0, 0, 0, 2071, 0, 49, 0, 23, 537, 0, 1, 607, 196, 0, 0, 0, 0, 0, 0, 93, 136, 1391, 2, 6, 0, 0, 0, 1356, 411, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3449, 52, 2, 0, 0, 363, 0, 0, 0, 0, 0, 0, 0, 12, 828, 3375, 0, 224, 0, 870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 0, 0, 0, 2715, 800, 43, 0, 0, 0, 0, 0, 2140, 6, 2443, 27, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 19, 2047, 1370, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4563, 3484, 3395, 3867, 5309, 4073, 4621, 3456] avg=4096.0000 max=5309 max_rank=4 min=3395 min_rank=2 imbalance_ratio=1.296143 max_load_reduction=27459 max_load_reduction_pct=0.837982 moved_to_echo=17459 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 3, 0, 73, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 2537, 0, 13, 2, 0, 632, 0, 243, 327, 297, 0, 0, 0, 0, 0, 0, 61, 57, 2405, 5, 51, 0, 0, 0, 1577, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3835, 10, 3, 0, 0, 552, 0, 0, 0, 0, 0, 0, 0, 728, 1363, 479, 0, 680, 0, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 241, 0, 0, 0, 3009, 931, 45, 0, 0, 0, 0, 0, 292, 0, 1317, 209, 0, 86, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 5, 0, 8, 3238, 2078, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4051, 4378, 4400, 4261, 4226, 1932, 5395] avg=4096.0000 max=5395 max_rank=7 min=1932 min_rank=6 imbalance_ratio=1.317139 max_load_reduction=27373 max_load_reduction_pct=0.835358 moved_to_echo=15814 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 39, 0, 0, 95, 0, 1911, 0, 4033, 0, 0, 0, 0, 0, 0, 0, 1821, 0, 7, 0, 0, 187, 0, 52, 658, 385, 0, 0, 0, 0, 0, 0, 35, 474, 825, 52, 168, 0, 0, 1, 233, 446, 0, 0, 0, 0, 0, 0, 2, 0, 0, 720, 20, 1, 0, 0, 715, 0, 0, 0, 0, 0, 0, 0, 60, 363, 3875, 0, 1456, 0, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 23, 258, 0, 0, 0, 2951, 137, 44, 0, 0, 0, 0, 0, 2452, 0, 2646, 722, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 10, 2034, 2691, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6078, 3110, 2234, 1458, 5905, 3417, 5824, 4742] avg=4096.0000 max=6078 max_rank=0 min=1458 min_rank=3 imbalance_ratio=1.483887 max_load_reduction=26690 max_load_reduction_pct=0.814514 moved_to_echo=19888 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 64, 0, 68, 7, 38, 0, 1813, 0, 0, 0, 0, 0, 0, 0, 3235, 0, 0, 0, 11, 54, 1, 1354, 635, 343, 0, 0, 0, 0, 0, 0, 724, 147, 2106, 38, 4, 1, 0, 0, 59, 430, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4027, 0, 3014, 0, 0, 636, 0, 0, 0, 0, 0, 0, 0, 0, 559, 1332, 0, 373, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 205, 0, 0, 0, 3628, 34, 19, 0, 0, 0, 0, 0, 567, 130, 2556, 127, 0, 44, 275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 1646, 2397, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1990, 5633, 3509, 7679, 2303, 3889, 3699, 4066] avg=4096.0000 max=7679 max_rank=3 min=1990 min_rank=0 imbalance_ratio=1.874756 max_load_reduction=25089 max_load_reduction_pct=0.765656 moved_to_echo=13957 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 1, 0, 1, 0, 1096, 0, 3923, 0, 0, 0, 0, 0, 0, 0, 2204, 0, 3, 0, 0, 21, 0, 619, 515, 378, 0, 0, 0, 0, 0, 0, 638, 657, 387, 10, 480, 0, 0, 0, 1173, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2230, 3, 10, 0, 0, 702, 0, 0, 0, 0, 0, 0, 0, 24, 248, 1022, 0, 245, 0, 741, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1659, 0, 0, 0, 2651, 695, 42, 0, 0, 0, 0, 0, 2703, 7, 2042, 920, 12, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 2, 1647, 2620, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5026, 3740, 3695, 2945, 2280, 5051, 5703, 4328] avg=4096.0000 max=5703 max_rank=6 min=2280 min_rank=4 imbalance_ratio=1.392334 max_load_reduction=27065 max_load_reduction_pct=0.825958 moved_to_echo=17362 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 67, 0, 3998, 0, 0, 0, 0, 0, 0, 0, 2440, 0, 0, 51, 0, 100, 0, 22, 210, 110, 0, 0, 0, 0, 0, 0, 0, 30, 3552, 98, 29, 0, 0, 0, 789, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3137, 1, 0, 0, 0, 204, 0, 0, 0, 0, 0, 0, 0, 17, 540, 1309, 0, 3089, 0, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1932, 0, 0, 0, 2069, 466, 44, 0, 0, 0, 0, 0, 2356, 640, 845, 212, 1, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 9, 2936, 772, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4065, 2933, 4640, 3342, 5462, 4511, 4095, 3720] avg=4096.0000 max=5462 max_rank=4 min=2933 min_rank=1 imbalance_ratio=1.333496 max_load_reduction=27306 max_load_reduction_pct=0.833313 moved_to_echo=17788 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4480, 2432, 14464, 896, 896, 128, 0, 128, 7168, 2816, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[896, 5504, 14720, 128, 7296, 0, 4608, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 0, 128, 5504, 128, 0, 0, 22784, 4224, 384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[256, 0, 128, 128, 128, 17792, 14848, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 24448, 1024, 3456, 0, 128, 3968, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[21760, 0, 128, 256, 128, 1920, 128, 2816, 4096, 2176, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[12288, 896, 16512, 2688, 128, 384, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 384, 128, 256, 128, 3840, 0, 28544, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[51, 0, 0, 0, 0, 0, 0, 57, 0, 2107, 3526, 0, 0, 587, 0, 3204, 0, 0, 800, 250, 0, 13, 0, 0, 3, 0, 65, 4035, 0, 2791, 0, 580, 0, 0, 389, 0, 0, 0, 0, 0, 18, 0, 86, 3667, 24, 0, 0, 318, 0, 3946, 132, 0, 1704, 0, 26, 0, 0, 1, 0, 6, 4082, 275, 0, 25] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 13, 0, 257, 0, 1468, 3405, 0, 0, 2828, 0, 3524, 0, 0, 1041, 785, 0, 1, 309, 2, 0, 0, 58, 3119, 11, 4066, 0, 66, 0, 0, 254, 0, 0, 3, 0, 1, 2, 0, 46, 3570, 3, 0, 0, 289, 0, 2460, 75, 0, 676, 0, 88, 0, 0, 0, 0, 1, 4096, 250, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[3, 0, 0, 0, 1, 3, 0, 7, 0, 1712, 2878, 0, 0, 3644, 0, 4031, 0, 0, 2, 259, 0, 1, 1, 0, 0, 0, 207, 3132, 0, 221, 0, 14, 0, 0, 1943, 0, 0, 3, 0, 0, 64, 0, 304, 4046, 0, 0, 0, 2422, 0, 2807, 2, 0, 99, 0, 194, 0, 0, 20, 0, 0, 4092, 2, 0, 654] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 0, 10, 0, 10, 0, 3757, 4010, 0, 0, 971, 2, 3356, 0, 0, 1449, 1388, 4, 3, 150, 0, 0, 0, 276, 3123, 2, 201, 0, 0, 0, 0, 26, 0, 0, 0, 0, 3, 111, 0, 0, 3933, 25, 0, 0, 2037, 2, 2570, 16, 0, 1049, 0, 3, 0, 0, 0, 0, 0, 4079, 193, 0, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[9, 0, 0, 0, 0, 0, 0, 0, 0, 1185, 2242, 0, 0, 1927, 0, 4029, 0, 0, 174, 890, 0, 3, 5, 0, 0, 0, 0, 3868, 1, 1498, 0, 201, 0, 0, 176, 0, 0, 0, 0, 0, 18, 0, 58, 4083, 0, 0, 0, 2690, 0, 2805, 0, 0, 1423, 0, 0, 0, 0, 0, 0, 0, 4090, 541, 0, 852] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[78, 5, 11, 0, 0, 0, 0, 51, 0, 460, 2431, 0, 0, 2117, 3, 4050, 0, 0, 1, 304, 5, 0, 12, 0, 0, 4, 0, 4040, 0, 1939, 0, 2042, 0, 0, 1086, 0, 0, 4, 4, 7, 66, 0, 682, 4066, 96, 0, 0, 957, 0, 820, 0, 0, 2395, 0, 8, 1, 0, 0, 0, 0, 4079, 60, 3, 881] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[4, 0, 0, 0, 0, 4, 0, 0, 0, 3579, 746, 0, 0, 3008, 0, 3966, 0, 2, 322, 263, 0, 0, 1, 0, 0, 0, 63, 4086, 0, 320, 0, 892, 0, 0, 138, 0, 4, 0, 0, 0, 0, 0, 518, 4048, 65, 0, 0, 42, 0, 3154, 22, 0, 2904, 0, 315, 0, 0, 0, 0, 0, 4096, 145, 0, 61] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 131, 0, 20, 0, 3472, 3245, 0, 0, 438, 0, 4096, 0, 0, 38, 81, 0, 0, 10, 0, 0, 0, 109, 3805, 0, 1405, 0, 705, 0, 0, 3770, 0, 0, 0, 0, 0, 1, 0, 2, 4062, 0, 0, 0, 50, 0, 2968, 0, 0, 215, 0, 88, 0, 0, 0, 0, 0, 4021, 0, 0, 36] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[51, 0, 0, 0, 0, 0, 0, 57, 3204, 279, 0, 0, 0, 0, 0, 0, 0, 2048, 0, 0, 0, 587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 800, 250, 0, 13, 0, 0, 1370, 1130, 485, 59, 36, 0, 0, 0, 3, 0, 65, 2077, 0, 2791, 0, 580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 389, 0, 0, 0, 0, 0, 3247, 588, 0, 0, 0, 0, 0, 0, 18, 0, 86, 2537, 24, 0, 0, 318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3910, 132, 0, 1704, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 3597, 275, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3591, 2635, 4143, 5516, 4224, 2983, 5772, 3904] avg=4096.0000 max=5772 max_rank=6 min=2635 min_rank=1 imbalance_ratio=1.409180 max_load_reduction=26996 max_load_reduction_pct=0.823853 moved_to_echo=16883 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 13, 0, 257, 3524, 270, 0, 0, 0, 0, 0, 0, 0, 1427, 0, 0, 0, 2828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 785, 0, 1, 309, 2, 1059, 1100, 486, 41, 22, 0, 0, 0, 0, 0, 58, 1606, 11, 4066, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 0, 0, 3, 0, 1, 3135, 454, 0, 0, 0, 0, 0, 0, 2, 0, 46, 2470, 3, 0, 0, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2438, 75, 0, 676, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3610, 250, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4064, 4255, 4846, 5807, 3847, 2810, 3277, 3862] avg=4096.0000 max=5807 max_rank=3 min=2810 min_rank=5 imbalance_ratio=1.417725 max_load_reduction=26961 max_load_reduction_pct=0.822784 moved_to_echo=13796 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 0, 10, 0, 10, 3356, 319, 0, 0, 0, 0, 0, 0, 0, 3652, 0, 0, 0, 971, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1449, 1388, 4, 3, 150, 0, 1060, 1212, 484, 105, 23, 0, 0, 0, 0, 0, 276, 1608, 2, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 3, 3691, 455, 0, 0, 0, 0, 0, 0, 111, 0, 0, 2721, 25, 0, 0, 2037, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2547, 16, 0, 1049, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3595, 193, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3699, 4625, 5878, 2087, 4175, 4894, 3617, 3793] avg=4096.0000 max=5878 max_rank=2 min=2087 min_rank=3 imbalance_ratio=1.435059 max_load_reduction=26890 max_load_reduction_pct=0.820618 moved_to_echo=16479 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 0, 1, 3, 0, 7, 4031, 229, 0, 0, 0, 0, 0, 0, 0, 1665, 0, 0, 0, 3644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 259, 0, 1, 1, 0, 1064, 1247, 486, 47, 25, 0, 0, 0, 0, 0, 207, 1612, 0, 221, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1943, 0, 0, 3, 0, 0, 2649, 456, 0, 0, 0, 0, 0, 0, 64, 0, 304, 2799, 0, 0, 0, 2422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2782, 2, 0, 99, 0, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 3606, 2, 0, 654, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4274, 5309, 3132, 2054, 5051, 5589, 3077, 4282] avg=4096.0000 max=5589 max_rank=5 min=2054 min_rank=3 imbalance_ratio=1.364502 max_load_reduction=27179 max_load_reduction_pct=0.829437 moved_to_echo=17999 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[9, 0, 0, 0, 0, 0, 0, 0, 4029, 177, 0, 0, 0, 0, 0, 0, 0, 1152, 0, 0, 0, 1927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174, 890, 0, 3, 5, 0, 1314, 1258, 486, 33, 25, 0, 0, 0, 0, 0, 0, 1991, 1, 1498, 0, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 0, 0, 0, 0, 2065, 563, 0, 0, 0, 0, 0, 0, 18, 0, 58, 2825, 0, 0, 0, 2690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2780, 0, 0, 1423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3604, 541, 0, 852, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4215, 3079, 4188, 3691, 2804, 5591, 4203, 4997] avg=4096.0000 max=5591 max_rank=5 min=2804 min_rank=4 imbalance_ratio=1.364990 max_load_reduction=27177 max_load_reduction_pct=0.829376 moved_to_echo=17595 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[78, 5, 11, 0, 0, 0, 0, 51, 4050, 192, 0, 0, 0, 0, 0, 0, 0, 448, 0, 0, 0, 2117, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 304, 5, 0, 12, 0, 1372, 1253, 484, 12, 7, 0, 0, 0, 0, 4, 0, 2080, 0, 1939, 0, 2042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1086, 0, 0, 4, 4, 7, 2239, 588, 0, 0, 0, 0, 0, 0, 66, 0, 682, 2813, 96, 0, 0, 957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 813, 0, 0, 2395, 0, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3595, 60, 3, 881, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4387, 2568, 3450, 6065, 3928, 4614, 3217, 4539] avg=4096.0000 max=6065 max_rank=3 min=2568 min_rank=1 imbalance_ratio=1.480713 max_load_reduction=26703 max_load_reduction_pct=0.814911 moved_to_echo=16298 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 0, 0, 0, 4, 0, 0, 3966, 60, 0, 0, 0, 0, 0, 0, 0, 3476, 0, 0, 0, 3008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 322, 263, 0, 0, 1, 0, 1392, 1250, 491, 103, 34, 0, 0, 0, 0, 0, 63, 2095, 0, 320, 0, 892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 0, 4, 0, 0, 0, 686, 599, 0, 0, 0, 0, 0, 0, 0, 0, 518, 2798, 65, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3120, 22, 0, 2904, 0, 315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3605, 145, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4034, 6484, 3858, 3370, 1427, 3423, 6361, 3811] avg=4096.0000 max=6484 max_rank=1 min=1427 min_rank=4 imbalance_ratio=1.583008 max_load_reduction=26284 max_load_reduction_pct=0.802124 moved_to_echo=15022 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 131, 0, 20, 4096, 257, 0, 0, 0, 0, 0, 0, 0, 3375, 0, 0, 0, 438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 81, 0, 0, 10, 0, 1292, 1251, 477, 97, 27, 0, 0, 0, 0, 0, 109, 1959, 0, 1405, 0, 705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3770, 0, 0, 0, 0, 0, 2988, 554, 0, 0, 0, 0, 0, 0, 1, 0, 2, 2811, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2941, 0, 0, 215, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3544, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4504, 3813, 3273, 4178, 7312, 2864, 3244, 3580] avg=4096.0000 max=7312 max_rank=4 min=2864 min_rank=5 imbalance_ratio=1.785156 max_load_reduction=25456 max_load_reduction_pct=0.776855 moved_to_echo=17000 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 3840, 4224, 128, 128, 512, 128, 9984, 9728, 3968, 512, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 0, 7808, 0, 128, 128, 128, 128, 20736, 4352, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 896, 15104, 128, 12544, 0, 4608, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[384, 0, 1792, 21888, 256, 0, 0, 8832, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 17280, 0, 0, 0, 15616, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 21376, 256, 0, 10496, 0, 768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 0, 128, 28800, 1536, 128, 2560, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[256, 128, 128, 0, 128, 256, 0, 512, 30336, 1792, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 4084, 0, 0, 2731, 0, 0, 0, 2013, 0, 6, 0, 260, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4, 2151, 0, 0, 0, 0, 2060, 0, 0, 3570, 0, 0, 19, 0, 0, 0, 1310, 0, 0, 3862, 11, 648, 0, 3144, 471, 1, 0, 0, 0, 0, 65, 34, 4, 0, 2272, 0, 3, 0, 0, 0, 0, 4044] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 4045, 0, 1, 1551, 0, 0, 0, 158, 0, 0, 0, 17, 0, 0, 0, 17, 0, 0, 0, 0, 0, 27, 2122, 0, 0, 0, 0, 4087, 0, 0, 140, 0, 0, 244, 0, 0, 0, 119, 0, 0, 4095, 0, 456, 0, 3549, 4084, 2, 0, 0, 0, 27, 17, 3, 422, 0, 3961, 0, 22, 0, 0, 0, 0, 3602] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 4078, 0, 29, 777, 0, 0, 4, 3961, 16, 0, 0, 38, 1, 6, 0, 12, 0, 0, 3, 0, 12, 4, 3432, 1, 0, 0, 0, 3915, 15, 0, 1255, 0, 1, 0, 0, 0, 0, 398, 6, 0, 4058, 1, 1383, 0, 402, 1587, 0, 0, 0, 0, 0, 175, 0, 157, 0, 2994, 0, 0, 0, 0, 0, 6, 4041] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 3518, 0, 0, 1355, 0, 0, 0, 2877, 0, 0, 0, 908, 0, 0, 0, 8, 0, 0, 0, 0, 0, 13, 2815, 0, 0, 0, 0, 4004, 0, 0, 341, 0, 0, 2, 0, 0, 0, 21, 0, 0, 4082, 0, 68, 0, 2335, 3334, 0, 0, 0, 1, 0, 4, 0, 0, 0, 3030, 0, 0, 0, 0, 0, 0, 4052] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 3675, 3, 30, 3092, 8, 0, 52, 1068, 0, 12, 0, 144, 1, 0, 0, 0, 0, 0, 0, 0, 1, 14, 3111, 0, 7, 0, 0, 4088, 0, 0, 1986, 3, 0, 31, 0, 0, 0, 1026, 0, 0, 4087, 0, 1964, 0, 1862, 2245, 0, 0, 0, 1, 0, 226, 129, 34, 0, 56, 0, 0, 0, 0, 0, 0, 3812] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 3716, 0, 1, 1455, 0, 0, 16, 244, 2, 0, 0, 2167, 0, 0, 0, 0, 31, 0, 0, 0, 0, 2, 2168, 0, 0, 0, 0, 3700, 0, 0, 2824, 0, 0, 0, 0, 0, 0, 1389, 0, 0, 4096, 0, 728, 0, 427, 3528, 15, 0, 0, 0, 0, 2, 37, 102, 0, 2153, 0, 32, 0, 0, 0, 0, 3933] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 4076, 0, 0, 2700, 0, 0, 4, 718, 0, 0, 0, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2508, 0, 0, 0, 0, 3904, 0, 0, 857, 0, 0, 486, 0, 0, 7, 1470, 0, 0, 4093, 0, 74, 0, 1549, 1991, 0, 0, 0, 0, 0, 0, 1, 189, 0, 3925, 0, 0, 0, 0, 0, 0, 4068] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 4069, 0, 0, 1094, 0, 0, 1, 860, 0, 0, 0, 16, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 3470, 0, 0, 0, 0, 4021, 0, 0, 180, 0, 0, 4, 0, 0, 0, 1878, 0, 0, 4096, 0, 19, 0, 3645, 1272, 0, 0, 0, 0, 2, 41, 17, 0, 0, 3971, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2335, 0, 0, 2731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2013, 0, 6, 0, 260, 0, 0, 0, 1533, 372, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4, 2151, 204, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1495, 0, 0, 3570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 1310, 0, 2722, 216, 165, 0, 0, 0, 0, 0, 0, 0, 11, 648, 0, 3144, 267, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 34, 4, 0, 3697, 0, 0, 0, 0, 0, 0, 0, 2272, 0, 3, 0, 0, 0, 0, 1322, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5066, 4184, 2553, 5065, 4432, 4071, 3800, 3597] avg=4096.0000 max=5066 max_rank=0 min=2553 min_rank=2 imbalance_ratio=1.236816 max_load_reduction=27702 max_load_reduction_pct=0.845398 moved_to_echo=15900 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2313, 0, 1, 1551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 0, 0, 0, 17, 0, 0, 0, 1518, 739, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 27, 2122, 1770, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2964, 0, 0, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 244, 0, 0, 0, 119, 0, 2424, 214, 176, 0, 0, 0, 0, 0, 0, 0, 0, 456, 0, 3549, 2314, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 17, 3, 422, 0, 3919, 0, 0, 0, 0, 0, 0, 0, 3961, 0, 22, 0, 0, 0, 0, 1178, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3865, 2432, 4320, 3104, 3177, 6321, 4388, 5161] avg=4096.0000 max=6321 max_rank=5 min=2432 min_rank=1 imbalance_ratio=1.543213 max_load_reduction=26447 max_load_reduction_pct=0.807098 moved_to_echo=19047 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2011, 0, 0, 1355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2877, 0, 0, 0, 908, 0, 0, 0, 1321, 724, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 13, 2815, 1445, 376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2904, 0, 0, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 21, 0, 2727, 186, 174, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 2335, 1889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 3908, 0, 0, 0, 0, 0, 0, 0, 3030, 0, 0, 0, 0, 0, 0, 1325, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3366, 5830, 4657, 3245, 3110, 4292, 3913, 4355] avg=4096.0000 max=5830 max_rank=1 min=3110 min_rank=4 imbalance_ratio=1.423340 max_load_reduction=26938 max_load_reduction_pct=0.822083 moved_to_echo=15670 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2101, 3, 30, 3092, 8, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 0, 12, 0, 144, 1, 0, 0, 1380, 739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 14, 3111, 973, 384, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 2965, 0, 0, 1986, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 31, 0, 0, 0, 1026, 0, 2566, 194, 175, 0, 0, 0, 0, 0, 0, 0, 0, 1964, 0, 1862, 1272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 226, 129, 34, 0, 3912, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 1246, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5286, 3344, 4483, 4958, 3995, 5098, 4302, 1302] avg=4096.0000 max=5286 max_rank=0 min=1302 min_rank=7 imbalance_ratio=1.290527 max_load_reduction=27482 max_load_reduction_pct=0.838684 moved_to_echo=14697 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2331, 0, 29, 777, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3961, 16, 0, 0, 38, 1, 6, 0, 1531, 708, 0, 0, 0, 0, 0, 0, 12, 0, 0, 3, 0, 12, 4, 3432, 688, 367, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2840, 15, 0, 1255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 398, 6, 2720, 216, 174, 0, 0, 0, 0, 0, 0, 0, 1, 1383, 0, 402, 899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 175, 0, 157, 0, 3884, 0, 0, 0, 0, 0, 0, 0, 2994, 0, 0, 0, 0, 0, 6, 1321, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3141, 6261, 4518, 4111, 3515, 2685, 4216, 4321] avg=4096.0000 max=6261 max_rank=1 min=2685 min_rank=5 imbalance_ratio=1.528564 max_load_reduction=26507 max_load_reduction_pct=0.808929 moved_to_echo=14737 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2124, 0, 1, 1455, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 244, 2, 0, 0, 2167, 0, 0, 0, 1395, 669, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 2, 2168, 1529, 347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2684, 0, 0, 2824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1389, 0, 2647, 197, 176, 0, 0, 0, 0, 0, 0, 0, 0, 728, 0, 427, 1999, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 37, 102, 0, 3920, 0, 0, 0, 0, 0, 0, 0, 2153, 0, 32, 0, 0, 0, 0, 1286, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3596, 4477, 4077, 5508, 4409, 3169, 4061, 3471] avg=4096.0000 max=5508 max_rank=3 min=3169 min_rank=5 imbalance_ratio=1.344727 max_load_reduction=27260 max_load_reduction_pct=0.831909 moved_to_echo=15110 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2322, 0, 0, 2700, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 718, 0, 0, 0, 148, 0, 0, 0, 1534, 710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2508, 867, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2823, 0, 0, 857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 0, 0, 7, 1470, 0, 2742, 220, 176, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 1549, 1124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 189, 0, 3917, 0, 0, 0, 0, 0, 0, 0, 3925, 0, 0, 0, 0, 0, 0, 1326, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5026, 3110, 3746, 3680, 5101, 2747, 4107, 5251] avg=4096.0000 max=5251 max_rank=7 min=2747 min_rank=5 imbalance_ratio=1.281982 max_load_reduction=27517 max_load_reduction_pct=0.839752 moved_to_echo=17206 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2327, 0, 0, 1094, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 860, 0, 0, 0, 16, 0, 0, 0, 1527, 727, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 3470, 551, 377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2917, 0, 0, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1878, 0, 2757, 215, 175, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 3645, 721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 41, 17, 0, 0, 3921, 0, 0, 0, 0, 0, 0, 0, 3971, 0, 0, 0, 0, 0, 0, 1339, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3422, 3130, 4414, 3097, 5029, 4385, 3981, 5310] avg=4096.0000 max=5310 max_rank=7 min=3097 min_rank=3 imbalance_ratio=1.296387 max_load_reduction=27458 max_load_reduction_pct=0.837952 moved_to_echo=18705 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[11904, 128, 128, 0, 3712, 128, 128, 0, 11776, 5504, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[22400, 0, 128, 0, 0, 0, 128, 10368, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 17920, 128, 128, 14848, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 896, 0, 0, 128, 7680, 128, 21376, 1664, 1408, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 128, 5376, 0, 17024, 10496, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 0, 21632, 128, 0, 11264, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 0, 128, 128, 21888, 8064, 2816, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 128, 128, 640, 256, 1024, 0, 31104, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 4090, 0, 3906, 2, 0, 5, 0, 0, 49, 0, 3382, 1, 0, 0, 3622, 123, 2162, 0, 0, 0, 0, 23, 17, 34, 0, 0, 2854, 0, 0, 1078, 0, 0, 0, 0, 0, 2, 0, 0, 35, 0, 0, 587, 4081, 0, 1763, 0, 0, 1017, 0, 0, 0, 0, 0, 918, 0, 0, 3017, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1, 0, 0, 17, 0, 3720, 6, 0, 0, 0, 0, 6, 0, 3451, 0, 0, 0, 1457, 30, 102, 0, 0, 42, 1, 534, 3722, 0, 0, 0, 4062, 0, 0, 201, 0, 0, 0, 0, 0, 30, 0, 4, 1, 0, 0, 771, 4092, 0, 2957, 0, 0, 4062, 0, 0, 0, 0, 0, 1722, 2, 0, 1775, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 809, 0, 3958, 12, 9, 1, 0, 0, 9, 0, 2784, 2, 0, 0, 3469, 43, 1284, 0, 6, 5, 13, 27, 2133, 26, 0, 0, 4091, 0, 0, 742, 0, 0, 0, 6, 208, 1, 14, 0, 47, 0, 0, 2093, 3837, 0, 721, 0, 0, 3200, 0, 0, 0, 0, 0, 2401, 0, 2, 815, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 119, 0, 2101, 33, 0, 0, 0, 0, 128, 2, 198, 0, 0, 0, 2858, 65, 1965, 0, 0, 222, 0, 0, 3776, 2, 0, 0, 4082, 0, 0, 3072, 0, 0, 0, 0, 126, 2, 0, 0, 2, 0, 0, 1362, 3981, 0, 2717, 0, 0, 1611, 0, 0, 0, 0, 0, 320, 0, 0, 4024, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 4049, 0, 3264, 0, 0, 4, 0, 0, 6, 0, 3308, 0, 0, 0, 1859, 321, 139, 0, 0, 0, 9, 114, 1634, 0, 0, 0, 4078, 0, 0, 359, 0, 0, 0, 0, 3, 1582, 0, 0, 0, 0, 0, 363, 3988, 0, 500, 0, 0, 1095, 0, 0, 0, 0, 0, 2994, 0, 0, 3099, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[14, 0, 0, 2381, 0, 3796, 0, 0, 0, 1, 6, 8, 0, 3475, 9, 0, 5, 2787, 2, 502, 1, 0, 216, 0, 1535, 1596, 0, 0, 1, 3925, 0, 0, 3006, 8, 0, 0, 0, 102, 141, 0, 0, 0, 0, 0, 2037, 3241, 0, 27, 0, 3, 1223, 0, 0, 0, 4, 3, 2212, 0, 0, 497, 0, 2, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 3995, 0, 2204, 0, 0, 0, 0, 0, 56, 0, 2715, 21, 0, 0, 3008, 6, 2465, 0, 0, 0, 0, 2070, 2386, 0, 0, 0, 3914, 0, 0, 218, 0, 0, 0, 0, 30, 5, 1, 0, 3, 0, 0, 7, 4092, 0, 724, 0, 0, 3450, 0, 0, 0, 0, 0, 1187, 0, 0, 211, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 3409, 0, 1949, 1, 0, 0, 0, 0, 76, 0, 3380, 4, 0, 0, 3642, 8, 1497, 0, 0, 97, 0, 41, 1805, 10, 0, 0, 4071, 0, 0, 2608, 0, 0, 0, 4, 1, 9, 0, 0, 20, 0, 15, 18, 4096, 0, 2401, 0, 0, 2251, 0, 0, 0, 0, 0, 8, 0, 0, 1347, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 17, 0, 2066, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 3451, 0, 0, 915, 462, 0, 0, 0, 0, 0, 0, 0, 1376, 30, 102, 0, 0, 42, 1, 0, 0, 0, 0, 0, 0, 0, 0, 534, 3722, 0, 0, 0, 1476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 201, 0, 0, 0, 0, 0, 30, 0, 2510, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 771, 1772, 0, 2957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 0, 1858, 76, 0, 0, 0, 0, 0, 0, 1722, 2, 0, 1775, 0, 0, 0, 0, 739, 81, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2090, 4834, 1551, 5732, 2741, 5505, 5996, 4319] avg=4096.0000 max=5996 max_rank=6 min=1551 min_rank=2 imbalance_ratio=1.463867 max_load_reduction=26772 max_load_reduction_pct=0.817017 moved_to_echo=18561 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 809, 0, 2198, 12, 9, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 9, 0, 2784, 2, 0, 974, 433, 0, 0, 0, 0, 0, 0, 0, 3276, 43, 1284, 0, 6, 5, 13, 0, 0, 0, 0, 0, 0, 0, 0, 27, 2133, 26, 0, 0, 1486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 742, 0, 0, 0, 6, 208, 1, 14, 2528, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 2093, 1662, 0, 721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3200, 0, 0, 0, 0, 0, 1742, 77, 0, 0, 0, 0, 0, 0, 2401, 0, 2, 815, 0, 0, 0, 0, 786, 193, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3028, 4203, 4627, 3672, 3499, 4523, 5019, 4197] avg=4096.0000 max=5019 max_rank=6 min=3028 min_rank=0 imbalance_ratio=1.225342 max_load_reduction=27749 max_load_reduction_pct=0.846832 moved_to_echo=17238 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4090, 0, 2169, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 49, 0, 3382, 1, 0, 961, 461, 0, 0, 0, 0, 0, 0, 0, 3421, 123, 2162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 17, 34, 0, 0, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1078, 0, 0, 0, 0, 0, 2, 0, 1764, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 587, 1767, 0, 1763, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 0, 0, 0, 0, 0, 1853, 53, 0, 0, 0, 0, 0, 0, 918, 0, 0, 3017, 0, 0, 0, 0, 776, 201, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6261, 4859, 5706, 1111, 2844, 4152, 2923, 4912] avg=4096.0000 max=6261 max_rank=0 min=1111 min_rank=3 imbalance_ratio=1.528564 max_load_reduction=26507 max_load_reduction_pct=0.808929 moved_to_echo=14831 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 119, 0, 1167, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 2, 198, 0, 0, 517, 450, 0, 0, 0, 0, 0, 0, 0, 2699, 65, 1965, 0, 0, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3776, 2, 0, 0, 1483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3072, 0, 0, 0, 0, 126, 2, 0, 2523, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1362, 1724, 0, 2717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1611, 0, 0, 0, 0, 0, 1807, 76, 0, 0, 0, 0, 0, 0, 320, 0, 0, 4024, 0, 0, 0, 0, 417, 159, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1319, 1295, 4951, 5261, 5723, 5805, 3494, 4920] avg=4096.0000 max=5805 max_rank=5 min=1295 min_rank=1 imbalance_ratio=1.417236 max_load_reduction=26963 max_load_reduction_pct=0.822845 moved_to_echo=19942 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3995, 0, 1224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 2715, 21, 0, 542, 462, 0, 0, 0, 0, 0, 0, 0, 2841, 6, 2465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2070, 2386, 0, 0, 0, 1422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 218, 0, 0, 0, 0, 30, 5, 1, 2419, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 7, 1772, 0, 724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3450, 0, 0, 0, 0, 0, 1858, 73, 0, 0, 0, 0, 0, 0, 1187, 0, 0, 211, 0, 0, 0, 0, 438, 167, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5219, 3796, 5312, 5878, 2673, 2506, 5381, 2003] avg=4096.0000 max=5878 max_rank=3 min=2003 min_rank=7 imbalance_ratio=1.435059 max_load_reduction=26890 max_load_reduction_pct=0.820618 moved_to_echo=12563 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[14, 0, 0, 2381, 0, 2108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 8, 0, 3475, 9, 0, 934, 366, 0, 0, 0, 0, 0, 0, 5, 2632, 2, 502, 1, 0, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1535, 1596, 0, 0, 1, 1425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3006, 8, 0, 0, 0, 102, 141, 0, 2426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2037, 1404, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1223, 0, 0, 0, 4, 3, 1471, 74, 0, 0, 0, 0, 0, 0, 2212, 0, 0, 497, 0, 2, 2, 0, 754, 155, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4503, 4799, 3358, 4557, 5683, 3468, 2778, 3622] avg=4096.0000 max=5683 max_rank=4 min=2778 min_rank=6 imbalance_ratio=1.387451 max_load_reduction=27085 max_load_reduction_pct=0.826569 moved_to_echo=15551 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4049, 0, 1806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 6, 0, 3308, 0, 0, 807, 455, 0, 0, 0, 0, 0, 0, 0, 1752, 321, 139, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 114, 1634, 0, 0, 0, 1474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 359, 0, 0, 0, 0, 3, 1582, 0, 2523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 363, 1719, 0, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1095, 0, 0, 0, 0, 0, 1814, 81, 0, 0, 0, 0, 0, 0, 2994, 0, 0, 3099, 0, 0, 0, 0, 651, 107, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5855, 4580, 2221, 3222, 4467, 2582, 2990, 6851] avg=4096.0000 max=6851 max_rank=7 min=2221 min_rank=2 imbalance_ratio=1.672607 max_load_reduction=25917 max_load_reduction_pct=0.790924 moved_to_echo=16890 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3409, 0, 1083, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 3380, 4, 0, 479, 463, 0, 0, 0, 0, 0, 0, 0, 3440, 8, 1497, 0, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 1805, 10, 0, 0, 1479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2608, 0, 0, 0, 4, 1, 9, 0, 2516, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 15, 18, 1773, 0, 2401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2251, 0, 0, 0, 0, 0, 1860, 76, 0, 0, 0, 0, 0, 0, 8, 0, 0, 1347, 0, 0, 0, 0, 387, 202, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4493, 4402, 5042, 3335, 5138, 4227, 4187, 1944] avg=4096.0000 max=5138 max_rank=4 min=1944 min_rank=7 imbalance_ratio=1.254395 max_load_reduction=27630 max_load_reduction_pct=0.843201 moved_to_echo=15496 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 0, 18944, 0, 13824, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4352, 17152, 128, 0, 128, 11392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[11392, 128, 0, 0, 128, 512, 1792, 128, 19328, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[11776, 128, 128, 14848, 0, 128, 128, 0, 4992, 1280, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 7296, 13696, 0, 11904, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 21504, 640, 10240, 128, 128, 640, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 17920, 0, 0, 0, 128, 128, 14336, 640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 384, 128, 22784, 128, 0, 6144, 3584, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 2716, 0, 0, 2, 0, 0, 0, 0, 0, 0, 91, 2372, 0, 247, 10, 0, 0, 0, 0, 826, 4, 241, 0, 0, 86, 4089, 0, 3994, 0, 3290, 596, 0, 0, 0, 4064, 0, 0, 1, 0, 0, 3814, 0, 1, 0, 1027, 1069, 0, 0, 0, 0, 0, 171, 12, 0, 0, 0, 0, 0, 0, 4045, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 3770, 0, 0, 0, 0, 0, 0, 0, 0, 1, 230, 3863, 0, 3357, 1, 0, 0, 0, 0, 591, 0, 537, 0, 0, 101, 4090, 2, 3753, 0, 643, 1323, 0, 0, 0, 4090, 0, 0, 0, 0, 7, 3172, 0, 0, 0, 61, 1170, 4, 0, 10, 0, 0, 1536, 0, 0, 0, 0, 0, 0, 0, 454, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 2842, 0, 1, 3, 0, 0, 0, 0, 0, 0, 271, 3451, 0, 1210, 0, 0, 0, 0, 0, 169, 0, 1381, 0, 0, 483, 4093, 0, 3295, 1, 1942, 2318, 0, 0, 0, 4044, 0, 7, 0, 0, 12, 656, 0, 0, 0, 41, 68, 3, 0, 5, 0, 0, 3483, 0, 0, 0, 0, 0, 0, 0, 2989, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 3043, 0, 0, 47, 0, 0, 0, 0, 0, 0, 1487, 4092, 0, 2458, 0, 0, 0, 0, 0, 1106, 0, 235, 0, 0, 3, 4092, 0, 2123, 0, 2437, 222, 1, 0, 0, 4094, 0, 0, 0, 0, 0, 1717, 0, 0, 0, 2495, 508, 0, 0, 14, 0, 0, 2274, 0, 0, 0, 0, 0, 0, 0, 320, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[50, 0, 2896, 0, 2, 11, 0, 0, 8, 0, 6, 4, 1, 3815, 0, 2060, 0, 0, 0, 0, 0, 3508, 33, 3, 0, 0, 0, 4086, 0, 3113, 0, 3081, 1442, 1, 9, 0, 4051, 9, 4, 0, 0, 0, 2329, 0, 15, 2, 1093, 135, 0, 4, 28, 0, 0, 516, 36, 4, 0, 17, 5, 0, 0, 384, 7, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 3949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 328, 3153, 0, 399, 1, 0, 0, 0, 0, 2997, 1, 27, 0, 0, 3, 4088, 0, 1418, 0, 3495, 1290, 0, 0, 0, 3218, 0, 0, 0, 0, 0, 3020, 0, 0, 0, 801, 20, 0, 0, 0, 0, 0, 2068, 0, 0, 0, 0, 0, 0, 0, 2492, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 3873, 0, 0, 32, 0, 0, 0, 0, 0, 0, 31, 3894, 0, 3524, 0, 0, 0, 0, 0, 546, 0, 558, 0, 0, 0, 4096, 0, 3142, 0, 2687, 70, 0, 0, 0, 4001, 0, 0, 0, 0, 0, 332, 0, 0, 0, 483, 5, 0, 0, 1, 0, 0, 2329, 0, 0, 0, 0, 0, 0, 0, 3164, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4079, 0, 0, 2, 0, 0, 0, 0, 0, 0, 489, 4095, 0, 2148, 0, 0, 0, 0, 0, 1912, 0, 33, 0, 0, 56, 4096, 0, 3022, 0, 3397, 7, 0, 0, 0, 3975, 0, 0, 0, 0, 0, 1366, 0, 0, 0, 167, 70, 0, 0, 7, 0, 0, 1828, 0, 0, 0, 0, 0, 0, 0, 2019, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2716, 0, 0, 2, 0, 0, 702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 1191, 0, 247, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 826, 4, 241, 1786, 309, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 1853, 0, 3290, 0, 0, 0, 0, 0, 0, 0, 0, 596, 0, 0, 0, 3283, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3814, 0, 1, 0, 1027, 1069, 1086, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 171, 12, 0, 2303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 0, 872, 1055, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3420, 1529, 3176, 5229, 3880, 7076, 2486, 5972] avg=4096.0000 max=7076 max_rank=5 min=1529 min_rank=1 imbalance_ratio=1.727539 max_load_reduction=25692 max_load_reduction_pct=0.784058 moved_to_echo=19414 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 3770, 0, 0, 0, 0, 0, 706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 230, 1939, 0, 3357, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 591, 0, 537, 1787, 504, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 2, 1740, 0, 643, 0, 0, 0, 0, 0, 0, 0, 0, 1323, 0, 0, 0, 3304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3172, 0, 0, 0, 61, 1170, 1021, 80, 0, 0, 0, 0, 0, 0, 4, 0, 10, 0, 0, 1536, 0, 0, 2303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 454, 0, 1, 1420, 992, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4477, 5527, 3420, 2486, 4627, 5511, 3853, 2867] avg=4096.0000 max=5527 max_rank=1 min=2486 min_rank=3 imbalance_ratio=1.349365 max_load_reduction=27241 max_load_reduction_pct=0.831329 moved_to_echo=16858 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2842, 0, 1, 3, 0, 0, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 271, 1733, 0, 1210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 0, 1381, 1787, 450, 0, 0, 0, 0, 0, 0, 0, 0, 483, 0, 0, 1529, 1, 1942, 0, 0, 0, 0, 0, 0, 0, 0, 2318, 0, 0, 0, 3267, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 656, 0, 0, 0, 41, 68, 896, 79, 0, 0, 0, 0, 0, 0, 3, 0, 5, 0, 0, 3483, 0, 0, 2306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2989, 0, 0, 1268, 870, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3544, 3214, 3787, 3955, 5592, 1752, 5797, 5127] avg=4096.0000 max=5797 max_rank=6 min=1752 min_rank=5 imbalance_ratio=1.415283 max_load_reduction=26971 max_load_reduction_pct=0.823090 moved_to_echo=18268 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3873, 0, 0, 32, 0, 0, 691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 1955, 0, 3524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 0, 558, 1788, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1458, 0, 2687, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 3232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 332, 0, 0, 0, 483, 5, 854, 78, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2329, 0, 0, 2308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3164, 0, 0, 1431, 830, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4596, 5510, 3400, 4145, 3302, 1752, 4638, 5425] avg=4096.0000 max=5510 max_rank=1 min=1752 min_rank=5 imbalance_ratio=1.345215 max_load_reduction=27258 max_load_reduction_pct=0.831848 moved_to_echo=15117 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3043, 0, 0, 47, 0, 0, 707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1487, 2054, 0, 2458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1106, 0, 235, 1788, 534, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 985, 0, 2437, 0, 0, 0, 0, 0, 0, 0, 0, 222, 1, 0, 0, 3307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1717, 0, 0, 0, 2495, 508, 577, 80, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 2274, 0, 0, 2304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 320, 0, 0, 1504, 561, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3797, 5999, 3663, 3425, 3530, 5377, 4592, 2385] avg=4096.0000 max=5999 max_rank=1 min=2385 min_rank=7 imbalance_ratio=1.464600 max_load_reduction=26769 max_load_reduction_pct=0.816925 moved_to_echo=15884 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[50, 0, 2896, 0, 2, 11, 0, 0, 699, 0, 0, 0, 0, 0, 0, 0, 8, 0, 6, 4, 1, 1915, 0, 2060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3508, 33, 3, 1785, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1444, 0, 3081, 0, 0, 0, 0, 0, 0, 0, 0, 1442, 1, 9, 0, 3273, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2329, 0, 15, 2, 1093, 135, 847, 79, 0, 0, 0, 0, 0, 0, 0, 4, 28, 0, 0, 516, 36, 4, 2301, 0, 0, 0, 0, 0, 0, 0, 0, 17, 5, 0, 0, 384, 7, 0, 1402, 822, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3658, 3994, 5827, 4525, 4738, 4500, 2889, 2637] avg=4096.0000 max=5827 max_rank=2 min=2637 min_rank=7 imbalance_ratio=1.422607 max_load_reduction=26941 max_load_reduction_pct=0.822174 moved_to_echo=14764 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3949, 0, 0, 0, 0, 0, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 328, 1575, 0, 399, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2997, 1, 27, 1786, 416, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 650, 0, 3495, 0, 0, 0, 0, 0, 0, 0, 0, 1290, 0, 0, 0, 2591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3020, 0, 0, 0, 801, 20, 389, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2068, 0, 0, 2302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2492, 0, 0, 1162, 379, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4509, 2302, 5228, 4148, 3881, 4297, 4370, 4033] avg=4096.0000 max=5228 max_rank=2 min=2302 min_rank=1 imbalance_ratio=1.276367 max_load_reduction=27540 max_load_reduction_pct=0.840454 moved_to_echo=16581 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4079, 0, 0, 2, 0, 0, 686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 2056, 0, 2148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1912, 0, 33, 1788, 534, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 1402, 0, 3397, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 3211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1366, 0, 0, 0, 167, 70, 822, 78, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 1828, 0, 0, 2308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2019, 0, 0, 1505, 798, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4767, 4693, 4267, 4855, 3218, 2503, 4143, 4322] avg=4096.0000 max=4855 max_rank=3 min=2503 min_rank=5 imbalance_ratio=1.185303 max_load_reduction=27913 max_load_reduction_pct=0.851837 moved_to_echo=14186 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 0, 11776, 128, 3072, 14336, 3840, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 0, 15872, 128, 128, 10624, 6400, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 2944, 14464, 0, 15488, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 16512, 0, 128, 128, 6272, 3072, 6528, 640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 768, 0, 128, 11136, 128, 20992, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 0, 14208, 128, 128, 18560, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 27264, 0, 128, 128, 0, 0, 5504, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[7296, 128, 128, 0, 25472, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1374, 378, 23, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 8, 0, 0, 4085, 9, 0, 65, 40, 2476, 0, 0, 0, 0, 0, 0, 572, 2670, 2607, 119, 0, 0, 2, 3189, 0, 0, 0, 1294, 4084, 0, 3249, 0, 0, 0, 0, 0, 1673, 0, 0, 755, 0, 1, 0, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 505, 1, 0, 0, 0, 0, 0, 4018, 0, 0, 0, 0, 0, 0, 0, 1, 3940, 0, 0, 40, 32, 635, 0, 0, 0, 0, 1, 0, 3416, 4096, 4076, 1723, 0, 0, 0, 3788, 0, 0, 0, 1030, 3946, 0, 497, 0, 0, 3, 0, 1, 23, 0, 0, 963, 0, 0, 0, 31, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 0, 0, 323, 10, 0, 0, 0, 0, 0, 0, 3894, 0, 0, 0, 0, 0, 0, 0, 1, 4090, 0, 0, 137, 58, 2125, 0, 0, 0, 0, 1, 0, 1422, 2440, 3936, 873, 1, 0, 0, 4015, 0, 0, 0, 191, 3736, 0, 3509, 0, 0, 10, 0, 0, 209, 0, 0, 1033, 0, 0, 0, 734, 15] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[2, 0, 3, 0, 0, 0, 0, 38, 0, 1, 12, 0, 5, 12, 4060, 0, 7, 1, 21, 0, 3, 0, 0, 4090, 0, 6, 491, 185, 364, 4, 5, 2, 0, 0, 0, 2824, 3170, 4054, 662, 21, 14, 0, 3834, 0, 0, 1, 2, 3433, 0, 304, 0, 0, 0, 0, 0, 209, 0, 0, 3988, 16, 0, 2, 922, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 242, 0, 0, 101, 3, 1, 0, 8, 2, 0, 0, 3976, 0, 0, 0, 0, 0, 58, 0, 0, 4042, 0, 0, 31, 1, 3358, 0, 0, 0, 0, 0, 0, 2667, 3767, 3287, 1, 0, 0, 5, 3351, 0, 0, 0, 1260, 2832, 0, 2582, 0, 0, 0, 0, 32, 280, 0, 0, 879, 0, 2, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 2, 0, 0, 136, 0, 0, 0, 0, 0, 0, 4038, 0, 0, 0, 0, 0, 2, 0, 0, 3998, 0, 0, 171, 28, 989, 0, 1, 0, 0, 0, 0, 2286, 4031, 4077, 1155, 0, 0, 0, 1296, 0, 0, 0, 1451, 3776, 0, 989, 0, 0, 2, 0, 0, 55, 0, 0, 3569, 0, 0, 0, 715, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 173, 0, 0, 1, 202, 1, 0, 54, 0, 0, 0, 4030, 0, 0, 0, 0, 0, 5, 0, 6, 4007, 0, 0, 2, 173, 3022, 0, 2, 0, 0, 0, 0, 3746, 376, 4050, 216, 0, 0, 24, 4062, 0, 0, 0, 293, 3989, 0, 1420, 0, 0, 0, 0, 0, 1736, 0, 0, 930, 0, 0, 0, 244, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 16, 0, 0, 0, 279, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3930, 0, 0, 0, 57, 1348, 0, 0, 0, 0, 0, 0, 3457, 2914, 4073, 169, 0, 0, 0, 3305, 0, 0, 0, 1088, 3525, 0, 842, 0, 0, 0, 0, 0, 16, 0, 0, 3087, 0, 0, 0, 563, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 0, 0, 323, 10, 3781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3894, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4090, 42, 19, 0, 0, 0, 0, 0, 0, 0, 0, 137, 58, 2125, 0, 0, 0, 2003, 155, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1422, 775, 0, 873, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3907, 0, 0, 0, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3509, 0, 0, 10, 0, 0, 209, 1733, 159, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 0, 0, 0, 734, 15, 1487, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 3960, 4152, 4478, 3072, 4098, 5620, 3269] avg=4096.0000 max=5620 max_rank=6 min=3072 min_rank=4 imbalance_ratio=1.372070 max_load_reduction=27148 max_load_reduction_pct=0.828491 moved_to_echo=16059 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1374, 378, 2505, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 4093, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4085, 33, 21, 0, 0, 0, 0, 0, 0, 9, 0, 65, 40, 2476, 0, 0, 0, 2191, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 572, 847, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3104, 0, 0, 0, 1294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3249, 0, 0, 0, 0, 0, 1673, 1893, 174, 0, 0, 0, 0, 0, 0, 0, 0, 755, 0, 1, 0, 0, 2, 1628, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4257, 4168, 4147, 4883, 1538, 4400, 6989, 2386] avg=4096.0000 max=6989 max_rank=6 min=1538 min_rank=4 imbalance_ratio=1.706299 max_load_reduction=25779 max_load_reduction_pct=0.786713 moved_to_echo=15313 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 505, 3916, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4018, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3940, 40, 32, 0, 0, 0, 0, 0, 0, 0, 0, 40, 32, 635, 0, 0, 0, 2117, 160, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3416, 1300, 0, 1723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3686, 0, 0, 0, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 0, 0, 3, 0, 1, 23, 1829, 267, 0, 0, 0, 0, 0, 0, 0, 0, 963, 0, 0, 0, 31, 1, 2497, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4422, 4081, 4013, 2984, 6440, 4716, 2620, 3492] avg=4096.0000 max=6440 max_rank=4 min=2620 min_rank=6 imbalance_ratio=1.572266 max_load_reduction=26328 max_load_reduction_pct=0.803467 moved_to_echo=17268 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 2, 0, 0, 136, 3916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3998, 13, 32, 0, 0, 0, 0, 0, 0, 0, 0, 171, 28, 989, 0, 1, 0, 2024, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2286, 1279, 0, 1155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1262, 0, 0, 0, 1451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 989, 0, 0, 2, 0, 0, 55, 1752, 262, 0, 0, 0, 0, 0, 0, 0, 0, 3569, 0, 0, 0, 715, 0, 2458, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4055, 4059, 4045, 3374, 4720, 2713, 3060, 6742] avg=4096.0000 max=6742 max_rank=7 min=2713 min_rank=5 imbalance_ratio=1.645996 max_load_reduction=26026 max_load_reduction_pct=0.794250 moved_to_echo=17235 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 242, 0, 0, 101, 3, 3157, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8, 2, 0, 0, 3976, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 4042, 35, 30, 0, 0, 0, 0, 0, 0, 0, 0, 31, 1, 3358, 0, 0, 0, 1518, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2667, 1195, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3261, 0, 0, 0, 1260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2582, 0, 0, 0, 0, 32, 280, 1314, 245, 0, 0, 0, 0, 0, 0, 0, 0, 879, 0, 2, 0, 0, 0, 2297, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3503, 4042, 4165, 5038, 3863, 4526, 4453, 3178] avg=4096.0000 max=5038 max_rank=3 min=3178 min_rank=7 imbalance_ratio=1.229980 max_load_reduction=27730 max_load_reduction_pct=0.846252 moved_to_echo=16020 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 3, 0, 0, 0, 0, 38, 3895, 0, 0, 0, 0, 0, 0, 0, 0, 1, 12, 0, 5, 12, 4060, 0, 63, 0, 0, 0, 0, 0, 0, 0, 7, 1, 21, 0, 3, 0, 0, 4090, 40, 25, 0, 0, 0, 0, 0, 0, 0, 6, 491, 185, 364, 4, 5, 2, 1841, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2824, 1006, 0, 662, 21, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 3731, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 304, 0, 0, 0, 0, 0, 209, 1592, 206, 0, 0, 0, 0, 0, 0, 0, 0, 3988, 16, 0, 2, 922, 0, 1933, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3938, 4153, 4187, 3057, 4513, 3748, 2311, 6861] avg=4096.0000 max=6861 max_rank=7 min=2311 min_rank=6 imbalance_ratio=1.675049 max_load_reduction=25907 max_load_reduction_pct=0.790619 moved_to_echo=17433 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 173, 0, 0, 1, 202, 3891, 0, 0, 0, 0, 0, 0, 0, 1, 0, 54, 0, 0, 0, 4030, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 6, 4007, 47, 7, 0, 0, 0, 0, 0, 0, 0, 0, 2, 173, 3022, 0, 2, 0, 2140, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3746, 110, 0, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 3945, 0, 0, 0, 293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1420, 0, 0, 0, 0, 0, 1736, 1849, 27, 0, 0, 0, 0, 0, 0, 0, 0, 930, 0, 0, 0, 244, 4, 232, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4267, 4155, 4072, 5498, 4072, 4262, 5032, 1410] avg=4096.0000 max=5498 max_rank=3 min=1410 min_rank=7 imbalance_ratio=1.342285 max_load_reduction=27270 max_load_reduction_pct=0.832214 moved_to_echo=14776 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 16, 0, 0, 0, 279, 3912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3930, 34, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 1348, 0, 0, 0, 1890, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3457, 924, 0, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3217, 0, 0, 0, 1088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 842, 0, 0, 0, 0, 0, 16, 1635, 190, 0, 0, 0, 0, 0, 0, 0, 0, 3087, 0, 0, 0, 563, 3, 1777, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4207, 4150, 3987, 3456, 4550, 4305, 2683, 5430] avg=4096.0000 max=5430 max_rank=7 min=2683 min_rank=6 imbalance_ratio=1.325684 max_load_reduction=27338 max_load_reduction_pct=0.834290 moved_to_echo=16968 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 0, 15232, 128, 128, 128, 3328, 128, 14336, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 1024, 640, 14336, 128, 128, 128, 15744, 1280, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 0, 20480, 7552, 0, 4992, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 13440, 0, 0, 128, 0, 128, 4224, 13696, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 26240, 0, 0, 128, 6656, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 0, 128, 512, 128, 0, 1920, 1664, 29056, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 128, 0, 128, 32256, 384, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 128, 32256, 0, 512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 4068, 0, 1623, 0, 0, 4096, 0, 3979, 4074, 0, 0, 1, 0, 0, 0, 625, 0, 0, 4096, 0, 0, 0, 962, 0, 0, 82, 0, 843, 0, 0, 0, 0, 0, 0, 4093, 11, 0, 0, 4090, 0, 0, 0, 2, 113, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 4081, 0, 9, 0, 0, 4091, 8, 3874, 3639, 0, 0, 21, 0, 0, 0, 218, 0, 0, 4093, 0, 0, 0, 2558, 0, 0, 33, 0, 1428, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 0, 0, 0, 218, 308, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 4079, 0, 189, 0, 0, 4085, 2, 3956, 4092, 0, 0, 42, 0, 0, 0, 81, 0, 0, 4057, 0, 0, 0, 3410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 65, 0, 0, 3994, 0, 0, 0, 507, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 4068, 0, 876, 0, 0, 4096, 8, 3970, 1833, 0, 0, 483, 0, 0, 0, 297, 0, 0, 4096, 0, 0, 0, 2827, 0, 0, 3, 0, 366, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 97, 1534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 3915, 0, 24, 0, 0, 4093, 3, 4040, 4096, 0, 0, 1907, 0, 0, 0, 128, 0, 0, 4093, 0, 0, 0, 1191, 0, 0, 7, 0, 807, 0, 0, 0, 0, 0, 0, 4061, 11, 0, 0, 4044, 0, 0, 0, 75, 113, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 107, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 3918, 0, 19, 0, 0, 4085, 0, 4094, 4087, 0, 0, 124, 0, 0, 4, 111, 0, 0, 4095, 0, 0, 0, 3899, 0, 0, 6, 0, 71, 0, 0, 0, 0, 0, 0, 4050, 78, 0, 0, 3768, 3, 0, 0, 11, 41, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 298, 0, 0, 0, 0, 2, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 4087, 11, 820, 16, 5, 4087, 0, 4025, 3374, 0, 0, 15, 0, 10, 0, 123, 1, 0, 4064, 0, 0, 0, 3817, 0, 0, 59, 0, 580, 11, 0, 8, 11, 0, 0, 4017, 12, 14, 9, 3339, 0, 0, 0, 65, 96, 0, 0, 0, 1, 15, 0, 13, 47, 6, 0, 1, 1, 0, 3, 0, 0, 0, 5, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 0, 69, 0, 0, 4096, 1, 3859, 4042, 0, 0, 28, 0, 0, 0, 554, 0, 0, 4094, 0, 0, 0, 3587, 0, 0, 0, 0, 290, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 0, 3975, 0, 0, 0, 11, 11, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3662, 0, 1623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 4074, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 625, 0, 0, 1050, 0, 0, 0, 962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 843, 0, 0, 0, 3046, 406, 79, 0, 0, 0, 0, 0, 0, 0, 0, 147, 11, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 113, 0, 0, 0, 3698, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8, 3943, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5285, 4189, 2637, 4456, 4248, 3813, 4041, 4099] avg=4096.0000 max=5285 max_rank=0 min=2637 min_rank=2 imbalance_ratio=1.290283 max_load_reduction=27483 max_load_reduction_pct=0.838715 moved_to_echo=16201 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3525, 0, 24, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 115, 4096, 0, 0, 1907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 1050, 0, 0, 0, 1191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 807, 0, 0, 0, 3043, 390, 81, 0, 0, 0, 0, 0, 0, 0, 0, 146, 11, 0, 0, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 113, 0, 0, 0, 3754, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 107, 3912, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3552, 6118, 2369, 4328, 4201, 3942, 4162, 4096] avg=4096.0000 max=6118 max_rank=1 min=2369 min_rank=2 imbalance_ratio=1.493652 max_load_reduction=26650 max_load_reduction_pct=0.813293 moved_to_echo=16401 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3672, 0, 189, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 113, 4092, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 1040, 0, 0, 0, 3410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3017, 407, 79, 0, 0, 0, 0, 0, 0, 0, 0, 146, 65, 0, 0, 3994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 7, 0, 0, 0, 3676, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 3934, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3863, 4247, 4531, 3503, 4205, 4190, 4141, 4088] avg=4096.0000 max=4531 max_rank=2 min=3503 min_rank=3 imbalance_ratio=1.106201 max_load_reduction=28237 max_load_reduction_pct=0.861725 moved_to_echo=16624 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3674, 0, 9, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 111, 3639, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 218, 0, 0, 1050, 0, 0, 0, 2558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 1428, 0, 0, 0, 3043, 407, 77, 0, 0, 0, 0, 0, 0, 0, 0, 147, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 218, 308, 0, 0, 0, 3600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3946, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3691, 3771, 3826, 4988, 4239, 4126, 4033, 4094] avg=4096.0000 max=4988 max_rank=3 min=3691 min_rank=0 imbalance_ratio=1.217773 max_load_reduction=27780 max_load_reduction_pct=0.847778 moved_to_echo=16492 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3527, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 4087, 0, 0, 124, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 111, 0, 0, 1050, 0, 0, 0, 3899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 71, 0, 0, 0, 3045, 391, 82, 0, 0, 0, 0, 0, 0, 0, 0, 145, 78, 0, 0, 3768, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 11, 41, 0, 0, 4, 3804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, 3902, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4085, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3546, 4332, 5060, 3595, 3991, 3863, 4291, 4090] avg=4096.0000 max=5060 max_rank=2 min=3546 min_rank=0 imbalance_ratio=1.235352 max_load_reduction=27708 max_load_reduction_pct=0.845581 moved_to_echo=16235 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3679, 11, 820, 16, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 3374, 0, 0, 15, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 1, 0, 1042, 0, 0, 0, 3817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 580, 11, 0, 8, 3022, 408, 80, 0, 0, 0, 0, 0, 11, 0, 0, 144, 12, 14, 9, 3339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 96, 0, 0, 0, 3740, 0, 0, 0, 0, 0, 0, 0, 1, 15, 0, 13, 47, 6, 0, 1, 3870, 90, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 5, 0, 4087, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4531, 3514, 4983, 4168, 3529, 3901, 4043, 4099] avg=4096.0000 max=4983 max_rank=2 min=3514 min_rank=1 imbalance_ratio=1.216553 max_load_reduction=27785 max_load_reduction_pct=0.847931 moved_to_echo=15572 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3659, 0, 876, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 101, 1833, 0, 0, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 297, 0, 0, 1047, 0, 0, 0, 2827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 366, 0, 0, 0, 3049, 409, 83, 0, 0, 0, 0, 0, 0, 0, 0, 139, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 1534, 0, 0, 0, 3694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 3950, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4543, 2417, 4171, 3910, 4235, 5325, 4064, 4103] avg=4096.0000 max=5325 max_rank=5 min=2417 min_rank=1 imbalance_ratio=1.300049 max_load_reduction=27443 max_load_reduction_pct=0.837494 moved_to_echo=17727 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3687, 0, 69, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 110, 4042, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 0, 0, 1050, 0, 0, 0, 3587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 290, 0, 0, 0, 3044, 409, 77, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 0, 0, 3975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 11, 0, 0, 0, 3586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3906, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3757, 4180, 5191, 3820, 4120, 3608, 3993, 4099] avg=4096.0000 max=5191 max_rank=2 min=3608 min_rank=5 imbalance_ratio=1.267334 max_load_reduction=27577 max_load_reduction_pct=0.841583 moved_to_echo=15820 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[2176, 128, 0, 8448, 0, 0, 0, 22272, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 256, 0, 4480, 128, 0, 128, 24320, 3328, 640, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 29184, 128, 3712, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[896, 29312, 0, 0, 2688, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 0, 1024, 2304, 0, 0, 128, 29568, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 0, 1280, 256, 128, 128, 31488, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 128, 0, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 128, 0, 640, 31488, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 31, 12, 0, 0, 0, 2983, 1, 15, 54, 0, 0, 1840, 3252, 0, 16, 1, 0, 0, 0, 4081, 0, 0, 140, 0, 0, 0, 0, 1883, 0, 0, 0, 0, 0, 0, 0, 997, 10, 0, 4096, 0, 0, 0, 0, 4086, 0, 4061, 0, 0, 0, 0, 1127, 4082, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 232, 0, 0, 0, 0, 3463, 1, 3, 1304, 0, 0, 238, 743, 0, 89, 0, 0, 0, 0, 4092, 0, 0, 209, 0, 0, 0, 0, 1121, 0, 0, 0, 0, 0, 0, 0, 521, 306, 0, 4096, 0, 0, 0, 0, 4096, 0, 4079, 0, 0, 0, 0, 4087, 4088, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 651, 139, 0, 0, 1, 2767, 0, 10, 54, 0, 0, 17, 2657, 0, 105, 3, 0, 0, 0, 4088, 0, 0, 14, 0, 0, 0, 0, 324, 0, 0, 0, 0, 0, 0, 0, 2496, 418, 0, 4096, 0, 0, 0, 0, 4081, 0, 4073, 0, 0, 0, 0, 2679, 4095, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2908, 20, 0, 0, 0, 1055, 0, 79, 2625, 0, 0, 243, 1490, 0, 32, 0, 0, 0, 0, 4050, 0, 0, 89, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 4095, 0, 0, 0, 0, 3999, 0, 4049, 0, 0, 0, 0, 4038, 3897, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 549, 71, 0, 0, 0, 3832, 0, 0, 0, 0, 0, 368, 1818, 0, 270, 0, 0, 0, 13, 4022, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 2101, 1, 0, 4096, 1, 0, 0, 0, 4081, 0, 4092, 0, 0, 0, 0, 3800, 3628, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 23, 0, 6, 320, 0, 0, 0, 0, 4059, 71, 0, 343, 0, 0, 1828, 1793, 6, 377, 0, 0, 0, 0, 4065, 18, 0, 8, 7, 0, 0, 0, 215, 53, 5, 0, 8, 0, 0, 0, 66, 2, 0, 4024, 0, 7, 15, 0, 3692, 0, 4067, 0, 7, 3, 0, 4077, 3577, 0, 3, 0, 3, 9, 7, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 474, 7, 0, 0, 0, 3951, 0, 57, 522, 0, 0, 792, 2927, 0, 13, 0, 0, 0, 0, 4080, 0, 0, 620, 0, 0, 0, 0, 131, 0, 0, 0, 0, 0, 0, 0, 6, 1, 0, 4096, 0, 0, 0, 0, 3955, 0, 4093, 0, 0, 0, 0, 3185, 3858, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 2911, 0, 14, 423, 0, 0, 1790, 1935, 0, 282, 1, 0, 0, 0, 3872, 0, 0, 436, 0, 0, 0, 0, 162, 0, 0, 0, 0, 0, 0, 0, 569, 31, 0, 4096, 0, 0, 0, 0, 4086, 0, 4059, 0, 0, 0, 0, 3907, 4094, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 31, 12, 0, 2992, 465, 0, 0, 0, 0, 0, 0, 0, 0, 2983, 1, 15, 54, 0, 0, 271, 0, 0, 0, 0, 0, 0, 0, 1840, 3252, 0, 16, 1, 0, 0, 0, 639, 347, 0, 0, 0, 0, 0, 0, 3940, 0, 0, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1883, 0, 0, 0, 0, 0, 0, 0, 3596, 0, 0, 0, 0, 0, 0, 0, 997, 10, 0, 3149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 747, 0, 0, 0, 0, 0, 0, 1127, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 141, 37, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3500, 3324, 6095, 4080, 5479, 4156, 1874, 4260] avg=4096.0000 max=6095 max_rank=2 min=1874 min_rank=6 imbalance_ratio=1.488037 max_load_reduction=26673 max_load_reduction_pct=0.813995 moved_to_echo=15769 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 232, 0, 0, 2999, 467, 0, 0, 0, 0, 0, 0, 0, 0, 3463, 1, 3, 1304, 0, 0, 271, 0, 0, 0, 0, 0, 0, 0, 238, 743, 0, 89, 0, 0, 0, 0, 639, 348, 0, 0, 0, 0, 0, 0, 3950, 0, 0, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1121, 0, 0, 0, 0, 0, 0, 0, 3612, 0, 0, 0, 0, 0, 0, 0, 521, 306, 0, 3149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 749, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 142, 37, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3698, 5042, 2057, 4159, 4733, 3976, 4836, 4267] avg=4096.0000 max=5042 max_rank=1 min=2057 min_rank=2 imbalance_ratio=1.230957 max_load_reduction=27726 max_load_reduction_pct=0.846130 moved_to_echo=17812 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 651, 139, 0, 2988, 467, 0, 0, 0, 0, 0, 0, 0, 1, 2767, 0, 10, 54, 0, 0, 271, 0, 0, 0, 0, 0, 0, 0, 17, 2657, 0, 105, 3, 0, 0, 0, 639, 346, 0, 0, 0, 0, 0, 0, 3947, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 324, 0, 0, 0, 0, 0, 0, 0, 3606, 0, 0, 0, 0, 0, 0, 0, 2496, 418, 0, 3149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 747, 0, 0, 0, 0, 0, 0, 2679, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 141, 37, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4245, 3103, 3767, 3961, 3930, 6063, 3426, 4273] avg=4096.0000 max=6063 max_rank=5 min=3103 min_rank=1 imbalance_ratio=1.480225 max_load_reduction=26705 max_load_reduction_pct=0.814972 moved_to_echo=17692 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 549, 71, 0, 2988, 468, 0, 0, 0, 0, 0, 0, 0, 0, 3832, 0, 0, 0, 0, 0, 271, 0, 0, 0, 0, 0, 0, 0, 368, 1818, 0, 270, 0, 0, 0, 13, 639, 346, 0, 0, 0, 0, 0, 0, 3883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 3624, 0, 0, 0, 0, 0, 0, 0, 2101, 1, 0, 3149, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 747, 0, 0, 0, 0, 0, 0, 3800, 0, 0, 0, 0, 0, 0, 0, 0, 3628, 0, 0, 0, 0, 0, 0, 0, 139, 37, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4103, 3454, 3883, 3649, 5252, 4547, 3804] avg=4096.0000 max=5252 max_rank=5 min=3454 min_rank=2 imbalance_ratio=1.282227 max_load_reduction=27516 max_load_reduction_pct=0.839722 moved_to_echo=17252 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2908, 20, 0, 2928, 465, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 0, 79, 2625, 0, 0, 271, 0, 0, 0, 0, 0, 0, 0, 243, 1490, 0, 32, 0, 0, 0, 0, 639, 339, 0, 0, 0, 0, 0, 0, 3910, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3584, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 3148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 732, 0, 0, 0, 0, 0, 0, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 3897, 0, 0, 0, 0, 0, 0, 0, 140, 37, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6321, 4030, 2743, 3999, 3585, 3246, 4770, 4074] avg=4096.0000 max=6321 max_rank=0 min=2743 min_rank=2 imbalance_ratio=1.543213 max_load_reduction=26447 max_load_reduction_pct=0.807098 moved_to_echo=15675 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 23, 0, 6, 320, 0, 0, 2703, 466, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 71, 0, 343, 0, 0, 267, 0, 0, 0, 0, 0, 0, 0, 1828, 1793, 6, 377, 0, 0, 0, 0, 628, 313, 0, 0, 0, 0, 0, 0, 3924, 18, 0, 8, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 215, 53, 5, 0, 8, 0, 0, 0, 3601, 0, 0, 0, 0, 0, 0, 0, 66, 2, 0, 3093, 0, 7, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 676, 0, 0, 0, 7, 3, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 3577, 0, 3, 0, 3, 9, 7, 4, 141, 36, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3518, 4740, 4945, 3957, 3882, 3183, 4763, 3780] avg=4096.0000 max=4945 max_rank=2 min=3183 min_rank=5 imbalance_ratio=1.207275 max_load_reduction=27823 max_load_reduction_pct=0.849091 moved_to_echo=15608 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 474, 7, 0, 2902, 470, 0, 0, 0, 0, 0, 0, 0, 0, 3951, 0, 57, 522, 0, 0, 277, 0, 0, 0, 0, 0, 0, 0, 792, 2927, 0, 13, 0, 0, 0, 0, 641, 340, 0, 0, 0, 0, 0, 0, 3935, 0, 0, 620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 0, 0, 0, 0, 0, 0, 0, 3623, 0, 0, 0, 0, 0, 0, 0, 6, 1, 0, 3136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 713, 0, 0, 0, 0, 0, 0, 3185, 0, 0, 0, 0, 0, 0, 0, 0, 3858, 0, 0, 0, 0, 0, 0, 0, 145, 42, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3853, 4807, 4713, 4555, 3754, 3143, 3898, 4045] avg=4096.0000 max=4807 max_rank=1 min=3143 min_rank=5 imbalance_ratio=1.173584 max_load_reduction=27961 max_load_reduction_pct=0.853302 moved_to_echo=14840 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 100, 0, 0, 2992, 465, 0, 0, 0, 0, 0, 0, 0, 0, 2911, 0, 14, 423, 0, 0, 271, 0, 0, 0, 0, 0, 0, 0, 1790, 1935, 0, 282, 1, 0, 0, 0, 639, 347, 0, 0, 0, 0, 0, 0, 3738, 0, 0, 436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 0, 0, 0, 0, 0, 0, 0, 3594, 0, 0, 0, 0, 0, 0, 0, 569, 31, 0, 3149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 747, 0, 0, 0, 0, 0, 0, 3907, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 134, 37, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3557, 3619, 4994, 4174, 3756, 3749, 4654, 4265] avg=4096.0000 max=4994 max_rank=2 min=3557 min_rank=0 imbalance_ratio=1.219238 max_load_reduction=27774 max_load_reduction_pct=0.847595 moved_to_echo=16424 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 128, 5376, 256, 0, 23552, 3840, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[7168, 16640, 128, 1280, 128, 0, 0, 128, 5120, 2816, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[5888, 0, 0, 0, 128, 128, 0, 27008, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[6912, 896, 0, 25216, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3968, 128, 128, 0, 128, 0, 0, 0, 28928, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 25088, 128, 256, 5376, 0, 0, 2176, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[31360, 0, 128, 0, 128, 128, 128, 128, 1152, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[31232, 128, 0, 1536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 12, 0, 9, 0, 368, 3, 0, 1763, 0, 0, 4009, 0, 0, 0, 0, 0, 490, 0, 3815, 0, 1, 0, 0, 3950, 3, 2145, 0, 0, 0, 0, 197, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4081, 0, 0, 0, 4096, 0, 15, 3713, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 413, 0, 13, 0, 4050, 280, 0, 390, 0, 0, 4029, 0, 0, 0, 0, 0, 38, 0, 3397, 0, 16, 0, 0, 4027, 4, 297, 0, 0, 0, 0, 225, 0, 0, 0, 0, 0, 0, 0, 3326, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 0, 2, 3992, 0, 0, 3, 0, 0, 76, 0, 2, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 8, 1, 0, 0, 3203, 9, 0, 671, 0, 3, 4033, 0, 0, 1, 0, 0, 1, 0, 4093, 0, 0, 1, 0, 4093, 60, 57, 0, 0, 0, 0, 505, 0, 0, 0, 1, 0, 0, 0, 4016, 0, 0, 0, 0, 3827, 0, 0, 0, 4096, 0, 3, 3448, 0, 0, 0, 0, 0, 633, 0, 5, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 1675, 26, 1, 0, 2195, 0, 0, 321, 41, 0, 4088, 0, 0, 0, 0, 0, 27, 0, 4080, 0, 1, 0, 0, 3942, 12, 16, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 4084, 0, 0, 0, 4096, 0, 0, 4005, 0, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 1072, 59, 24, 0, 1945, 0, 9, 935, 0, 0, 4001, 11, 0, 0, 11, 0, 1, 0, 3888, 0, 0, 0, 1, 3790, 0, 676, 0, 0, 2, 0, 423, 0, 0, 0, 96, 0, 0, 0, 3956, 0, 4, 11, 13, 3630, 0, 0, 9, 4058, 0, 15, 3992, 0, 0, 0, 12, 3, 10, 14, 0, 73, 6, 8, 10] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 105, 2, 0, 0, 2740, 2, 0, 838, 1, 0, 3903, 0, 0, 0, 0, 0, 520, 0, 3639, 0, 0, 0, 0, 3961, 0, 48, 0, 0, 0, 0, 699, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 4089, 0, 0, 0, 4096, 0, 0, 3947, 0, 0, 0, 0, 0, 97, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 107, 0, 0, 0, 2129, 1, 0, 1361, 0, 0, 4096, 0, 0, 0, 0, 0, 181, 0, 4029, 0, 0, 0, 0, 4077, 63, 96, 0, 0, 0, 0, 263, 0, 0, 0, 3, 0, 0, 0, 4085, 0, 0, 0, 0, 4031, 0, 0, 0, 4091, 0, 9, 4017, 0, 0, 1, 0, 0, 128, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 310, 0, 3, 0, 2736, 0, 0, 2435, 0, 0, 4083, 0, 0, 0, 0, 0, 6, 0, 3684, 0, 0, 0, 0, 3661, 118, 94, 0, 0, 0, 0, 679, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 4051, 0, 0, 0, 4096, 0, 0, 2622, 0, 0, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 12, 0, 9, 0, 368, 3, 0, 803, 350, 0, 0, 0, 0, 0, 0, 1763, 0, 0, 2984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 0, 3815, 0, 1, 0, 0, 107, 0, 0, 0, 0, 0, 0, 0, 3272, 3, 2145, 0, 0, 0, 0, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 373, 0, 15, 3713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3723, 222, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1545, 4747, 4413, 5617, 4211, 4186, 4101, 3948] avg=4096.0000 max=5617 max_rank=3 min=1545 min_rank=0 imbalance_ratio=1.371338 max_load_reduction=27151 max_load_reduction_pct=0.828583 moved_to_echo=16446 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 105, 2, 0, 0, 2740, 2, 0, 782, 351, 0, 0, 0, 0, 0, 0, 838, 1, 0, 2905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 0, 3639, 0, 0, 0, 0, 108, 0, 0, 0, 0, 0, 0, 0, 3281, 0, 48, 0, 0, 0, 0, 699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 373, 0, 0, 3947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 0, 0, 0, 0, 3723, 216, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3982, 3744, 4267, 4028, 4197, 4194, 4320, 4036] avg=4096.0000 max=4320 max_rank=6 min=3744 min_rank=1 imbalance_ratio=1.054688 max_load_reduction=28448 max_load_reduction_pct=0.868164 moved_to_echo=16747 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 413, 0, 13, 0, 4050, 280, 0, 807, 357, 0, 0, 0, 0, 0, 0, 390, 0, 0, 2999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 3397, 0, 16, 0, 0, 109, 0, 0, 0, 0, 0, 0, 0, 3336, 4, 297, 0, 0, 0, 0, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3326, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 107, 0, 0, 0, 0, 0, 0, 0, 372, 0, 2, 3992, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 2, 0, 0, 0, 0, 3720, 223, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5920, 3389, 3560, 3862, 3444, 4203, 4369, 4021] avg=4096.0000 max=5920 max_rank=0 min=3389 min_rank=1 imbalance_ratio=1.445312 max_load_reduction=26848 max_load_reduction_pct=0.819336 moved_to_echo=16037 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 1, 0, 0, 3203, 9, 0, 808, 363, 0, 0, 0, 0, 0, 0, 671, 0, 3, 3001, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4093, 0, 0, 1, 0, 111, 0, 0, 0, 0, 0, 0, 0, 3390, 60, 57, 0, 0, 0, 0, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4016, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3827, 0, 0, 0, 109, 0, 0, 0, 0, 0, 0, 0, 373, 0, 3, 3448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 633, 0, 5, 0, 0, 0, 0, 3723, 224, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4392, 3676, 4206, 4012, 4137, 3936, 3824, 4585] avg=4096.0000 max=4585 max_rank=7 min=3676 min_rank=1 imbalance_ratio=1.119385 max_load_reduction=28183 max_load_reduction_pct=0.860077 moved_to_echo=16482 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1675, 26, 1, 0, 2195, 0, 0, 819, 349, 0, 0, 0, 0, 0, 0, 321, 41, 0, 3042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 4080, 0, 1, 0, 0, 107, 0, 0, 0, 0, 0, 0, 0, 3266, 12, 16, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 373, 0, 0, 4005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 3723, 227, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5065, 3404, 4215, 3300, 4201, 4189, 4378, 4016] avg=4096.0000 max=5065 max_rank=0 min=3300 min_rank=3 imbalance_ratio=1.236572 max_load_reduction=27703 max_load_reduction_pct=0.845428 moved_to_echo=16784 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1072, 59, 24, 0, 1945, 0, 9, 802, 336, 0, 0, 0, 0, 0, 0, 935, 0, 0, 2977, 11, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3888, 0, 0, 0, 1, 103, 0, 0, 0, 0, 0, 0, 0, 3139, 0, 676, 0, 0, 2, 0, 423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 3956, 111, 0, 0, 0, 0, 0, 0, 0, 0, 4, 11, 13, 3630, 0, 0, 9, 101, 0, 0, 0, 0, 0, 0, 0, 369, 0, 15, 3992, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 3, 10, 14, 0, 73, 6, 8, 10, 3689, 222, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4247, 3934, 3993, 4240, 4163, 3768, 4388, 4035] avg=4096.0000 max=4388 max_rank=6 min=3768 min_rank=5 imbalance_ratio=1.071289 max_load_reduction=28380 max_load_reduction_pct=0.866089 moved_to_echo=16354 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 107, 0, 0, 0, 2129, 1, 0, 825, 364, 0, 0, 0, 0, 0, 0, 1361, 0, 0, 3039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 0, 4029, 0, 0, 0, 0, 115, 0, 0, 0, 0, 0, 0, 0, 3362, 63, 96, 0, 0, 0, 0, 263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4085, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4031, 0, 0, 0, 113, 0, 0, 0, 0, 0, 0, 0, 366, 0, 9, 4017, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 3725, 232, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3426, 4400, 4325, 3784, 4211, 4144, 4393, 4085] avg=4096.0000 max=4400 max_rank=1 min=3426 min_rank=0 imbalance_ratio=1.074219 max_load_reduction=28368 max_load_reduction_pct=0.865723 moved_to_echo=16833 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 310, 0, 3, 0, 2736, 0, 0, 818, 324, 0, 0, 0, 0, 0, 0, 2435, 0, 0, 3039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 3684, 0, 0, 0, 0, 99, 0, 0, 0, 0, 0, 0, 0, 3034, 118, 94, 0, 0, 0, 0, 679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 0, 0, 0, 97, 0, 0, 0, 0, 0, 0, 0, 373, 0, 0, 2622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 3723, 226, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4191, 5474, 3789, 3925, 4204, 4148, 2995, 4042] avg=4096.0000 max=5474 max_rank=1 min=2995 min_rank=6 imbalance_ratio=1.336426 max_load_reduction=27294 max_load_reduction_pct=0.832947 moved_to_echo=15389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 3712, 128, 128, 0, 19456, 384, 128, 6528, 2816, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[8832, 128, 128, 24064, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[26112, 384, 3456, 0, 0, 128, 0, 3072, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 1152, 128, 128, 128, 128, 128, 128, 29824, 1792, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 1280, 0, 30720, 0, 128, 128, 128, 896, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 0, 128, 0, 0, 0, 31744, 1024, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3072, 0, 128, 29824, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 32000, 0, 0, 128, 896, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1262, 0, 4096, 542, 0, 0, 4096, 0, 4008, 0, 0, 0, 1, 3951, 0, 0, 4050, 3274, 0, 0, 3188, 0, 0, 0, 4, 0, 946, 0, 81, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 27, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1694, 0, 4096, 91, 0, 0, 4096, 0, 3957, 0, 0, 1, 8, 4096, 0, 0, 4002, 1750, 0, 0, 1708, 0, 0, 0, 83, 0, 3488, 0, 12, 577, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1135, 0, 4096, 6, 0, 0, 4096, 0, 4085, 0, 0, 0, 0, 4096, 0, 0, 4016, 890, 0, 0, 3296, 0, 0, 0, 0, 0, 3749, 0, 178, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4040, 0, 4089, 181, 0, 0, 4096, 0, 3704, 0, 0, 0, 0, 3500, 0, 0, 3999, 2779, 0, 0, 1196, 0, 0, 0, 62, 0, 2185, 0, 756, 935, 0, 0, 0, 0, 0, 0, 0, 7, 0, 1229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1943, 0, 4095, 1767, 0, 0, 4096, 0, 3982, 0, 0, 0, 3, 4096, 0, 0, 3764, 1, 0, 0, 2646, 0, 0, 0, 1, 0, 3054, 0, 277, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2897, 0, 4085, 271, 0, 0, 4075, 11, 2237, 9, 0, 0, 0, 4068, 5, 0, 4066, 904, 1, 1, 2345, 0, 0, 0, 2, 10, 3794, 20, 29, 0, 3, 0, 2, 0, 7, 2, 0, 0, 0, 3735, 0, 43, 13, 0, 0, 7, 12, 0, 11, 0, 10, 50, 0, 0, 0, 31, 0, 0, 12, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1383, 0, 4096, 1522, 0, 0, 4096, 0, 3881, 0, 0, 0, 0, 4096, 0, 0, 4057, 634, 0, 0, 3332, 0, 0, 0, 3, 0, 4077, 0, 10, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2927, 0, 4096, 30, 0, 0, 4096, 0, 4077, 0, 0, 0, 0, 4096, 0, 0, 4017, 954, 0, 0, 1051, 0, 0, 0, 0, 0, 3339, 0, 150, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1262, 0, 1386, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 353, 0, 4008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2733, 3274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3188, 0, 0, 0, 4, 0, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 114, 0, 0, 0, 0, 0, 0, 2282, 415, 44, 0, 0, 0, 0, 0, 0, 0, 0, 3120, 0, 0, 0, 0, 1273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3656, 428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 3951, 87, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3190, 4361, 6008, 3723, 2936, 4393, 4092, 4065] avg=4096.0000 max=6008 max_rank=2 min=2936 min_rank=4 imbalance_ratio=1.466797 max_load_reduction=26760 max_load_reduction_pct=0.816650 moved_to_echo=15486 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1694, 0, 1386, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 353, 0, 3957, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 2700, 1750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1708, 0, 0, 0, 83, 0, 1958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 577, 0, 0, 0, 0, 0, 0, 2282, 1530, 44, 0, 0, 0, 0, 0, 0, 1, 0, 3108, 0, 0, 0, 0, 1258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3656, 428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 87, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3171, 4311, 4458, 3749, 4445, 4367, 4084, 4183] avg=4096.0000 max=4458 max_rank=2 min=3171 min_rank=0 imbalance_ratio=1.088379 max_load_reduction=28310 max_load_reduction_pct=0.863953 moved_to_echo=17079 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4040, 0, 1384, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 353, 0, 3704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2698, 2779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1196, 0, 0, 0, 62, 0, 1227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 756, 935, 0, 0, 0, 0, 0, 0, 2278, 958, 44, 0, 0, 0, 0, 0, 0, 7, 0, 1229, 0, 0, 0, 0, 1257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3656, 427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3500, 87, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5605, 4057, 5477, 2485, 4971, 2493, 4091, 3589] avg=4096.0000 max=5605 max_rank=0 min=2485 min_rank=3 imbalance_ratio=1.368408 max_load_reduction=27163 max_load_reduction_pct=0.828949 moved_to_echo=15144 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1135, 0, 1386, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 353, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2709, 890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3296, 0, 0, 0, 0, 0, 2104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 33, 0, 0, 0, 0, 0, 0, 2282, 1645, 44, 0, 0, 0, 0, 0, 0, 0, 0, 3092, 0, 0, 0, 0, 1263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3656, 428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 87, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2527, 4438, 3599, 5400, 4182, 4355, 4084, 4183] avg=4096.0000 max=5400 max_rank=3 min=2527 min_rank=0 imbalance_ratio=1.318359 max_load_reduction=27368 max_load_reduction_pct=0.835205 moved_to_echo=16804 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1943, 0, 1386, 1767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 353, 0, 3982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 2540, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2646, 0, 0, 0, 1, 0, 1714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 277, 1, 0, 0, 0, 0, 0, 0, 2281, 1340, 41, 0, 0, 0, 0, 0, 0, 0, 0, 3041, 0, 0, 0, 0, 1183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3656, 428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 87, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5096, 4335, 2544, 4361, 3940, 4224, 4085, 4183] avg=4096.0000 max=5096 max_rank=0 min=2544 min_rank=2 imbalance_ratio=1.244141 max_load_reduction=27672 max_load_reduction_pct=0.844482 moved_to_echo=16432 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2897, 0, 1382, 271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 352, 11, 2237, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2744, 904, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2345, 0, 0, 0, 2, 10, 2129, 20, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 3, 0, 2, 0, 7, 2, 2276, 1665, 44, 0, 0, 0, 0, 0, 0, 0, 0, 3735, 0, 43, 13, 0, 1278, 0, 0, 0, 0, 0, 0, 0, 0, 7, 12, 0, 11, 0, 10, 50, 3637, 427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 12, 0, 4068, 86, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4550, 2609, 3655, 4506, 4028, 5069, 4154, 4197] avg=4096.0000 max=5069 max_rank=5 min=2609 min_rank=1 imbalance_ratio=1.237549 max_load_reduction=27699 max_load_reduction_pct=0.845306 moved_to_echo=17448 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1383, 0, 1381, 1522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 346, 0, 3881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2729, 634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3332, 0, 0, 0, 3, 0, 2284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 112, 0, 0, 0, 0, 0, 0, 2284, 1793, 48, 0, 0, 0, 0, 0, 0, 0, 0, 1467, 0, 0, 0, 0, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3661, 431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 89, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4286, 4227, 3363, 5619, 4247, 2747, 4094, 4185] avg=4096.0000 max=5619 max_rank=3 min=2747 min_rank=5 imbalance_ratio=1.371826 max_load_reduction=27149 max_load_reduction_pct=0.828522 moved_to_echo=15273 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2927, 0, 1386, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 353, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2710, 954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 0, 0, 0, 0, 0, 1874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 78, 0, 0, 0, 0, 0, 0, 2282, 1465, 44, 0, 0, 0, 0, 0, 0, 0, 0, 3857, 0, 0, 0, 0, 1263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3656, 428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 87, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4343, 4430, 3664, 2925, 4019, 5120, 4084, 4183] avg=4096.0000 max=5120 max_rank=5 min=2925 min_rank=3 imbalance_ratio=1.250000 max_load_reduction=27648 max_load_reduction_pct=0.843750 moved_to_echo=17406 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[1536, 1920, 128, 0, 128, 0, 128, 128, 18304, 10880, 384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 21632, 11264, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[18816, 0, 0, 0, 256, 128, 13824, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 17408, 0, 11136, 4480, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 0, 22656, 0, 128, 128, 0, 10112, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 2816, 128, 29952, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 128, 0, 128, 128, 29312, 3456, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 0, 0, 128, 0, 32000, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[611, 174, 0, 2621, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 32, 3062, 0, 0, 0, 0, 3720, 0, 0, 3, 11, 78, 0, 0, 1445, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1506, 0, 3824, 0, 0, 3322, 0, 5, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 4093, 0, 50, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[412, 3139, 0, 4064, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 67, 3748, 0, 0, 0, 0, 3076, 0, 0, 155, 0, 154, 0, 0, 1399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 0, 1090, 0, 0, 2914, 0, 4, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 100, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[986, 974, 0, 3400, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 3777, 0, 0, 0, 0, 2176, 0, 0, 130, 1, 438, 0, 0, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1160, 0, 1648, 0, 0, 3978, 0, 9, 0, 1, 9, 0, 0, 0, 4080, 5, 0, 4074, 0, 1707, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[162, 219, 0, 3987, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3965, 0, 0, 0, 0, 3845, 0, 0, 10, 0, 15, 0, 0, 1473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 768, 0, 3367, 0, 0, 2438, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 4090, 0, 205, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1030, 87, 0, 3520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 3098, 0, 0, 0, 0, 3179, 0, 0, 1366, 1, 438, 0, 0, 29, 0, 0, 0, 1, 0, 0, 0, 0, 0, 110, 0, 3820, 0, 0, 3778, 0, 3, 0, 0, 1, 0, 0, 0, 4091, 0, 0, 4096, 0, 25, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[3100, 68, 0, 790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3298, 0, 3, 0, 0, 3410, 0, 0, 7, 0, 1201, 0, 0, 872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 3560, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 9, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 368, 9, 4012, 1, 12, 0, 0, 11, 52, 7, 0, 0, 0, 0, 13, 0, 4067, 0, 0, 4076, 0, 103, 0, 5, 3900, 0, 11, 6, 0, 574, 0, 0, 71, 0, 0, 0, 14, 0, 0, 4, 0, 0, 37, 0, 3928, 13, 0, 2563, 0, 0, 0, 0, 0, 0, 13, 0, 4077, 0, 0, 3970, 0, 847, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[553, 127, 0, 1369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3268, 0, 0, 0, 0, 4019, 0, 0, 0, 0, 1, 0, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2055, 0, 4062, 0, 0, 3676, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1235, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[986, 974, 0, 2969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 535, 0, 0, 3777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2176, 0, 0, 130, 1, 438, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 108, 0, 0, 0, 0, 0, 0, 3394, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1160, 0, 1648, 0, 0, 223, 0, 0, 0, 0, 0, 0, 0, 3978, 0, 9, 0, 1, 9, 0, 0, 503, 166, 89, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3571, 0, 1707, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4929, 4080, 4312, 2864, 3514, 3031, 4755, 5283] avg=4096.0000 max=5283 max_rank=7 min=2864 min_rank=3 imbalance_ratio=1.289795 max_load_reduction=27485 max_load_reduction_pct=0.838776 moved_to_echo=16583 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[611, 174, 0, 2289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 535, 0, 32, 3062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3720, 0, 0, 3, 11, 78, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 1445, 0, 0, 0, 0, 1, 0, 3395, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1506, 0, 3824, 0, 0, 172, 0, 0, 0, 0, 0, 0, 0, 3322, 0, 5, 0, 0, 0, 0, 0, 505, 166, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3588, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3074, 4101, 3629, 3904, 4854, 5502, 4066, 3638] avg=4096.0000 max=5502 max_rank=5 min=3074 min_rank=0 imbalance_ratio=1.343262 max_load_reduction=27266 max_load_reduction_pct=0.832092 moved_to_echo=18060 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1030, 87, 0, 3073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 535, 0, 0, 3098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3179, 0, 0, 1366, 1, 438, 0, 124, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 1, 0, 0, 3394, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 3820, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 3778, 0, 3, 0, 0, 1, 0, 0, 506, 166, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3590, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4190, 4078, 3633, 5108, 3437, 4161, 4546, 3615] avg=4096.0000 max=5108 max_rank=3 min=3437 min_rank=4 imbalance_ratio=1.247070 max_load_reduction=27660 max_load_reduction_pct=0.844116 moved_to_echo=15759 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3100, 68, 0, 692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 535, 0, 0, 3298, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3410, 0, 0, 7, 0, 1201, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 872, 0, 0, 0, 0, 0, 0, 3395, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 3560, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 506, 166, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3590, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3860, 4083, 3836, 4645, 4280, 3705, 4760, 3599] avg=4096.0000 max=4760 max_rank=6 min=3599 min_rank=7 imbalance_ratio=1.162109 max_load_reduction=28008 max_load_reduction_pct=0.854736 moved_to_echo=16344 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[412, 3139, 0, 3548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 535, 0, 67, 3748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3076, 0, 0, 155, 0, 154, 0, 143, 0, 0, 0, 0, 0, 0, 0, 0, 1399, 0, 0, 0, 0, 0, 0, 3395, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 0, 1090, 0, 0, 267, 0, 0, 0, 0, 0, 0, 0, 2914, 0, 4, 0, 0, 0, 0, 0, 506, 166, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3590, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7099, 4083, 4350, 3528, 4807, 1515, 3696, 3690] avg=4096.0000 max=7099 max_rank=0 min=1515 min_rank=5 imbalance_ratio=1.733154 max_load_reduction=25669 max_load_reduction_pct=0.783356 moved_to_echo=13708 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 368, 9, 3502, 1, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 52, 7, 0, 0, 0, 0, 13, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 532, 0, 0, 4076, 0, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3900, 0, 11, 6, 0, 574, 0, 141, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 14, 0, 0, 3371, 13, 0, 0, 0, 0, 0, 0, 4, 0, 0, 37, 0, 3928, 13, 0, 264, 0, 0, 0, 0, 0, 0, 0, 2563, 0, 0, 0, 0, 0, 0, 13, 490, 164, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3480, 0, 847, 4, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3892, 4147, 4711, 4637, 3469, 4246, 3335, 4331] avg=4096.0000 max=4711 max_rank=2 min=3335 min_rank=6 imbalance_ratio=1.150146 max_load_reduction=28057 max_load_reduction_pct=0.856232 moved_to_echo=15381 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[162, 219, 0, 3468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 529, 0, 0, 3965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3845, 0, 0, 10, 0, 15, 0, 144, 0, 0, 0, 0, 0, 0, 0, 0, 1473, 0, 0, 0, 0, 0, 0, 3399, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 768, 0, 3367, 0, 0, 266, 0, 0, 0, 0, 0, 0, 0, 2438, 0, 0, 0, 0, 3, 0, 0, 508, 168, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3582, 0, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3849, 4113, 4494, 4014, 4884, 4401, 3226, 3787] avg=4096.0000 max=4884 max_rank=4 min=3226 min_rank=6 imbalance_ratio=1.192383 max_load_reduction=27884 max_load_reduction_pct=0.850952 moved_to_echo=16298 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[553, 127, 0, 1195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 535, 0, 0, 3268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4019, 0, 0, 0, 0, 1, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 0, 0, 0, 0, 3395, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2055, 0, 4062, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 3676, 0, 0, 0, 0, 0, 0, 0, 506, 166, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3590, 0, 1235, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1875, 4083, 3803, 4068, 3523, 6207, 4384, 4825] avg=4096.0000 max=6207 max_rank=5 min=1875 min_rank=0 imbalance_ratio=1.515381 max_load_reduction=26561 max_load_reduction_pct=0.810577 moved_to_echo=18939 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 0, 5888, 0, 25344, 128, 0, 1664, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 5632, 0, 0, 0, 128, 128, 0, 27264, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 28672, 0, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[6912, 5248, 128, 20736, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 4352, 0, 128, 28416, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 27392, 0, 128, 1792, 128, 2944, 0, 896, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[26752, 0, 128, 0, 128, 128, 0, 128, 4096, 1408, 640, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 256, 128, 0, 0, 0, 0, 128, 32640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2093, 0, 4091, 0, 0, 2, 48, 10, 31, 0, 0, 0, 12, 0, 0, 0, 4072, 0, 0, 4082, 1986, 0, 4, 0, 0, 4024, 0, 4054, 4062, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 107] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1094, 0, 4077, 0, 0, 1392, 324, 1, 15, 0, 0, 0, 108, 0, 0, 0, 3618, 0, 0, 4085, 1573, 0, 50, 0, 0, 3462, 0, 3919, 4072, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 626] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3654, 0, 4094, 0, 0, 360, 51, 5, 5, 0, 0, 0, 15, 0, 0, 0, 3545, 0, 0, 3894, 598, 0, 133, 0, 0, 4059, 0, 4094, 4096, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2098, 0, 4096, 0, 0, 912, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 4059, 0, 0, 4059, 59, 0, 24, 0, 0, 3847, 0, 4019, 4096, 442, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 894] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3011, 0, 4039, 0, 0, 131, 0, 0, 7, 0, 0, 0, 7, 0, 0, 0, 4065, 0, 0, 3854, 1650, 0, 4, 0, 0, 4080, 0, 3052, 4095, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 640] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4065, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3967, 0, 4093, 0, 0, 342, 56, 0, 10, 0, 0, 0, 15, 0, 0, 0, 4081, 0, 0, 3202, 675, 0, 79, 0, 0, 3903, 0, 3894, 4083, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[12, 6, 0, 4, 0, 3, 4084, 0, 0, 0, 0, 0, 2, 15, 0, 0, 0, 0, 0, 5, 1, 3802, 0, 4069, 61, 5, 256, 250, 0, 0, 9, 10, 24, 1101, 0, 32, 7, 4082, 12, 11, 2587, 347, 2, 0, 0, 0, 3806, 0, 3881, 4083, 7, 0, 0, 0, 0, 147, 0, 0, 0, 0, 0, 0, 0, 45] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4072, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3153, 0, 4094, 0, 0, 1346, 4, 0, 9, 0, 0, 0, 2, 0, 0, 0, 4003, 0, 0, 3455, 432, 0, 0, 0, 0, 4038, 0, 3920, 4096, 60, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 34] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4068, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3654, 0, 1240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 360, 51, 5, 5, 0, 0, 2854, 541, 35, 2, 0, 0, 0, 0, 0, 15, 0, 0, 0, 3528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3353, 598, 0, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 74, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 3939, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4107, 4894, 3853, 3543, 4084, 4198, 4006] avg=4096.0000 max=4894 max_rank=2 min=3543 min_rank=4 imbalance_ratio=1.194824 max_load_reduction=27874 max_load_reduction_pct=0.850647 moved_to_echo=15831 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4070, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3988, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2093, 0, 1240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 48, 10, 31, 0, 0, 2851, 568, 33, 3, 0, 0, 0, 0, 0, 12, 0, 0, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3514, 1986, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 74, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 107, 3907, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4073, 3333, 3546, 4063, 5504, 4146, 4015] avg=4096.0000 max=5504 max_rank=5 min=3333 min_rank=2 imbalance_ratio=1.343750 max_load_reduction=27264 max_load_reduction_pct=0.832031 moved_to_echo=17728 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4052, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3998, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1094, 0, 1235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1392, 324, 1, 15, 0, 0, 2842, 568, 29, 2, 0, 0, 0, 0, 0, 108, 0, 0, 0, 3600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3517, 1573, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3919, 74, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 626, 3360, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4071, 2329, 5173, 3708, 5140, 4293, 3986] avg=4096.0000 max=5173 max_rank=3 min=2329 min_rank=2 imbalance_ratio=1.262939 max_load_reduction=27595 max_load_reduction_pct=0.842133 moved_to_echo=17127 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2098, 0, 1241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 912, 0, 0, 0, 0, 0, 2855, 564, 33, 3, 0, 0, 0, 0, 0, 62, 0, 0, 0, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3495, 59, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4019, 74, 442, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 894, 3734, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4102, 3339, 4367, 4100, 3578, 4540, 4628] avg=4096.0000 max=4628 max_rank=7 min=3339 min_rank=2 imbalance_ratio=1.129883 max_load_reduction=28140 max_load_reduction_pct=0.858765 moved_to_echo=16846 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4092, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4021, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3011, 0, 1224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 0, 0, 7, 0, 0, 2815, 536, 34, 3, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3318, 1650, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3052, 74, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 640, 3959, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4108, 4235, 3526, 4051, 4972, 3163, 4603] avg=4096.0000 max=4972 max_rank=5 min=3163 min_rank=6 imbalance_ratio=1.213867 max_load_reduction=27796 max_load_reduction_pct=0.848267 moved_to_echo=16789 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[12, 6, 0, 4, 0, 3, 4084, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 15, 0, 0, 4009, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 3802, 0, 1233, 0, 0, 0, 0, 0, 0, 0, 0, 61, 5, 256, 250, 0, 0, 9, 10, 2836, 360, 33, 3, 0, 0, 0, 0, 24, 1101, 0, 32, 7, 4061, 12, 11, 0, 0, 0, 0, 0, 0, 0, 0, 2227, 347, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3881, 74, 7, 0, 0, 0, 0, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 3693, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4127, 4106, 5041, 3823, 5248, 2576, 4109, 3738] avg=4096.0000 max=5248 max_rank=4 min=2576 min_rank=5 imbalance_ratio=1.281250 max_load_reduction=27520 max_load_reduction_pct=0.839844 moved_to_echo=15671 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4065, 2, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4012, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3967, 0, 1237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 342, 56, 0, 10, 0, 0, 2856, 449, 32, 5, 0, 0, 0, 0, 0, 15, 0, 0, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2753, 675, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3894, 71, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 3788, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4095, 5204, 3750, 4070, 3507, 4216, 3838] avg=4096.0000 max=5204 max_rank=2 min=3507 min_rank=5 imbalance_ratio=1.270508 max_load_reduction=27564 max_load_reduction_pct=0.841187 moved_to_echo=15631 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4072, 1, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3153, 0, 1240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1346, 4, 0, 9, 0, 0, 2854, 480, 34, 3, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2975, 432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3920, 74, 60, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 3920, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4106, 4393, 4730, 3985, 3407, 4103, 3954] avg=4096.0000 max=4730 max_rank=3 min=3407 min_rank=5 imbalance_ratio=1.154785 max_load_reduction=28038 max_load_reduction_pct=0.855652 moved_to_echo=15449 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 128, 22912, 0, 9984, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33792 observed_loss=-1024 tokens_per_expert=[128, 128, 4864, 768, 128, 128, 128, 128, 22784, 4096, 384, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[25216, 7424, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 128, 0, 2560, 30336, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 1408, 0, 128, 128, 31360, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[30848, 640, 1152, 0, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 0, 128, 32640, 128, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 128, 32128, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33792 slack_slots=31744 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4077, 4096, 0, 0, 26, 0, 0, 15, 0, 0, 0, 0, 3, 4077, 0, 0, 4079, 0, 0, 0, 4, 0, 58, 0, 0, 0, 0, 63, 4088, 0, 0, 0, 776, 0, 0, 0, 18, 0, 0, 3221, 0, 4089, 4024, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4078, 4096, 0, 0, 3, 4, 0, 6, 0, 0, 0, 0, 0, 4092, 0, 0, 4035, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 172, 4096, 0, 0, 0, 6, 0, 0, 0, 2, 0, 0, 3885, 0, 4096, 4066, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 4096, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 3010, 0, 4096, 4083, 0, 0, 1, 0, 0, 0, 0, 978, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4050, 4096, 0, 0, 35, 6, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4083, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 11, 4093, 0, 0, 0, 216, 0, 0, 0, 6, 0, 0, 3477, 0, 4081, 3807, 0, 0, 0, 0, 0, 0, 0, 685, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4088, 4096, 0, 0, 0, 7, 0, 6, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 233, 4096, 0, 0, 0, 100, 0, 0, 0, 1, 0, 0, 2816, 0, 4090, 3559, 0, 0, 0, 0, 0, 0, 0, 1480, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4019, 4096, 0, 0, 15, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 644, 4096, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 2202, 0, 4096, 4049, 0, 0, 85, 0, 0, 0, 0, 1270, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4082, 3989, 3, 11, 0, 4, 5, 3, 0, 0, 0, 0, 0, 4081, 0, 0, 4080, 9, 0, 0, 0, 0, 0, 0, 0, 11, 2, 3493, 4083, 0, 0, 7, 14, 51, 0, 6, 0, 0, 5, 437, 0, 4084, 4080, 13, 0, 0, 0, 0, 0, 0, 177, 0, 13, 0, 0, 10, 0, 0, 7, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 4096, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 4094, 0, 0, 0, 108, 0, 0, 0, 0, 0, 0, 4040, 0, 4081, 3911, 0, 0, 0, 0, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4078, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 6, 0, 0, 0, 0, 4072, 6, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 172, 3505, 0, 0, 0, 0, 0, 0, 0, 3927, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3885, 0, 0, 1241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 2825, 563, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4091, 4053, 3680, 3933, 5128, 3685, 4098] avg=4096.0000 max=5128 max_rank=5 min=3680 min_rank=3 imbalance_ratio=1.251953 max_load_reduction=27640 max_load_reduction_pct=0.843506 moved_to_echo=16844 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4077, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 15, 0, 0, 0, 0, 4057, 6, 0, 0, 0, 0, 0, 0, 3, 18, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 58, 0, 0, 0, 0, 63, 3505, 0, 0, 0, 0, 0, 0, 0, 3920, 0, 0, 0, 776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 3221, 0, 0, 1228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 2796, 563, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4104, 4100, 3630, 4696, 4467, 3581, 4091] avg=4096.0000 max=4696 max_rank=4 min=3581 min_rank=6 imbalance_ratio=1.146484 max_load_reduction=28072 max_load_reduction_pct=0.856689 moved_to_echo=16835 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 6, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 3503, 0, 0, 0, 0, 0, 0, 0, 3927, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3010, 0, 0, 1246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 978, 0, 2837, 563, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4082, 4114, 3572, 3980, 4256, 4548, 4098] avg=4096.0000 max=4548 max_rank=6 min=3572 min_rank=3 imbalance_ratio=1.110352 max_load_reduction=28220 max_load_reduction_pct=0.861206 moved_to_echo=16882 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4050, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 6, 0, 0, 0, 0, 0, 0, 4075, 6, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 11, 3505, 0, 0, 0, 0, 0, 0, 0, 3924, 0, 0, 0, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 3477, 0, 0, 1162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 685, 0, 2645, 563, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4122, 4101, 3543, 4140, 4645, 4062, 4083] avg=4096.0000 max=4645 max_rank=5 min=3543 min_rank=3 imbalance_ratio=1.134033 max_load_reduction=28123 max_load_reduction_pct=0.858246 moved_to_echo=16930 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4088, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 6, 0, 0, 0, 0, 4076, 6, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 233, 3505, 0, 0, 0, 0, 0, 0, 0, 3927, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2816, 0, 0, 1086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1480, 0, 2473, 563, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4095, 4114, 3741, 4027, 3903, 4685, 4092] avg=4096.0000 max=4685 max_rank=6 min=3741 min_rank=3 imbalance_ratio=1.143799 max_load_reduction=28083 max_load_reduction_pct=0.857025 moved_to_echo=16707 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4082, 22, 3, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, 3, 0, 0, 0, 0, 4061, 6, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 4080, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 2, 3493, 3413, 0, 0, 0, 0, 0, 0, 0, 3915, 0, 0, 7, 14, 51, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 437, 0, 0, 1245, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177, 0, 2835, 548, 168, 0, 0, 0, 0, 0, 13, 0, 0, 10, 0, 0, 7, 7, 4084, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4079, 4107, 6919, 3993, 1700, 3728, 4123] avg=4096.0000 max=6919 max_rank=3 min=1700 min_rank=5 imbalance_ratio=1.689209 max_load_reduction=25849 max_load_reduction_pct=0.788849 moved_to_echo=13544 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4019, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 1, 0, 0, 0, 0, 4082, 9, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 644, 3506, 0, 0, 0, 0, 0, 0, 0, 3924, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2202, 0, 0, 1233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 1270, 0, 2816, 568, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4032, 4107, 4067, 4150, 3966, 3435, 4911, 4100] avg=4096.0000 max=4911 max_rank=6 min=3435 min_rank=5 imbalance_ratio=1.198975 max_load_reduction=27857 max_load_reduction_pct=0.850128 moved_to_echo=16412 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4076, 6, 0, 0, 0, 0, 0, 0, 0, 18, 0, 1, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 3505, 0, 0, 0, 0, 0, 0, 0, 3925, 0, 0, 0, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4040, 0, 0, 1194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 0, 2717, 563, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4117, 4088, 4112, 3533, 4033, 5234, 3568, 4083] avg=4096.0000 max=5234 max_rank=5 min=3533 min_rank=3 imbalance_ratio=1.277832 max_load_reduction=27534 max_load_reduction_pct=0.840271 moved_to_echo=16918 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 23168, 0, 0, 9728, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 0, 0, 4992, 0, 22016, 4608, 1408, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 0, 0, 128, 128, 4736, 28032, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 32640, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 0, 128, 0, 0, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[31488, 0, 0, 128, 1408, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 256, 0, 128, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 0, 0, 0, 0, 32640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 3753, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 3898, 0, 0, 4095, 0, 0, 16, 0, 0, 0, 0, 325, 260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 3998, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 3136, 0, 0, 4093, 0, 0, 25, 0, 0, 0, 0, 1059, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 1, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3842, 0, 0, 3978, 0, 0, 0, 0, 0, 0, 0, 164, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 1, 3866, 0, 0, 10, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3993, 0, 0, 4096, 0, 0, 37, 0, 0, 0, 0, 49, 273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4072, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2826, 0, 0, 4031, 0, 0, 1, 0, 0, 0, 0, 1199, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3999, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4086, 0, 31, 0, 0, 0, 4078, 0, 12, 1, 0, 2, 4045, 0, 0, 0, 0, 0, 278, 6, 0, 0, 0, 11, 11, 0, 0, 4, 0, 0, 0, 0, 0, 4078, 20, 10, 16, 2646, 26, 0, 4076, 0, 13, 26, 1, 11, 0, 0, 1316, 1, 0, 0, 0, 0, 0, 24, 0, 0, 0, 3859, 4080, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 1, 4046, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 4062, 0, 0, 4093, 0, 0, 2, 0, 0, 0, 0, 10, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4067, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3706, 0, 0, 4016, 4, 0, 20, 0, 0, 0, 0, 532, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3955, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3753, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4019, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 576, 0, 0, 0, 3898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 16, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 325, 260, 0, 0, 0, 0, 3349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3845, 4101, 4060, 4094, 4474, 4124, 3934, 4136] avg=4096.0000 max=4474 max_rank=4 min=3845 min_rank=0 imbalance_ratio=1.092285 max_load_reduction=28294 max_load_reduction_pct=0.863464 moved_to_echo=16668 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3998, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4019, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 578, 0, 0, 0, 3136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 25, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 43, 0, 0, 0, 0, 3365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4101, 4061, 4094, 3714, 4131, 4467, 4110] avg=4096.0000 max=4467 max_rank=6 min=3714 min_rank=4 imbalance_ratio=1.090576 max_load_reduction=28301 max_load_reduction_pct=0.863678 moved_to_echo=16422 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 3866, 0, 0, 10, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4019, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 580, 0, 0, 0, 3993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 37, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 273, 0, 0, 0, 0, 3369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3969, 4101, 4061, 4094, 4573, 4146, 3691, 4133] avg=4096.0000 max=4573 max_rank=4 min=3691 min_rank=6 imbalance_ratio=1.116455 max_load_reduction=28195 max_load_reduction_pct=0.860443 moved_to_echo=16543 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4096, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4019, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 580, 0, 0, 0, 3842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3978, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 204, 0, 0, 0, 0, 3369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4189, 4101, 4064, 4094, 4422, 3991, 3737, 4170] avg=4096.0000 max=4422 max_rank=4 min=3737 min_rank=6 imbalance_ratio=1.079590 max_load_reduction=28346 max_load_reduction_pct=0.865051 moved_to_echo=16320 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4072, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4019, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 580, 0, 0, 0, 2826, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4031, 0, 0, 1, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1199, 256, 0, 0, 0, 0, 3369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3999, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4164, 4101, 4061, 4094, 3406, 4045, 4824, 4073] avg=4096.0000 max=4824 max_rank=6 min=3406 min_rank=4 imbalance_ratio=1.177734 max_load_reduction=27944 max_load_reduction_pct=0.852783 moved_to_echo=16348 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4086, 0, 31, 0, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 5, 0, 12, 1, 0, 2, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 278, 6, 0, 0, 4003, 42, 0, 0, 0, 0, 0, 0, 0, 11, 11, 0, 0, 4, 0, 0, 4073, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 576, 20, 10, 16, 2646, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 4076, 0, 13, 26, 1, 11, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1316, 1, 0, 0, 0, 0, 3355, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 3859, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4209, 4065, 4329, 4102, 3268, 4166, 4672, 3957] avg=4096.0000 max=4672 max_rank=6 min=3268 min_rank=4 imbalance_ratio=1.140625 max_load_reduction=28096 max_load_reduction_pct=0.857422 moved_to_echo=16063 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4046, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4024, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 563, 0, 0, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 2, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 82, 0, 0, 0, 0, 3366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4143, 4097, 4071, 4102, 4625, 4112, 3458, 4160] avg=4096.0000 max=4625 max_rank=4 min=3458 min_rank=6 imbalance_ratio=1.129150 max_load_reduction=28143 max_load_reduction_pct=0.858856 moved_to_echo=16355 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4067, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4019, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 580, 0, 0, 0, 3706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4016, 4, 0, 20, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 84, 0, 0, 0, 0, 3369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3955, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4159, 4101, 4061, 4094, 4286, 4053, 3985, 4029] avg=4096.0000 max=4286 max_rank=4 min=3985 min_rank=6 imbalance_ratio=1.046387 max_load_reduction=28482 max_load_reduction_pct=0.869202 moved_to_echo=16353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 0, 4736, 128, 128, 128, 28160, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 4736, 1280, 0, 0, 0, 0, 27008, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 0, 32256, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 384, 128, 0, 0, 32256, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 32512, 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 32000, 0, 128, 128, 0, 0, 768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 0, 0, 128, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True + [2026-09-21 08:24:27] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 290.3 | throughput per GPU (TFLOP/s/GPU): 111.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.591475E+00 | loss scale: 1.0 | grad norm: 3.425 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-09-21 08:24:27 +[rank0]:[W921 08:24:28.333732974 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0921 08:24:28.440000 632486 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0921 08:24:28.441000 632486 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 632579 closing signal SIGTERM +W0921 08:24:28.442000 632486 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 632580 closing signal SIGTERM +W0921 08:24:28.443000 632486 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 632581 closing signal SIGTERM +W0921 08:24:28.443000 632486 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 632582 closing signal SIGTERM +W0921 08:24:28.444000 632486 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 632583 closing signal SIGTERM +W0921 08:24:28.445000 632486 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 632584 closing signal SIGTERM +W0921 08:24:28.446000 632486 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 632585 closing signal SIGTERM +W0921 08:24:28.446000 632486 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 632586 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 632486 got signal: 15 diff --git a/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv new file mode 100644 index 0000000000000000000000000000000000000000..9c50ab5e6d2ed33074e11226b47d968968c404a9 --- /dev/null +++ b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv @@ -0,0 +1,4 @@ +model backend purpose capacity status peak_mib zero_drop drop_rate_pct required_capacity run_dir +OLMOE_1B_7B_L16 echo measure 2.00 OK 41651 1 0.0 1.03515625 /workspace/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__cap2p00/rep1 +OLMOE_1B_7B_L16 echo validate 1.08984375 OK 38205 1 0.0 1.03515625 /workspace/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__cap1p08984375/rep1 +OLMOE_1B_7B_L16 echo forced 1.08984375 OK 38209 1 0.0 1.03515625 /workspace/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__cap1p08984375__forcebal/rep1 diff --git a/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv new file mode 100644 index 0000000000000000000000000000000000000000..d0023eb7f97937f9192ec68d2e19e7a866a39853 --- /dev/null +++ b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv @@ -0,0 +1,2 @@ +model backend status selected_capacity required_capacity drop_rate_pct peak_mib run_dir +OLMOE_1B_7B_L16 echo SELECTED 1.08984375 1.03515625 0.0 38205 /workspace/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__cap1p08984375/rep1 diff --git a/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/probe_summary.json b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/probe_summary.json new file mode 100644 index 0000000000000000000000000000000000000000..b4b2b65f20a9e37dbb65ff57836171f3bcd64d18 --- /dev/null +++ b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/probe_summary.json @@ -0,0 +1,266 @@ +[ + { + "arm": "natural", + "assignment_sum_mismatches": 0, + "aux_loss_coeff": "1e-2", + "backend": "echo", + "branch": "token-drop-analysis", + "capacity": "1.08984375", + "capacity_mode": "static", + "cold_expert_fraction_max": 0.515625, + "cold_expert_fraction_mean": 0.18001302083333334, + "cold_expert_fraction_min": 0.0, + "cold_expert_fraction_n": 192, + "commit": "b099fd84", + "config": "OLMOE_1B_7B_L16", + "cuda_graph": "0", + "dense_first_layers": "0", + "drop_accounting_source": "dispatch", + "drop_expected_assignments": 50331648.0, + "drop_rate_pct_overall": 0.0, + "echo_logged_load_reduction_valid": 0, + "echo_logged_pre_rank_metric_valid": 0, + "echo_moved_assignment_fraction_max": 0.5, + "echo_moved_assignment_fraction_mean": 0.5, + "echo_moved_assignment_fraction_min": 0.5, + "echo_moved_assignment_fraction_n": 192, + "experts_per_rank": "8", + "force_balance": "0", + "global_batch_size": "8", + "global_snapshots": 192, + "hottest_expert_token_fraction_max": 0.99993896484375, + "hottest_expert_token_fraction_mean": 0.9199026425679525, + "hottest_expert_token_fraction_min": 0.37872314453125, + "hottest_expert_token_fraction_n": 192, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "16", + "num_experts": "64", + "num_layers": "16", + "observed_dropped_tokens": 0, + "post_physical_rank_cv_max": 0.0, + "post_physical_rank_cv_mean": 0.0, + "post_physical_rank_cv_min": 0.0, + "post_physical_rank_cv_n": 192, + "post_physical_rank_imax_max": 1.0, + "post_physical_rank_imax_mean": 1.0, + "post_physical_rank_imax_min": 1.0, + "post_physical_rank_imax_n": 192, + "pre_home_rank_cv_max": 1.3013855491470965, + "pre_home_rank_cv_mean": 0.6770195310504833, + "pre_home_rank_cv_min": 0.19854123422407383, + "pre_home_rank_cv_n": 192, + "pre_home_rank_imax_max": 3.934173583984375, + "pre_home_rank_imax_mean": 2.185980478922526, + "pre_home_rank_imax_min": 1.308837890625, + "pre_home_rank_imax_n": 192, + "raw_expert_cv_max": 2.6267996370932805, + "raw_expert_cv_mean": 2.0626226931661713, + "raw_expert_cv_min": 0.8849421129348258, + "raw_expert_cv_n": 192, + "raw_expert_imax_ceiling": 8.0, + "raw_expert_imax_hit_ceiling": 1, + "raw_expert_imax_max": 7.99951171875, + "raw_expert_imax_mean": 7.35922114054362, + "raw_expert_imax_min": 3.02978515625, + "raw_expert_imax_n": 192, + "records": { + "drop_dispatch": 1536, + "drop_experts": 1536, + "drop_setup": 1536, + "echo_post": 1536, + "echo_pre": 1536 + }, + "required_capacity_max": 1.03515625, + "required_capacity_mean": 1.0187225341796875, + "required_capacity_min": 1.00390625, + "required_capacity_n": 1536, + "router_balancing": "aux_loss", + "run_dir": "OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__cap1p08984375/rep1", + "saturated_dispatch_rows": 0, + "seq_length": "4096", + "shared_expert_overlap": "0", + "topk": "8", + "valid_max_rank_reduction_max": 0.7458170112089362, + "valid_max_rank_reduction_mean": 0.515601266927917, + "valid_max_rank_reduction_min": 0.23596343965678046, + "valid_max_rank_reduction_n": 192, + "zero_drop": 1 + }, + { + "arm": "forced", + "assignment_sum_mismatches": 0, + "aux_loss_coeff": "1e-2", + "backend": "echo", + "branch": "token-drop-analysis", + "capacity": "1.08984375", + "capacity_mode": "static", + "cold_expert_fraction_max": 0.0, + "cold_expert_fraction_mean": 0.0, + "cold_expert_fraction_min": 0.0, + "cold_expert_fraction_n": 192, + "commit": "b099fd84", + "config": "OLMOE_1B_7B_L16", + "cuda_graph": "0", + "dense_first_layers": "0", + "drop_accounting_source": "dispatch", + "drop_expected_assignments": 50331648.0, + "drop_rate_pct_overall": 0.0, + "echo_logged_load_reduction_valid": 0, + "echo_logged_pre_rank_metric_valid": 0, + "echo_moved_assignment_fraction_max": 0.5, + "echo_moved_assignment_fraction_mean": 0.5, + "echo_moved_assignment_fraction_min": 0.5, + "echo_moved_assignment_fraction_n": 192, + "experts_per_rank": "8", + "force_balance": "1", + "global_batch_size": "8", + "global_snapshots": 192, + "hottest_expert_token_fraction_max": 0.13189697265625, + "hottest_expert_token_fraction_mean": 0.12941678365071616, + "hottest_expert_token_fraction_min": 0.1278076171875, + "hottest_expert_token_fraction_n": 192, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "16", + "num_experts": "64", + "num_layers": "16", + "observed_dropped_tokens": 0, + "post_physical_rank_cv_max": 0.0, + "post_physical_rank_cv_mean": 0.0, + "post_physical_rank_cv_min": 0.0, + "post_physical_rank_cv_n": 192, + "post_physical_rank_imax_max": 1.0, + "post_physical_rank_imax_mean": 1.0, + "post_physical_rank_imax_min": 1.0, + "post_physical_rank_imax_n": 192, + "pre_home_rank_cv_max": 0.01039906345549811, + "pre_home_rank_cv_mean": 0.004694553504518217, + "pre_home_rank_cv_min": 0.001428470427059137, + "pre_home_rank_cv_n": 192, + "pre_home_rank_imax_max": 1.015838623046875, + "pre_home_rank_imax_mean": 1.0072050094604492, + "pre_home_rank_imax_min": 1.002044677734375, + "pre_home_rank_imax_n": 192, + "raw_expert_cv_max": 0.018080370892918116, + "raw_expert_cv_mean": 0.014634272985474298, + "raw_expert_cv_min": 0.010177326341354839, + "raw_expert_cv_n": 192, + "raw_expert_imax_ceiling": 8.0, + "raw_expert_imax_hit_ceiling": 0, + "raw_expert_imax_max": 1.05517578125, + "raw_expert_imax_mean": 1.0353342692057292, + "raw_expert_imax_min": 1.0224609375, + "raw_expert_imax_n": 192, + "records": { + "drop_dispatch": 1536, + "drop_experts": 1536, + "drop_setup": 1536, + "echo_post": 1536, + "echo_pre": 1536 + }, + "required_capacity_max": 1.03515625, + "required_capacity_mean": 1.0179494222005208, + "required_capacity_min": 1.0078125, + "required_capacity_n": 1536, + "router_balancing": "aux_loss", + "run_dir": "OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__cap1p08984375__forcebal/rep1", + "saturated_dispatch_rows": 0, + "seq_length": "4096", + "shared_expert_overlap": "0", + "topk": "8", + "valid_max_rank_reduction_max": 0.015591672424670291, + "valid_max_rank_reduction_mean": 0.007147557943940151, + "valid_max_rank_reduction_min": 0.002040505558093498, + "valid_max_rank_reduction_n": 192, + "zero_drop": 1 + }, + { + "arm": "natural", + "assignment_sum_mismatches": 0, + "aux_loss_coeff": "1e-2", + "backend": "echo", + "branch": "token-drop-analysis", + "capacity": "2.00", + "capacity_mode": "static", + "cold_expert_fraction_max": 0.515625, + "cold_expert_fraction_mean": 0.18196614583333334, + "cold_expert_fraction_min": 0.0, + "cold_expert_fraction_n": 192, + "commit": "b099fd84", + "config": "OLMOE_1B_7B_L16", + "cuda_graph": "0", + "dense_first_layers": "0", + "drop_accounting_source": "dispatch", + "drop_expected_assignments": 50331648.0, + "drop_rate_pct_overall": 0.0, + "echo_logged_load_reduction_valid": 0, + "echo_logged_pre_rank_metric_valid": 0, + "echo_moved_assignment_fraction_max": 0.5, + "echo_moved_assignment_fraction_mean": 0.5, + "echo_moved_assignment_fraction_min": 0.5, + "echo_moved_assignment_fraction_n": 192, + "experts_per_rank": "8", + "force_balance": "0", + "global_batch_size": "8", + "global_snapshots": 192, + "hottest_expert_token_fraction_max": 0.99993896484375, + "hottest_expert_token_fraction_mean": 0.9194596608479818, + "hottest_expert_token_fraction_min": 0.381439208984375, + "hottest_expert_token_fraction_n": 192, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "16", + "num_experts": "64", + "num_layers": "16", + "observed_dropped_tokens": 0, + "post_physical_rank_cv_max": 0.0, + "post_physical_rank_cv_mean": 0.0, + "post_physical_rank_cv_min": 0.0, + "post_physical_rank_cv_n": 192, + "post_physical_rank_imax_max": 1.0, + "post_physical_rank_imax_mean": 1.0, + "post_physical_rank_imax_min": 1.0, + "post_physical_rank_imax_n": 192, + "pre_home_rank_cv_max": 1.302987648047582, + "pre_home_rank_cv_mean": 0.6782627371565456, + "pre_home_rank_cv_min": 0.21383062903722408, + "pre_home_rank_cv_n": 192, + "pre_home_rank_imax_max": 3.935455322265625, + "pre_home_rank_imax_mean": 2.1892340977986655, + "pre_home_rank_imax_min": 1.311676025390625, + "pre_home_rank_imax_n": 192, + "raw_expert_cv_max": 2.630844688232933, + "raw_expert_cv_mean": 2.062377122713016, + "raw_expert_cv_min": 0.8852160709247104, + "raw_expert_cv_n": 192, + "raw_expert_imax_ceiling": 8.0, + "raw_expert_imax_hit_ceiling": 1, + "raw_expert_imax_max": 7.99951171875, + "raw_expert_imax_mean": 7.3556772867838545, + "raw_expert_imax_min": 3.051513671875, + "raw_expert_imax_n": 192, + "records": { + "drop_dispatch": 1536, + "drop_experts": 1536, + "drop_setup": 1536, + "echo_post": 1536, + "echo_pre": 1536 + }, + "required_capacity_max": 1.03515625, + "required_capacity_mean": 1.0185597737630208, + "required_capacity_min": 1.00390625, + "required_capacity_n": 1536, + "router_balancing": "aux_loss", + "run_dir": "OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__cap2p00/rep1", + "saturated_dispatch_rows": 0, + "seq_length": "4096", + "shared_expert_overlap": "0", + "topk": "8", + "valid_max_rank_reduction_max": 0.7458997960560497, + "valid_max_rank_reduction_mean": 0.5166162191462842, + "valid_max_rank_reduction_min": 0.23761662129778274, + "valid_max_rank_reduction_n": 192, + "zero_drop": 1 + } +] \ No newline at end of file diff --git a/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/probe_summary.tsv b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/probe_summary.tsv new file mode 100644 index 0000000000000000000000000000000000000000..81759bbf6e0bbc4be21232bf9d2ca3073008f4ae --- /dev/null +++ b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/probe_summary.tsv @@ -0,0 +1,4 @@ +run_dir arm config backend num_experts topk experts_per_rank num_layers moe_layers dense_first_layers seq_length global_batch_size capacity_mode capacity force_balance cuda_graph branch commit router_balancing aux_loss_coeff shared_expert_overlap assignment_sum_mismatches cold_expert_fraction_max cold_expert_fraction_mean cold_expert_fraction_min cold_expert_fraction_n drop_accounting_source drop_expected_assignments drop_rate_pct_overall echo_logged_load_reduction_valid echo_logged_pre_rank_metric_valid echo_moved_assignment_fraction_max echo_moved_assignment_fraction_mean echo_moved_assignment_fraction_min echo_moved_assignment_fraction_n global_snapshots hottest_expert_token_fraction_max hottest_expert_token_fraction_mean hottest_expert_token_fraction_min hottest_expert_token_fraction_n incomplete_snapshots malformed_records observed_dropped_tokens post_physical_rank_cv_max post_physical_rank_cv_mean post_physical_rank_cv_min post_physical_rank_cv_n post_physical_rank_imax_max post_physical_rank_imax_mean post_physical_rank_imax_min post_physical_rank_imax_n pre_home_rank_cv_max pre_home_rank_cv_mean pre_home_rank_cv_min pre_home_rank_cv_n pre_home_rank_imax_max pre_home_rank_imax_mean pre_home_rank_imax_min pre_home_rank_imax_n raw_expert_cv_max raw_expert_cv_mean raw_expert_cv_min raw_expert_cv_n raw_expert_imax_ceiling raw_expert_imax_hit_ceiling raw_expert_imax_max raw_expert_imax_mean raw_expert_imax_min raw_expert_imax_n required_capacity_max required_capacity_mean required_capacity_min required_capacity_n saturated_dispatch_rows valid_max_rank_reduction_max valid_max_rank_reduction_mean valid_max_rank_reduction_min valid_max_rank_reduction_n zero_drop +OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__cap1p08984375/rep1 natural OLMOE_1B_7B_L16 echo 64 8 8 16 16 0 4096 8 static 1.08984375 0 0 token-drop-analysis b099fd84 aux_loss 1e-2 0 0 0.515625 0.18001302083333334 0.0 192 dispatch 50331648.0 0.0 0 0 0.5 0.5 0.5 192 192 0.99993896484375 0.9199026425679525 0.37872314453125 192 0 0 0 0.0 0.0 0.0 192 1.0 1.0 1.0 192 1.3013855491470965 0.6770195310504833 0.19854123422407383 192 3.934173583984375 2.185980478922526 1.308837890625 192 2.6267996370932805 2.0626226931661713 0.8849421129348258 192 8.0 1 7.99951171875 7.35922114054362 3.02978515625 192 1.03515625 1.0187225341796875 1.00390625 1536 0 0.7458170112089362 0.515601266927917 0.23596343965678046 192 1 +OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__cap1p08984375__forcebal/rep1 forced OLMOE_1B_7B_L16 echo 64 8 8 16 16 0 4096 8 static 1.08984375 1 0 token-drop-analysis b099fd84 aux_loss 1e-2 0 0 0.0 0.0 0.0 192 dispatch 50331648.0 0.0 0 0 0.5 0.5 0.5 192 192 0.13189697265625 0.12941678365071616 0.1278076171875 192 0 0 0 0.0 0.0 0.0 192 1.0 1.0 1.0 192 0.01039906345549811 0.004694553504518217 0.001428470427059137 192 1.015838623046875 1.0072050094604492 1.002044677734375 192 0.018080370892918116 0.014634272985474298 0.010177326341354839 192 8.0 0 1.05517578125 1.0353342692057292 1.0224609375 192 1.03515625 1.0179494222005208 1.0078125 1536 0 0.015591672424670291 0.007147557943940151 0.002040505558093498 192 1 +OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__cap2p00/rep1 natural OLMOE_1B_7B_L16 echo 64 8 8 16 16 0 4096 8 static 2.00 0 0 token-drop-analysis b099fd84 aux_loss 1e-2 0 0 0.515625 0.18196614583333334 0.0 192 dispatch 50331648.0 0.0 0 0 0.5 0.5 0.5 192 192 0.99993896484375 0.9194596608479818 0.381439208984375 192 0 0 0 0.0 0.0 0.0 192 1.0 1.0 1.0 192 1.302987648047582 0.6782627371565456 0.21383062903722408 192 3.935455322265625 2.1892340977986655 1.311676025390625 192 2.630844688232933 2.062377122713016 0.8852160709247104 192 8.0 1 7.99951171875 7.3556772867838545 3.051513671875 192 1.03515625 1.0185597737630208 1.00390625 1536 0 0.7458997960560497 0.5166162191462842 0.23761662129778274 192 1 diff --git a/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/results.tsv b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/results.tsv new file mode 100644 index 0000000000000000000000000000000000000000..7d28e5a0b38db746b65a079137b7e5acbb2cb2c4 --- /dev/null +++ b/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/results.tsv @@ -0,0 +1,4 @@ +config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir +OLMOE_1B_7B_L16 echo matched 8 4096 8 2.00 none 1 OK 282.23 264.20 41651 /workspace/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__cap2p00/rep1 +OLMOE_1B_7B_L16 echo matched 8 4096 8 1.08984375 none 1 OK 260.33 253.70 38205 /workspace/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__cap1p08984375/rep1 +OLMOE_1B_7B_L16 echo matched 8 4096 8 1.08984375 none+forcebal 1 OK 264.27 259.20 38209 /workspace/granularity_next_b200/tierb_models/attempts/OLMOE_1B_7B_L16/echo/tierbprobe/tierbprobe_gbs8/OLMOE_1B_7B_L16__echo__matched__seq4096__gbs8__cap1p08984375__forcebal/rep1